feat(open-webui): add image, knowledge and webpage attachments with retry
Extend the AI Chat widget with three ways to ground a conversation:
- Images: attach images to a message, sent inline as OpenAI multimodal
content parts for vision-capable models. A hint warns when the selected
model likely has no vision support.
- Knowledge bases: pick RAG collections exposed by /api/v1/knowledge/.
- Web pages: ingest a URL via /api/v1/retrieval/process/web.
Open WebUI does not run RAG from the chat-completions `files` parameter,
so attached collections (web pages and knowledge bases) are grounded by
retrieving the most relevant chunks at send time
(/api/v1/retrieval/query/collection) and injecting them as system context.
Also add a Retry action to re-run the last turn after a model failure, and
handle the paginated `{ items, total }` shape of the knowledge endpoint on
newer Open WebUI versions.
This commit is contained in:
1 parent
005c2e4cd1
commit
f61e7539d7
8 files changed
+624
-30
No files matched your search
@@ -6,6 +6,7 @@ import { createIntegrationAsync } from "@homarr/integrations";
|
||||
import {
|
||||
openWebUiChatRequestHandler,
|
||||
openWebUiChatsRequestHandler,
|
||||
openWebUiKnowledgeRequestHandler,
|
||||
openWebUiModelsRequestHandler,
|
||||
} from "@homarr/request-handler/open-webui";
|
||||
|
||||
@@ -19,6 +20,18 @@ const messageSchema = z.object({
|
||||
content: z.string(),
|
||||
});
|
||||
|
||||
// Outbound completion messages may carry multimodal parts (text + images),
|
||||
// so users can attach images to a message.
|
||||
const contentPartSchema = z.discriminatedUnion("type", [
|
||||
z.object({ type: z.literal("text"), text: z.string() }),
|
||||
z.object({ type: z.literal("image_url"), image_url: z.object({ url: z.string() }) }),
|
||||
]);
|
||||
|
||||
const completionMessageSchema = z.object({
|
||||
role: z.enum(["system", "user", "assistant"]),
|
||||
content: z.union([z.string(), z.array(contentPartSchema)]),
|
||||
});
|
||||
|
||||
const chatPayloadSchema = z.object({
|
||||
title: z.string(),
|
||||
models: z.array(z.string()),
|
||||
@@ -27,6 +40,47 @@ const chatPayloadSchema = z.object({
|
||||
|
||||
type StreamEvent = { type: "delta"; content: string } | { type: "done" } | { type: "error"; message: string };
|
||||
|
||||
type CompletionMessage = z.infer<typeof completionMessageSchema>;
|
||||
|
||||
// Extract the plain text of a message, flattening multimodal content parts.
|
||||
const messageText = (message: CompletionMessage): string =>
|
||||
typeof message.content === "string"
|
||||
? message.content
|
||||
: message.content
|
||||
.filter((part): part is { type: "text"; text: string } => part.type === "text")
|
||||
.map((part) => part.text)
|
||||
.join(" ");
|
||||
|
||||
// Retrieve relevant chunks from the attached collections and prepend them as a
|
||||
// system context message so the model can ground its reply on them.
|
||||
const groundMessagesAsync = async (
|
||||
client: { queryCollectionsAsync: (names: string[], query: string, k?: number) => Promise<string[]> },
|
||||
messages: CompletionMessage[],
|
||||
collections: string[],
|
||||
): Promise<CompletionMessage[]> => {
|
||||
if (collections.length === 0) return messages;
|
||||
|
||||
const lastUserMessage = messages.findLast((message) => message.role === "user");
|
||||
const query = lastUserMessage ? messageText(lastUserMessage) : "";
|
||||
|
||||
let documents: string[] = [];
|
||||
try {
|
||||
documents = await client.queryCollectionsAsync(collections, query, 4);
|
||||
} catch (error) {
|
||||
logger.warn("Open WebUI context retrieval failed; continuing without it", { error });
|
||||
}
|
||||
if (documents.length === 0) return messages;
|
||||
|
||||
const context = documents.join("\n\n");
|
||||
const contextMessage: CompletionMessage = {
|
||||
role: "system",
|
||||
content:
|
||||
"Use the following context from the user's attached sources (web pages and knowledge bases) " +
|
||||
`to answer. If the answer is not in the context, say so.\n\n${context}`,
|
||||
};
|
||||
return [contextMessage, ...messages];
|
||||
};
|
||||
|
||||
export const openWebUiRouter = createTRPCRouter({
|
||||
getModels: publicProcedure.concat(createOneIntegrationMiddleware("query", "openWebUi")).query(async ({ ctx }) => {
|
||||
const handler = openWebUiModelsRequestHandler.handler(ctx.integration, {});
|
||||
@@ -40,6 +94,22 @@ export const openWebUiRouter = createTRPCRouter({
|
||||
return data;
|
||||
}),
|
||||
|
||||
getKnowledge: publicProcedure.concat(createOneIntegrationMiddleware("query", "openWebUi")).query(async ({ ctx }) => {
|
||||
const handler = openWebUiKnowledgeRequestHandler.handler(ctx.integration, {});
|
||||
const { data } = await handler.getCachedOrUpdatedDataAsync({ forceUpdate: false });
|
||||
return data;
|
||||
}),
|
||||
|
||||
// Ingest a web page so its content can ground later messages. Returns the
|
||||
// collection to retrieve from plus a display title.
|
||||
processWeb: protectedProcedure
|
||||
.input(z.object({ url: z.string().url() }))
|
||||
.concat(createOneIntegrationMiddleware("interact", "openWebUi"))
|
||||
.mutation(async ({ ctx, input }) => {
|
||||
const client = await createIntegrationAsync(ctx.integration);
|
||||
return await client.processWebAsync(input.url);
|
||||
}),
|
||||
|
||||
getChat: publicProcedure
|
||||
.input(z.object({ chatId: z.string() }))
|
||||
.concat(createOneIntegrationMiddleware("query", "openWebUi"))
|
||||
@@ -78,7 +148,15 @@ export const openWebUiRouter = createTRPCRouter({
|
||||
// tokens can be emitted as they arrive. Uses "query" access (same level as
|
||||
// viewing the widget) to match the other integration subscriptions in the app.
|
||||
sendMessage: publicProcedure
|
||||
.input(z.object({ model: z.string(), messages: z.array(messageSchema) }))
|
||||
.input(
|
||||
z.object({
|
||||
model: z.string(),
|
||||
messages: z.array(completionMessageSchema),
|
||||
// Collections to ground the reply on (web pages + knowledge bases). We
|
||||
// retrieve relevant chunks and inject them as context before streaming.
|
||||
collections: z.array(z.string()).optional(),
|
||||
}),
|
||||
)
|
||||
.concat(createOneIntegrationMiddleware("query", "openWebUi"))
|
||||
.subscription(({ ctx, input }) => {
|
||||
return observable<StreamEvent>((emit) => {
|
||||
@@ -87,8 +165,9 @@ export const openWebUiRouter = createTRPCRouter({
|
||||
void (async () => {
|
||||
try {
|
||||
const client = await createIntegrationAsync(ctx.integration);
|
||||
const messages = await groundMessagesAsync(client, input.messages, input.collections ?? []);
|
||||
await client.streamChatCompletionAsync(
|
||||
{ model: input.model, messages: input.messages },
|
||||
{ model: input.model, messages },
|
||||
(delta) => emit.next({ type: "delta", content: delta }),
|
||||
controller.signal,
|
||||
);
|
||||
|
||||
@@ -8,15 +8,20 @@ import type { TestingResult } from "../base/test-connection/test-connection-serv
|
||||
import type {
|
||||
OpenWebUiChat,
|
||||
OpenWebUiChatListItem,
|
||||
OpenWebUiChatMessage,
|
||||
OpenWebUiChatPayload,
|
||||
OpenWebUiCompletionMessage,
|
||||
OpenWebUiKnowledge,
|
||||
OpenWebUiModel,
|
||||
OpenWebUiWebDocument,
|
||||
} from "./open-webui-types";
|
||||
import {
|
||||
openWebUiChatListItemSchema,
|
||||
openWebUiChatSchema,
|
||||
openWebUiCollectionQueryResponseSchema,
|
||||
openWebUiCompletionChunkSchema,
|
||||
openWebUiKnowledgeListSchema,
|
||||
openWebUiModelsResponseSchema,
|
||||
openWebUiProcessWebResponseSchema,
|
||||
} from "./open-webui-types";
|
||||
|
||||
const logger = createLogger({ module: "open-webui-integration" });
|
||||
@@ -49,6 +54,65 @@ export class OpenWebUiIntegration extends Integration {
|
||||
return openWebUiModelsResponseSchema.parse(await response.json()).data;
|
||||
}
|
||||
|
||||
/**
|
||||
* List the knowledge bases (RAG collections) the user can ground chats on.
|
||||
*/
|
||||
public async getKnowledgeAsync(): Promise<OpenWebUiKnowledge[]> {
|
||||
const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/knowledge/"), {
|
||||
headers: this.getAuthHeaders(),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new ResponseError(response);
|
||||
}
|
||||
|
||||
const parsed = openWebUiKnowledgeListSchema.parse(await response.json());
|
||||
return Array.isArray(parsed) ? parsed : parsed.items;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ingest a web page into a temporary vector collection and return a handle to
|
||||
* retrieve from it. Open WebUI does not crawl URLs at completion time, so we
|
||||
* embed the page here and inject retrieved chunks ourselves at send time.
|
||||
*/
|
||||
public async processWebAsync(url: string): Promise<OpenWebUiWebDocument> {
|
||||
const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/retrieval/process/web"), {
|
||||
method: "POST",
|
||||
headers: { ...this.getAuthHeaders(), "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ url, collection_name: "" }),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new ResponseError(response);
|
||||
}
|
||||
|
||||
const parsed = openWebUiProcessWebResponseSchema.parse(await response.json());
|
||||
return { collectionName: parsed.collection_name, title: parsed.file?.meta?.name ?? url };
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve the most relevant chunks for a query across the given collections
|
||||
* (web pages and/or knowledge bases). Returns the chunk texts, best-effort.
|
||||
*/
|
||||
public async queryCollectionsAsync(collectionNames: string[], query: string, k = 4): Promise<string[]> {
|
||||
if (collectionNames.length === 0 || query.trim() === "") {
|
||||
return [];
|
||||
}
|
||||
|
||||
const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/retrieval/query/collection"), {
|
||||
method: "POST",
|
||||
headers: { ...this.getAuthHeaders(), "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ collection_names: collectionNames, query, k }),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new ResponseError(response);
|
||||
}
|
||||
|
||||
const parsed = openWebUiCollectionQueryResponseSchema.parse(await response.json());
|
||||
return (parsed.documents ?? []).flat();
|
||||
}
|
||||
|
||||
/**
|
||||
* List the user's chats (native Open WebUI history).
|
||||
*/
|
||||
@@ -119,7 +183,7 @@ export class OpenWebUiIntegration extends Integration {
|
||||
* content delta. Resolves when the stream completes ([DONE]) or is aborted.
|
||||
*/
|
||||
public async streamChatCompletionAsync(
|
||||
params: { model: string; messages: OpenWebUiChatMessage[] },
|
||||
params: { model: string; messages: OpenWebUiCompletionMessage[] },
|
||||
onDelta: (delta: string) => void,
|
||||
signal: AbortSignal,
|
||||
): Promise<void> {
|
||||
|
||||
@@ -10,6 +10,17 @@ export const openWebUiModelSchema = z.object({
|
||||
name: z.string().nullish(),
|
||||
object: z.string().nullish(),
|
||||
owned_by: z.string().nullish(),
|
||||
// Open WebUI exposes per-model capabilities here when configured; `vision`
|
||||
// tells us whether the model can accept image attachments.
|
||||
info: z
|
||||
.object({
|
||||
meta: z
|
||||
.object({
|
||||
capabilities: z.object({ vision: z.boolean().nullish() }).nullish(),
|
||||
})
|
||||
.nullish(),
|
||||
})
|
||||
.nullish(),
|
||||
});
|
||||
|
||||
export const openWebUiModelsResponseSchema = z.object({
|
||||
@@ -44,6 +55,57 @@ export const openWebUiChatMessageSchema = z.object({
|
||||
|
||||
export type OpenWebUiChatMessage = z.infer<typeof openWebUiChatMessageSchema>;
|
||||
|
||||
// Multimodal content part (OpenAI vision format). Used only for outbound
|
||||
// completion requests so users can attach images to a message.
|
||||
export const openWebUiContentPartSchema = z.discriminatedUnion("type", [
|
||||
z.object({ type: z.literal("text"), text: z.string() }),
|
||||
z.object({ type: z.literal("image_url"), image_url: z.object({ url: z.string() }) }),
|
||||
]);
|
||||
|
||||
// A completion message can carry plain text or multimodal parts (text + images).
|
||||
export const openWebUiCompletionMessageSchema = z.object({
|
||||
role: z.enum(["system", "user", "assistant"]),
|
||||
content: z.union([z.string(), z.array(openWebUiContentPartSchema)]),
|
||||
});
|
||||
|
||||
export type OpenWebUiCompletionMessage = z.infer<typeof openWebUiCompletionMessageSchema>;
|
||||
|
||||
// GET /api/v1/knowledge/ — knowledge bases (RAG collections owned by the user).
|
||||
// Newer Open WebUI versions paginate the list as `{ items, total }`; older ones
|
||||
// return a bare array. Accept both and normalise in the integration.
|
||||
export const openWebUiKnowledgeSchema = z.object({
|
||||
id: z.string(),
|
||||
name: z.string().nullish(),
|
||||
description: z.string().nullish(),
|
||||
});
|
||||
|
||||
export type OpenWebUiKnowledge = z.infer<typeof openWebUiKnowledgeSchema>;
|
||||
|
||||
export const openWebUiKnowledgeListSchema = z.union([
|
||||
z.array(openWebUiKnowledgeSchema),
|
||||
z.object({ items: z.array(openWebUiKnowledgeSchema) }),
|
||||
]);
|
||||
|
||||
// POST /api/v1/retrieval/process/web — ingests a URL into a temporary vector
|
||||
// collection. We retrieve from that collection at send time and inject the
|
||||
// results as context (Open WebUI does not fetch URLs at completion time).
|
||||
export const openWebUiProcessWebResponseSchema = z.object({
|
||||
collection_name: z.string(),
|
||||
filename: z.string().nullish(),
|
||||
file: z.object({ meta: z.object({ name: z.string().nullish() }).nullish() }).nullish(),
|
||||
});
|
||||
|
||||
// POST /api/v1/retrieval/query/collection — top-k chunks per collection.
|
||||
export const openWebUiCollectionQueryResponseSchema = z.object({
|
||||
documents: z.array(z.array(z.string())).nullish(),
|
||||
});
|
||||
|
||||
// A web page ingested for grounding: the collection to retrieve from + a label.
|
||||
export interface OpenWebUiWebDocument {
|
||||
collectionName: string;
|
||||
title: string;
|
||||
}
|
||||
|
||||
// GET /api/v1/chats/ — list of the user's chats (native Open WebUI shape).
|
||||
export const openWebUiChatListItemSchema = z.object({
|
||||
id: z.string(),
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
import dayjs from "dayjs";
|
||||
|
||||
import { createIntegrationAsync } from "@homarr/integrations";
|
||||
import type { OpenWebUiChat, OpenWebUiChatListItem, OpenWebUiModel } from "@homarr/integrations/types";
|
||||
import type {
|
||||
OpenWebUiChat,
|
||||
OpenWebUiChatListItem,
|
||||
OpenWebUiKnowledge,
|
||||
OpenWebUiModel,
|
||||
} from "@homarr/integrations/types";
|
||||
|
||||
import { createCachedIntegrationRequestHandler } from "./lib/cached-integration-request-handler";
|
||||
|
||||
@@ -18,6 +23,19 @@ export const openWebUiModelsRequestHandler = createCachedIntegrationRequestHandl
|
||||
queryKey: "openWebUiModels",
|
||||
});
|
||||
|
||||
export const openWebUiKnowledgeRequestHandler = createCachedIntegrationRequestHandler<
|
||||
OpenWebUiKnowledge[],
|
||||
"openWebUi",
|
||||
Record<string, never>
|
||||
>({
|
||||
async requestAsync(integration) {
|
||||
const integrationInstance = await createIntegrationAsync(integration);
|
||||
return await integrationInstance.getKnowledgeAsync();
|
||||
},
|
||||
cacheDuration: dayjs.duration(5, "minutes"),
|
||||
queryKey: "openWebUiKnowledge",
|
||||
});
|
||||
|
||||
export const openWebUiChatsRequestHandler = createCachedIntegrationRequestHandler<
|
||||
OpenWebUiChatListItem[],
|
||||
"openWebUi",
|
||||
|
||||
@@ -3562,6 +3562,15 @@
|
||||
"back": "Back",
|
||||
"send": "Send",
|
||||
"stop": "Stop",
|
||||
"retry": "Retry",
|
||||
"visionWarning": "This model may not support images",
|
||||
"attach": "Attach",
|
||||
"attachImage": "Attach image",
|
||||
"attachKnowledge": "Knowledge",
|
||||
"attachWebpage": "Webpage",
|
||||
"addUrl": "Add URL",
|
||||
"removeAttachment": "Remove attachment",
|
||||
"webError": "Could not load that web page",
|
||||
"error": {
|
||||
"internalServerError": "Failed to reach Open WebUI"
|
||||
}
|
||||
|
||||
@@ -38,6 +38,20 @@
|
||||
max-width: 85%;
|
||||
}
|
||||
|
||||
.thumb {
|
||||
position: relative;
|
||||
line-height: 0;
|
||||
}
|
||||
|
||||
.thumbRemove {
|
||||
position: absolute;
|
||||
top: -6px;
|
||||
right: -6px;
|
||||
background-color: var(--mantine-color-body);
|
||||
border-radius: var(--mantine-radius-xl);
|
||||
box-shadow: var(--mantine-shadow-xs);
|
||||
}
|
||||
|
||||
.historyOverlay {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
|
||||
@@ -4,26 +4,39 @@ import { useEffect, useMemo, useRef, useState } from "react";
|
||||
import {
|
||||
ActionIcon,
|
||||
Avatar,
|
||||
Badge,
|
||||
Box,
|
||||
Button,
|
||||
Checkbox,
|
||||
CloseButton,
|
||||
Group,
|
||||
Image,
|
||||
Loader,
|
||||
NavLink,
|
||||
Paper,
|
||||
Popover,
|
||||
ScrollArea,
|
||||
Select,
|
||||
Stack,
|
||||
Text,
|
||||
Textarea,
|
||||
TextInput,
|
||||
Tooltip,
|
||||
} from "@mantine/core";
|
||||
import {
|
||||
IconAlertTriangle,
|
||||
IconArrowLeft,
|
||||
IconArrowUp,
|
||||
IconDatabase,
|
||||
IconHistory,
|
||||
IconPaperclip,
|
||||
IconPhoto,
|
||||
IconPlayerStopFilled,
|
||||
IconPlus,
|
||||
IconRefresh,
|
||||
IconRobot,
|
||||
IconUser,
|
||||
IconWorld,
|
||||
} from "@tabler/icons-react";
|
||||
|
||||
import { clientApi } from "@homarr/api/client";
|
||||
@@ -35,8 +48,28 @@ import classes from "./chat.module.css";
|
||||
interface ChatMessage {
|
||||
role: "system" | "user" | "assistant";
|
||||
content: string;
|
||||
images?: string[];
|
||||
}
|
||||
|
||||
type CompletionPart = { type: "text"; text: string } | { type: "image_url"; image_url: { url: string } };
|
||||
type CompletionMessage = { role: ChatMessage["role"]; content: string | CompletionPart[] };
|
||||
|
||||
interface ImageAttachment {
|
||||
name: string;
|
||||
dataUrl: string;
|
||||
}
|
||||
|
||||
interface WebAttachment {
|
||||
url: string;
|
||||
collectionName: string;
|
||||
title: string;
|
||||
}
|
||||
|
||||
// Heuristic fallback for detecting vision-capable models when Open WebUI does
|
||||
// not declare the capability explicitly.
|
||||
const VISION_MODEL_PATTERN =
|
||||
/vision|llava|bakllava|moondream|minicpm-?v|qwen2?-?vl|pixtral|cogvlm|internvl|gemma3|smolvlm|granite[\w-]*vision/i;
|
||||
|
||||
export function Chat({ options, integrationIds, isEditMode }: WidgetComponentProps<"openWebUi">) {
|
||||
const t = useI18n();
|
||||
const integrationId = integrationIds.at(0);
|
||||
@@ -46,16 +79,26 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
|
||||
const [input, setInput] = useState("");
|
||||
const [streamingText, setStreamingText] = useState("");
|
||||
const [isStreaming, setIsStreaming] = useState(false);
|
||||
const [pendingMessages, setPendingMessages] = useState<ChatMessage[]>([]);
|
||||
const [pendingMessages, setPendingMessages] = useState<CompletionMessage[]>([]);
|
||||
const [pendingCollections, setPendingCollections] = useState<string[]>([]);
|
||||
const [activeChatId, setActiveChatId] = useState<string | null>(null);
|
||||
const [selectedChatId, setSelectedChatId] = useState<string | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [historyOpened, setHistoryOpened] = useState(false);
|
||||
|
||||
// Attachments grounding the conversation. Images are per-message; knowledge
|
||||
// collections and web pages are sticky context kept until removed.
|
||||
const [attachments, setAttachments] = useState<ImageAttachment[]>([]);
|
||||
const [knowledgeIds, setKnowledgeIds] = useState<string[]>([]);
|
||||
const [webItems, setWebItems] = useState<WebAttachment[]>([]);
|
||||
const [webInput, setWebInput] = useState("");
|
||||
const [attachOpened, setAttachOpened] = useState(false);
|
||||
|
||||
const messagesRef = useRef<ChatMessage[]>([]);
|
||||
const streamingTextRef = useRef("");
|
||||
const viewportRef = useRef<HTMLDivElement>(null);
|
||||
const composerRef = useRef<HTMLDivElement>(null);
|
||||
const fileInputRef = useRef<HTMLInputElement>(null);
|
||||
const [composerHeight, setComposerHeight] = useState(72);
|
||||
|
||||
const utils = clientApi.useUtils();
|
||||
@@ -65,6 +108,11 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
|
||||
{ enabled: Boolean(integrationId), refetchOnWindowFocus: false, retry: false },
|
||||
);
|
||||
|
||||
const { data: knowledge = [] } = clientApi.widget.openWebUi.getKnowledge.useQuery(
|
||||
{ integrationId: integrationId ?? "" },
|
||||
{ enabled: Boolean(integrationId), refetchOnWindowFocus: false, retry: false },
|
||||
);
|
||||
|
||||
const { data: chats = [] } = clientApi.widget.openWebUi.getChats.useQuery(
|
||||
{ integrationId: integrationId ?? "" },
|
||||
{ enabled: Boolean(integrationId) && options.showHistory, refetchOnWindowFocus: false, retry: false },
|
||||
@@ -77,6 +125,7 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
|
||||
|
||||
const createChat = clientApi.widget.openWebUi.createChat.useMutation();
|
||||
const updateChat = clientApi.widget.openWebUi.updateChat.useMutation();
|
||||
const processWeb = clientApi.widget.openWebUi.processWeb.useMutation();
|
||||
|
||||
// Default to the first available model once the list loads.
|
||||
useEffect(() => {
|
||||
@@ -113,7 +162,12 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
|
||||
}, [messages, streamingText, composerHeight]);
|
||||
|
||||
clientApi.widget.openWebUi.sendMessage.useSubscription(
|
||||
{ integrationId: integrationId ?? "", model: model ?? "", messages: pendingMessages },
|
||||
{
|
||||
integrationId: integrationId ?? "",
|
||||
model: model ?? "",
|
||||
messages: pendingMessages,
|
||||
collections: pendingCollections.length > 0 ? pendingCollections : undefined,
|
||||
},
|
||||
{
|
||||
enabled: isStreaming && pendingMessages.length > 0,
|
||||
onData: (event) => {
|
||||
@@ -146,12 +200,18 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
|
||||
setStreamingText("");
|
||||
setIsStreaming(false);
|
||||
setPendingMessages([]);
|
||||
setPendingCollections([]);
|
||||
};
|
||||
|
||||
const persistChat = async (finalMessages: ChatMessage[]) => {
|
||||
if (!integrationId || !model) return;
|
||||
// Native history stores plain text; image data URLs are dropped on persist.
|
||||
const title = finalMessages.find((message) => message.role === "user")?.content.slice(0, 80) ?? "New chat";
|
||||
const payload = { title, models: [model], messages: finalMessages };
|
||||
const payload = {
|
||||
title,
|
||||
models: [model],
|
||||
messages: finalMessages.map(({ role, content }) => ({ role, content })),
|
||||
};
|
||||
try {
|
||||
if (activeChatId) {
|
||||
await updateChat.mutateAsync({ integrationId, chatId: activeChatId, chat: payload });
|
||||
@@ -167,26 +227,56 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
|
||||
}
|
||||
};
|
||||
|
||||
const handleSend = () => {
|
||||
const trimmed = input.trim();
|
||||
if (!trimmed || !model || !integrationId || isStreaming) return;
|
||||
// Kick off a completion for the given conversation (ending with a user message).
|
||||
// Shared by sending a new message and retrying after a failure.
|
||||
const startCompletion = (baseMessages: ChatMessage[]) => {
|
||||
if (!model || !integrationId) return;
|
||||
setError(null);
|
||||
|
||||
const userMessage: ChatMessage = { role: "user", content: trimmed };
|
||||
const systemPrompt = options.systemPrompt.trim();
|
||||
const payloadMessages: ChatMessage[] = systemPrompt
|
||||
? [{ role: "system", content: systemPrompt }, ...baseMessages]
|
||||
: baseMessages;
|
||||
|
||||
// Knowledge ids double as their collection name; web pages bring their own.
|
||||
const collections = [...knowledgeIds, ...webItems.map((item) => item.collectionName)];
|
||||
|
||||
streamingTextRef.current = "";
|
||||
setStreamingText("");
|
||||
setPendingMessages(toCompletionMessages(payloadMessages));
|
||||
setPendingCollections(collections);
|
||||
setIsStreaming(true);
|
||||
};
|
||||
|
||||
const handleSend = () => {
|
||||
const trimmed = input.trim();
|
||||
if ((!trimmed && attachments.length === 0) || !model || !integrationId || isStreaming) return;
|
||||
|
||||
const userMessage: ChatMessage = {
|
||||
role: "user",
|
||||
content: trimmed,
|
||||
images: attachments.length > 0 ? attachments.map((attachment) => attachment.dataUrl) : undefined,
|
||||
};
|
||||
const nextMessages = [...messagesRef.current, userMessage];
|
||||
setMessages(nextMessages);
|
||||
messagesRef.current = nextMessages;
|
||||
setInput("");
|
||||
setAttachments([]);
|
||||
|
||||
const systemPrompt = options.systemPrompt.trim();
|
||||
const payloadMessages: ChatMessage[] = systemPrompt
|
||||
? [{ role: "system", content: systemPrompt }, ...nextMessages]
|
||||
: nextMessages;
|
||||
startCompletion(nextMessages);
|
||||
};
|
||||
|
||||
streamingTextRef.current = "";
|
||||
setStreamingText("");
|
||||
setPendingMessages(payloadMessages);
|
||||
setIsStreaming(true);
|
||||
// Retry the last turn after a failure: drop any partial assistant reply and
|
||||
// re-run the completion from the most recent user message.
|
||||
const handleRetry = () => {
|
||||
if (isStreaming) return;
|
||||
const lastUserIndex = messagesRef.current.findLastIndex((message) => message.role === "user");
|
||||
if (lastUserIndex === -1) return;
|
||||
|
||||
const truncated = messagesRef.current.slice(0, lastUserIndex + 1);
|
||||
setMessages(truncated);
|
||||
messagesRef.current = truncated;
|
||||
startCompletion(truncated);
|
||||
};
|
||||
|
||||
const handleNewChat = () => {
|
||||
@@ -199,10 +289,56 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
|
||||
setStreamingText("");
|
||||
setIsStreaming(false);
|
||||
setPendingMessages([]);
|
||||
setPendingCollections([]);
|
||||
setAttachments([]);
|
||||
setKnowledgeIds([]);
|
||||
setWebItems([]);
|
||||
setHistoryOpened(false);
|
||||
};
|
||||
|
||||
const handleFilesSelected = async (fileList: FileList | null) => {
|
||||
if (!fileList) return;
|
||||
const images = await Promise.all(
|
||||
Array.from(fileList)
|
||||
.filter((file) => file.type.startsWith("image/"))
|
||||
.map(async (file) => ({ name: file.name, dataUrl: await readFileAsDataUrl(file) })),
|
||||
);
|
||||
setAttachments((previous) => [...previous, ...images]);
|
||||
};
|
||||
|
||||
const addWebUrl = async () => {
|
||||
if (!integrationId) return;
|
||||
const raw = webInput.trim();
|
||||
if (!raw) return;
|
||||
const url = /^https?:\/\//i.test(raw) ? raw : `https://${raw}`;
|
||||
setWebInput("");
|
||||
setError(null);
|
||||
try {
|
||||
const document = await processWeb.mutateAsync({ integrationId, url });
|
||||
setWebItems((previous) =>
|
||||
previous.some((item) => item.collectionName === document.collectionName)
|
||||
? previous
|
||||
: [...previous, { url, collectionName: document.collectionName, title: document.title }],
|
||||
);
|
||||
} catch {
|
||||
setError(t("widget.openWebUi.webError"));
|
||||
}
|
||||
};
|
||||
|
||||
const modelData = useMemo(() => models.map((m) => ({ value: m.id, label: m.name ?? m.id })), [models]);
|
||||
const knowledgeName = (id: string) => knowledge.find((item) => item.id === id)?.name ?? id;
|
||||
const hasAttachments = attachments.length > 0 || knowledgeIds.length > 0 || webItems.length > 0;
|
||||
|
||||
// Whether the selected model can actually see attached images. Prefer Open
|
||||
// WebUI's declared capability; fall back to a name heuristic when it's absent.
|
||||
const selectedModel = useMemo(() => models.find((item) => item.id === model), [models, model]);
|
||||
const modelSupportsVision = useMemo(() => {
|
||||
if (!selectedModel) return true;
|
||||
const capability = selectedModel.info?.meta?.capabilities?.vision;
|
||||
if (typeof capability === "boolean") return capability;
|
||||
return VISION_MODEL_PATTERN.test(selectedModel.id) || VISION_MODEL_PATTERN.test(selectedModel.name ?? "");
|
||||
}, [selectedModel]);
|
||||
const showVisionWarning = attachments.length > 0 && !modelSupportsVision;
|
||||
|
||||
return (
|
||||
<Box className={classes.root} style={isEditMode ? { pointerEvents: "none" } : undefined}>
|
||||
@@ -215,18 +351,92 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
|
||||
</Stack>
|
||||
) : null}
|
||||
{messages.map((message, index) => (
|
||||
<MessageBubble key={index} from={message.role} content={message.content} />
|
||||
<MessageBubble key={index} from={message.role} content={message.content} images={message.images} />
|
||||
))}
|
||||
{isStreaming ? <MessageBubble from="assistant" content={streamingText || "…"} /> : null}
|
||||
{error ? (
|
||||
<Text size="xs" c="red.6">
|
||||
{error}
|
||||
</Text>
|
||||
<Group gap="xs" justify="center" wrap="nowrap">
|
||||
<Text size="xs" c="red.6">
|
||||
{error}
|
||||
</Text>
|
||||
<Button
|
||||
size="compact-xs"
|
||||
variant="light"
|
||||
color="red"
|
||||
leftSection={<IconRefresh size={14} />}
|
||||
onClick={handleRetry}
|
||||
>
|
||||
{t("widget.openWebUi.retry")}
|
||||
</Button>
|
||||
</Group>
|
||||
) : null}
|
||||
</Stack>
|
||||
</ScrollArea>
|
||||
|
||||
<Paper ref={composerRef} className={classes.composer} withBorder radius="md" p="xs">
|
||||
{hasAttachments ? (
|
||||
<Group gap={6} mb={6}>
|
||||
{attachments.map((attachment, index) => (
|
||||
<Box key={`${attachment.name}-${index}`} className={classes.thumb}>
|
||||
<Image src={attachment.dataUrl} alt={attachment.name} w={48} h={48} radius="sm" fit="cover" />
|
||||
<CloseButton
|
||||
size="xs"
|
||||
className={classes.thumbRemove}
|
||||
onClick={() => setAttachments((previous) => previous.filter((_, i) => i !== index))}
|
||||
aria-label={t("widget.openWebUi.removeAttachment")}
|
||||
/>
|
||||
</Box>
|
||||
))}
|
||||
{knowledgeIds.map((id) => (
|
||||
<Badge
|
||||
key={id}
|
||||
variant="light"
|
||||
color="grape"
|
||||
leftSection={<IconDatabase size={12} />}
|
||||
rightSection={
|
||||
<CloseButton
|
||||
size="xs"
|
||||
onClick={() => setKnowledgeIds((previous) => previous.filter((value) => value !== id))}
|
||||
aria-label={t("widget.openWebUi.removeAttachment")}
|
||||
/>
|
||||
}
|
||||
>
|
||||
{knowledgeName(id)}
|
||||
</Badge>
|
||||
))}
|
||||
{webItems.map((item) => (
|
||||
<Badge
|
||||
key={item.collectionName}
|
||||
variant="light"
|
||||
color="blue"
|
||||
leftSection={<IconWorld size={12} />}
|
||||
rightSection={
|
||||
<CloseButton
|
||||
size="xs"
|
||||
onClick={() =>
|
||||
setWebItems((previous) =>
|
||||
previous.filter((value) => value.collectionName !== item.collectionName),
|
||||
)
|
||||
}
|
||||
aria-label={t("widget.openWebUi.removeAttachment")}
|
||||
/>
|
||||
}
|
||||
>
|
||||
{hostnameOf(item.url)}
|
||||
</Badge>
|
||||
))}
|
||||
</Group>
|
||||
) : null}
|
||||
|
||||
{showVisionWarning ? (
|
||||
<Group gap={4} mb={6} wrap="nowrap">
|
||||
<IconAlertTriangle size={14} color="var(--mantine-color-yellow-6)" />
|
||||
<Text size="xs" c="yellow.7">
|
||||
{t("widget.openWebUi.visionWarning")}
|
||||
</Text>
|
||||
</Group>
|
||||
) : null}
|
||||
|
||||
<Textarea
|
||||
variant="unstyled"
|
||||
size="sm"
|
||||
@@ -255,6 +465,86 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
|
||||
<IconPlus size={18} />
|
||||
</ActionIcon>
|
||||
</Tooltip>
|
||||
<Popover
|
||||
opened={attachOpened}
|
||||
onChange={setAttachOpened}
|
||||
position="top-start"
|
||||
withinPortal={false}
|
||||
width={260}
|
||||
shadow="md"
|
||||
>
|
||||
<Popover.Target>
|
||||
<Tooltip label={t("widget.openWebUi.attach")} withinPortal={false}>
|
||||
<ActionIcon
|
||||
variant="subtle"
|
||||
color="gray"
|
||||
onClick={() => setAttachOpened((value) => !value)}
|
||||
aria-label={t("widget.openWebUi.attach")}
|
||||
>
|
||||
<IconPaperclip size={18} />
|
||||
</ActionIcon>
|
||||
</Tooltip>
|
||||
</Popover.Target>
|
||||
<Popover.Dropdown>
|
||||
<Stack gap="sm">
|
||||
<Button
|
||||
variant="light"
|
||||
color="gray"
|
||||
size="xs"
|
||||
leftSection={<IconPhoto size={16} />}
|
||||
justify="flex-start"
|
||||
onClick={() => fileInputRef.current?.click()}
|
||||
>
|
||||
{t("widget.openWebUi.attachImage")}
|
||||
</Button>
|
||||
|
||||
{knowledge.length > 0 ? (
|
||||
<Stack gap={4}>
|
||||
<Text size="xs" fw={600} c="dimmed">
|
||||
{t("widget.openWebUi.attachKnowledge")}
|
||||
</Text>
|
||||
<Checkbox.Group value={knowledgeIds} onChange={setKnowledgeIds}>
|
||||
<Stack gap={4}>
|
||||
{knowledge.map((item) => (
|
||||
<Checkbox key={item.id} value={item.id} label={item.name ?? item.id} size="xs" />
|
||||
))}
|
||||
</Stack>
|
||||
</Checkbox.Group>
|
||||
</Stack>
|
||||
) : null}
|
||||
|
||||
<Stack gap={4}>
|
||||
<Text size="xs" fw={600} c="dimmed">
|
||||
{t("widget.openWebUi.attachWebpage")}
|
||||
</Text>
|
||||
<Group gap={4} wrap="nowrap">
|
||||
<TextInput
|
||||
size="xs"
|
||||
flex={1}
|
||||
placeholder="https://…"
|
||||
value={webInput}
|
||||
onChange={(event) => setWebInput(event.currentTarget.value)}
|
||||
onKeyDown={(event) => {
|
||||
if (event.key === "Enter") {
|
||||
event.preventDefault();
|
||||
void addWebUrl();
|
||||
}
|
||||
}}
|
||||
/>
|
||||
<ActionIcon
|
||||
variant="light"
|
||||
color="gray"
|
||||
loading={processWeb.isPending}
|
||||
onClick={() => void addWebUrl()}
|
||||
aria-label={t("widget.openWebUi.addUrl")}
|
||||
>
|
||||
<IconPlus size={16} />
|
||||
</ActionIcon>
|
||||
</Group>
|
||||
</Stack>
|
||||
</Stack>
|
||||
</Popover.Dropdown>
|
||||
</Popover>
|
||||
{options.showHistory ? (
|
||||
<Tooltip label={t("widget.openWebUi.history")} withinPortal={false}>
|
||||
<ActionIcon
|
||||
@@ -301,7 +591,7 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
|
||||
radius="xl"
|
||||
variant="filled"
|
||||
onClick={handleSend}
|
||||
disabled={!input.trim() || !model}
|
||||
disabled={(!input.trim() && attachments.length === 0) || !model}
|
||||
aria-label={t("widget.openWebUi.send")}
|
||||
>
|
||||
<IconArrowUp size={18} />
|
||||
@@ -310,6 +600,19 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
|
||||
)}
|
||||
</Group>
|
||||
</Group>
|
||||
|
||||
<input
|
||||
ref={fileInputRef}
|
||||
type="file"
|
||||
accept="image/*"
|
||||
multiple
|
||||
hidden
|
||||
aria-label={t("widget.openWebUi.attachImage")}
|
||||
onChange={(event) => {
|
||||
void handleFilesSelected(event.currentTarget.files);
|
||||
event.currentTarget.value = "";
|
||||
}}
|
||||
/>
|
||||
</Paper>
|
||||
|
||||
{options.showHistory && historyOpened ? (
|
||||
@@ -353,6 +656,37 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
|
||||
);
|
||||
}
|
||||
|
||||
const readFileAsDataUrl = (file: File): Promise<string> =>
|
||||
new Promise((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
reader.addEventListener("load", () => resolve(reader.result as string));
|
||||
reader.addEventListener("error", () => reject(reader.error ?? new Error("Failed to read file")));
|
||||
reader.readAsDataURL(file);
|
||||
});
|
||||
|
||||
const hostnameOf = (url: string): string => {
|
||||
try {
|
||||
return new URL(url).hostname || url;
|
||||
} catch {
|
||||
return url;
|
||||
}
|
||||
};
|
||||
|
||||
// Convert displayed messages to the OpenAI completion format, inlining any
|
||||
// attached images on the message that carries them (vision content parts).
|
||||
const toCompletionMessages = (messages: ChatMessage[]): CompletionMessage[] =>
|
||||
messages.map((message) =>
|
||||
message.images && message.images.length > 0
|
||||
? {
|
||||
role: message.role,
|
||||
content: [
|
||||
...(message.content ? [{ type: "text" as const, text: message.content }] : []),
|
||||
...message.images.map((url) => ({ type: "image_url" as const, image_url: { url } })),
|
||||
],
|
||||
}
|
||||
: { role: message.role, content: message.content },
|
||||
);
|
||||
|
||||
type RawMessage = { role?: string | null; content?: string | null };
|
||||
type HistoryMessage = RawMessage & { parentId?: string | null };
|
||||
interface RawChat {
|
||||
@@ -393,7 +727,7 @@ const extractChatMessages = (chat: RawChat | null | undefined): ChatMessage[] =>
|
||||
.filter((message): message is ChatMessage => message !== null);
|
||||
};
|
||||
|
||||
function MessageBubble({ from, content }: { from: ChatMessage["role"]; content: string }) {
|
||||
function MessageBubble({ from, content, images }: { from: ChatMessage["role"]; content: string; images?: string[] }) {
|
||||
const isUser = from === "user";
|
||||
const avatar = (
|
||||
<Avatar size={22} radius="xl" variant="filled" color={isUser ? "gray" : "blue"} className={classes.avatar}>
|
||||
@@ -410,9 +744,18 @@ function MessageBubble({ from, content }: { from: ChatMessage["role"]; content:
|
||||
p="xs"
|
||||
radius="md"
|
||||
>
|
||||
<Text size="sm" style={{ whiteSpace: "pre-wrap", wordBreak: "break-word" }}>
|
||||
{content}
|
||||
</Text>
|
||||
{images && images.length > 0 ? (
|
||||
<Group gap={4} mb={content ? 6 : 0}>
|
||||
{images.map((src, index) => (
|
||||
<Image key={index} src={src} alt="" w={120} radius="sm" fit="contain" />
|
||||
))}
|
||||
</Group>
|
||||
) : null}
|
||||
{content ? (
|
||||
<Text size="sm" style={{ whiteSpace: "pre-wrap", wordBreak: "break-word" }}>
|
||||
{content}
|
||||
</Text>
|
||||
) : null}
|
||||
</Paper>
|
||||
{isUser ? avatar : null}
|
||||
</Group>
|
||||
|
||||
Reference in new issue
Block a user