From f61e7539d7e4c67aa321bf1cf38a0c5e5bcc21ec Mon Sep 17 00:00:00 2001 From: Lok3sh-Kumar Date: Sat, 27 Jun 2026 14:15:08 +0530 Subject: [PATCH] feat(open-webui): add image, knowledge and webpage attachments with retry Extend the AI Chat widget with three ways to ground a conversation: - Images: attach images to a message, sent inline as OpenAI multimodal content parts for vision-capable models. A hint warns when the selected model likely has no vision support. - Knowledge bases: pick RAG collections exposed by /api/v1/knowledge/. - Web pages: ingest a URL via /api/v1/retrieval/process/web. Open WebUI does not run RAG from the chat-completions `files` parameter, so attached collections (web pages and knowledge bases) are grounded by retrieving the most relevant chunks at send time (/api/v1/retrieval/query/collection) and injecting them as system context. Also add a Retry action to re-run the last turn after a model failure, and handle the paginated `{ items, total }` shape of the knowledge endpoint on newer Open WebUI versions. --- apps/docs/docs/widgets/open-webui/index.mdx | 7 +- packages/api/src/router/widgets/open-webui.ts | 83 +++- .../src/open-webui/open-webui-integration.ts | 68 ++- .../src/open-webui/open-webui-types.ts | 62 +++ packages/request-handler/src/open-webui.ts | 20 +- packages/translation/src/lang/en.json | 9 + .../widgets/src/open-webui/chat.module.css | 14 + packages/widgets/src/open-webui/chat.tsx | 391 ++++++++++++++++-- 8 files changed, 624 insertions(+), 30 deletions(-) diff --git a/apps/docs/docs/widgets/open-webui/index.mdx b/apps/docs/docs/widgets/open-webui/index.mdx index f6cede841..88602c95c 100644 --- a/apps/docs/docs/widgets/open-webui/index.mdx +++ b/apps/docs/docs/widgets/open-webui/index.mdx @@ -13,7 +13,7 @@ import { openWebUiIntegration } from "@site/docs/integrations/open-webui"; -The AI Chat widget lets you chat with AI models served by Open WebUI directly from your dashboard. Responses stream in token-by-token, you can pick any model your Open WebUI instance exposes (Ollama, OpenAI, and other configured providers), and you can browse and reopen your Open WebUI chat history. +The AI Chat widget lets you chat with AI models served by Open WebUI directly from your dashboard. Responses stream in token-by-token, you can pick any model your Open WebUI instance exposes (Ollama, OpenAI, and other configured providers), and you can browse and reopen your Open WebUI chat history. You can also ground a conversation by attaching images, knowledge bases, or web pages. ### Supported Integrations @@ -40,8 +40,13 @@ The AI Chat widget lets you chat with AI models served by Open WebUI directly fr | **Model picker** | Lists every model exposed by Open WebUI's `/api/models` endpoint, selectable from the chat input. The first model is selected by default. | | **Streaming responses** | Assistant replies stream in live as they are generated. | | **Chat history** | Browse and reopen previous chats. New conversations are saved back to Open WebUI. | +| **Image attachments** | Attach images to a message for vision-capable models (sent inline as multimodal content). | +| **Knowledge bases** | Ground replies on any RAG knowledge collection (`/api/v1/knowledge/`) your account owns. | +| **Web pages** | Attach a URL and Open WebUI fetches and embeds the page as context for the reply. | ### Notes - The widget uses a single Open WebUI integration. - Conversations are stored in Open WebUI itself, under the account that owns the API key, so they also appear in the Open WebUI app. +- Knowledge bases and attached web pages stay active for the whole conversation until you remove them; images apply only to the message you attach them to. +- Conversation text is saved to Open WebUI history, but attached image data is not persisted. diff --git a/packages/api/src/router/widgets/open-webui.ts b/packages/api/src/router/widgets/open-webui.ts index 36a79b376..1247dba62 100644 --- a/packages/api/src/router/widgets/open-webui.ts +++ b/packages/api/src/router/widgets/open-webui.ts @@ -6,6 +6,7 @@ import { createIntegrationAsync } from "@homarr/integrations"; import { openWebUiChatRequestHandler, openWebUiChatsRequestHandler, + openWebUiKnowledgeRequestHandler, openWebUiModelsRequestHandler, } from "@homarr/request-handler/open-webui"; @@ -19,6 +20,18 @@ const messageSchema = z.object({ content: z.string(), }); +// Outbound completion messages may carry multimodal parts (text + images), +// so users can attach images to a message. +const contentPartSchema = z.discriminatedUnion("type", [ + z.object({ type: z.literal("text"), text: z.string() }), + z.object({ type: z.literal("image_url"), image_url: z.object({ url: z.string() }) }), +]); + +const completionMessageSchema = z.object({ + role: z.enum(["system", "user", "assistant"]), + content: z.union([z.string(), z.array(contentPartSchema)]), +}); + const chatPayloadSchema = z.object({ title: z.string(), models: z.array(z.string()), @@ -27,6 +40,47 @@ const chatPayloadSchema = z.object({ type StreamEvent = { type: "delta"; content: string } | { type: "done" } | { type: "error"; message: string }; +type CompletionMessage = z.infer; + +// Extract the plain text of a message, flattening multimodal content parts. +const messageText = (message: CompletionMessage): string => + typeof message.content === "string" + ? message.content + : message.content + .filter((part): part is { type: "text"; text: string } => part.type === "text") + .map((part) => part.text) + .join(" "); + +// Retrieve relevant chunks from the attached collections and prepend them as a +// system context message so the model can ground its reply on them. +const groundMessagesAsync = async ( + client: { queryCollectionsAsync: (names: string[], query: string, k?: number) => Promise }, + messages: CompletionMessage[], + collections: string[], +): Promise => { + if (collections.length === 0) return messages; + + const lastUserMessage = messages.findLast((message) => message.role === "user"); + const query = lastUserMessage ? messageText(lastUserMessage) : ""; + + let documents: string[] = []; + try { + documents = await client.queryCollectionsAsync(collections, query, 4); + } catch (error) { + logger.warn("Open WebUI context retrieval failed; continuing without it", { error }); + } + if (documents.length === 0) return messages; + + const context = documents.join("\n\n"); + const contextMessage: CompletionMessage = { + role: "system", + content: + "Use the following context from the user's attached sources (web pages and knowledge bases) " + + `to answer. If the answer is not in the context, say so.\n\n${context}`, + }; + return [contextMessage, ...messages]; +}; + export const openWebUiRouter = createTRPCRouter({ getModels: publicProcedure.concat(createOneIntegrationMiddleware("query", "openWebUi")).query(async ({ ctx }) => { const handler = openWebUiModelsRequestHandler.handler(ctx.integration, {}); @@ -40,6 +94,22 @@ export const openWebUiRouter = createTRPCRouter({ return data; }), + getKnowledge: publicProcedure.concat(createOneIntegrationMiddleware("query", "openWebUi")).query(async ({ ctx }) => { + const handler = openWebUiKnowledgeRequestHandler.handler(ctx.integration, {}); + const { data } = await handler.getCachedOrUpdatedDataAsync({ forceUpdate: false }); + return data; + }), + + // Ingest a web page so its content can ground later messages. Returns the + // collection to retrieve from plus a display title. + processWeb: protectedProcedure + .input(z.object({ url: z.string().url() })) + .concat(createOneIntegrationMiddleware("interact", "openWebUi")) + .mutation(async ({ ctx, input }) => { + const client = await createIntegrationAsync(ctx.integration); + return await client.processWebAsync(input.url); + }), + getChat: publicProcedure .input(z.object({ chatId: z.string() })) .concat(createOneIntegrationMiddleware("query", "openWebUi")) @@ -78,7 +148,15 @@ export const openWebUiRouter = createTRPCRouter({ // tokens can be emitted as they arrive. Uses "query" access (same level as // viewing the widget) to match the other integration subscriptions in the app. sendMessage: publicProcedure - .input(z.object({ model: z.string(), messages: z.array(messageSchema) })) + .input( + z.object({ + model: z.string(), + messages: z.array(completionMessageSchema), + // Collections to ground the reply on (web pages + knowledge bases). We + // retrieve relevant chunks and inject them as context before streaming. + collections: z.array(z.string()).optional(), + }), + ) .concat(createOneIntegrationMiddleware("query", "openWebUi")) .subscription(({ ctx, input }) => { return observable((emit) => { @@ -87,8 +165,9 @@ export const openWebUiRouter = createTRPCRouter({ void (async () => { try { const client = await createIntegrationAsync(ctx.integration); + const messages = await groundMessagesAsync(client, input.messages, input.collections ?? []); await client.streamChatCompletionAsync( - { model: input.model, messages: input.messages }, + { model: input.model, messages }, (delta) => emit.next({ type: "delta", content: delta }), controller.signal, ); diff --git a/packages/integrations/src/open-webui/open-webui-integration.ts b/packages/integrations/src/open-webui/open-webui-integration.ts index 6a6fafd43..ff96db259 100644 --- a/packages/integrations/src/open-webui/open-webui-integration.ts +++ b/packages/integrations/src/open-webui/open-webui-integration.ts @@ -8,15 +8,20 @@ import type { TestingResult } from "../base/test-connection/test-connection-serv import type { OpenWebUiChat, OpenWebUiChatListItem, - OpenWebUiChatMessage, OpenWebUiChatPayload, + OpenWebUiCompletionMessage, + OpenWebUiKnowledge, OpenWebUiModel, + OpenWebUiWebDocument, } from "./open-webui-types"; import { openWebUiChatListItemSchema, openWebUiChatSchema, + openWebUiCollectionQueryResponseSchema, openWebUiCompletionChunkSchema, + openWebUiKnowledgeListSchema, openWebUiModelsResponseSchema, + openWebUiProcessWebResponseSchema, } from "./open-webui-types"; const logger = createLogger({ module: "open-webui-integration" }); @@ -49,6 +54,65 @@ export class OpenWebUiIntegration extends Integration { return openWebUiModelsResponseSchema.parse(await response.json()).data; } + /** + * List the knowledge bases (RAG collections) the user can ground chats on. + */ + public async getKnowledgeAsync(): Promise { + const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/knowledge/"), { + headers: this.getAuthHeaders(), + }); + + if (!response.ok) { + throw new ResponseError(response); + } + + const parsed = openWebUiKnowledgeListSchema.parse(await response.json()); + return Array.isArray(parsed) ? parsed : parsed.items; + } + + /** + * Ingest a web page into a temporary vector collection and return a handle to + * retrieve from it. Open WebUI does not crawl URLs at completion time, so we + * embed the page here and inject retrieved chunks ourselves at send time. + */ + public async processWebAsync(url: string): Promise { + const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/retrieval/process/web"), { + method: "POST", + headers: { ...this.getAuthHeaders(), "Content-Type": "application/json" }, + body: JSON.stringify({ url, collection_name: "" }), + }); + + if (!response.ok) { + throw new ResponseError(response); + } + + const parsed = openWebUiProcessWebResponseSchema.parse(await response.json()); + return { collectionName: parsed.collection_name, title: parsed.file?.meta?.name ?? url }; + } + + /** + * Retrieve the most relevant chunks for a query across the given collections + * (web pages and/or knowledge bases). Returns the chunk texts, best-effort. + */ + public async queryCollectionsAsync(collectionNames: string[], query: string, k = 4): Promise { + if (collectionNames.length === 0 || query.trim() === "") { + return []; + } + + const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/retrieval/query/collection"), { + method: "POST", + headers: { ...this.getAuthHeaders(), "Content-Type": "application/json" }, + body: JSON.stringify({ collection_names: collectionNames, query, k }), + }); + + if (!response.ok) { + throw new ResponseError(response); + } + + const parsed = openWebUiCollectionQueryResponseSchema.parse(await response.json()); + return (parsed.documents ?? []).flat(); + } + /** * List the user's chats (native Open WebUI history). */ @@ -119,7 +183,7 @@ export class OpenWebUiIntegration extends Integration { * content delta. Resolves when the stream completes ([DONE]) or is aborted. */ public async streamChatCompletionAsync( - params: { model: string; messages: OpenWebUiChatMessage[] }, + params: { model: string; messages: OpenWebUiCompletionMessage[] }, onDelta: (delta: string) => void, signal: AbortSignal, ): Promise { diff --git a/packages/integrations/src/open-webui/open-webui-types.ts b/packages/integrations/src/open-webui/open-webui-types.ts index f880aae04..8955a2728 100644 --- a/packages/integrations/src/open-webui/open-webui-types.ts +++ b/packages/integrations/src/open-webui/open-webui-types.ts @@ -10,6 +10,17 @@ export const openWebUiModelSchema = z.object({ name: z.string().nullish(), object: z.string().nullish(), owned_by: z.string().nullish(), + // Open WebUI exposes per-model capabilities here when configured; `vision` + // tells us whether the model can accept image attachments. + info: z + .object({ + meta: z + .object({ + capabilities: z.object({ vision: z.boolean().nullish() }).nullish(), + }) + .nullish(), + }) + .nullish(), }); export const openWebUiModelsResponseSchema = z.object({ @@ -44,6 +55,57 @@ export const openWebUiChatMessageSchema = z.object({ export type OpenWebUiChatMessage = z.infer; +// Multimodal content part (OpenAI vision format). Used only for outbound +// completion requests so users can attach images to a message. +export const openWebUiContentPartSchema = z.discriminatedUnion("type", [ + z.object({ type: z.literal("text"), text: z.string() }), + z.object({ type: z.literal("image_url"), image_url: z.object({ url: z.string() }) }), +]); + +// A completion message can carry plain text or multimodal parts (text + images). +export const openWebUiCompletionMessageSchema = z.object({ + role: z.enum(["system", "user", "assistant"]), + content: z.union([z.string(), z.array(openWebUiContentPartSchema)]), +}); + +export type OpenWebUiCompletionMessage = z.infer; + +// GET /api/v1/knowledge/ — knowledge bases (RAG collections owned by the user). +// Newer Open WebUI versions paginate the list as `{ items, total }`; older ones +// return a bare array. Accept both and normalise in the integration. +export const openWebUiKnowledgeSchema = z.object({ + id: z.string(), + name: z.string().nullish(), + description: z.string().nullish(), +}); + +export type OpenWebUiKnowledge = z.infer; + +export const openWebUiKnowledgeListSchema = z.union([ + z.array(openWebUiKnowledgeSchema), + z.object({ items: z.array(openWebUiKnowledgeSchema) }), +]); + +// POST /api/v1/retrieval/process/web — ingests a URL into a temporary vector +// collection. We retrieve from that collection at send time and inject the +// results as context (Open WebUI does not fetch URLs at completion time). +export const openWebUiProcessWebResponseSchema = z.object({ + collection_name: z.string(), + filename: z.string().nullish(), + file: z.object({ meta: z.object({ name: z.string().nullish() }).nullish() }).nullish(), +}); + +// POST /api/v1/retrieval/query/collection — top-k chunks per collection. +export const openWebUiCollectionQueryResponseSchema = z.object({ + documents: z.array(z.array(z.string())).nullish(), +}); + +// A web page ingested for grounding: the collection to retrieve from + a label. +export interface OpenWebUiWebDocument { + collectionName: string; + title: string; +} + // GET /api/v1/chats/ — list of the user's chats (native Open WebUI shape). export const openWebUiChatListItemSchema = z.object({ id: z.string(), diff --git a/packages/request-handler/src/open-webui.ts b/packages/request-handler/src/open-webui.ts index 79fbb34bd..b7306eb58 100644 --- a/packages/request-handler/src/open-webui.ts +++ b/packages/request-handler/src/open-webui.ts @@ -1,7 +1,12 @@ import dayjs from "dayjs"; import { createIntegrationAsync } from "@homarr/integrations"; -import type { OpenWebUiChat, OpenWebUiChatListItem, OpenWebUiModel } from "@homarr/integrations/types"; +import type { + OpenWebUiChat, + OpenWebUiChatListItem, + OpenWebUiKnowledge, + OpenWebUiModel, +} from "@homarr/integrations/types"; import { createCachedIntegrationRequestHandler } from "./lib/cached-integration-request-handler"; @@ -18,6 +23,19 @@ export const openWebUiModelsRequestHandler = createCachedIntegrationRequestHandl queryKey: "openWebUiModels", }); +export const openWebUiKnowledgeRequestHandler = createCachedIntegrationRequestHandler< + OpenWebUiKnowledge[], + "openWebUi", + Record +>({ + async requestAsync(integration) { + const integrationInstance = await createIntegrationAsync(integration); + return await integrationInstance.getKnowledgeAsync(); + }, + cacheDuration: dayjs.duration(5, "minutes"), + queryKey: "openWebUiKnowledge", +}); + export const openWebUiChatsRequestHandler = createCachedIntegrationRequestHandler< OpenWebUiChatListItem[], "openWebUi", diff --git a/packages/translation/src/lang/en.json b/packages/translation/src/lang/en.json index 440f03999..473578cec 100644 --- a/packages/translation/src/lang/en.json +++ b/packages/translation/src/lang/en.json @@ -3562,6 +3562,15 @@ "back": "Back", "send": "Send", "stop": "Stop", + "retry": "Retry", + "visionWarning": "This model may not support images", + "attach": "Attach", + "attachImage": "Attach image", + "attachKnowledge": "Knowledge", + "attachWebpage": "Webpage", + "addUrl": "Add URL", + "removeAttachment": "Remove attachment", + "webError": "Could not load that web page", "error": { "internalServerError": "Failed to reach Open WebUI" } diff --git a/packages/widgets/src/open-webui/chat.module.css b/packages/widgets/src/open-webui/chat.module.css index fd08d54e6..98b6cd903 100644 --- a/packages/widgets/src/open-webui/chat.module.css +++ b/packages/widgets/src/open-webui/chat.module.css @@ -38,6 +38,20 @@ max-width: 85%; } +.thumb { + position: relative; + line-height: 0; +} + +.thumbRemove { + position: absolute; + top: -6px; + right: -6px; + background-color: var(--mantine-color-body); + border-radius: var(--mantine-radius-xl); + box-shadow: var(--mantine-shadow-xs); +} + .historyOverlay { position: absolute; inset: 0; diff --git a/packages/widgets/src/open-webui/chat.tsx b/packages/widgets/src/open-webui/chat.tsx index 8fa10844e..af0257656 100644 --- a/packages/widgets/src/open-webui/chat.tsx +++ b/packages/widgets/src/open-webui/chat.tsx @@ -4,26 +4,39 @@ import { useEffect, useMemo, useRef, useState } from "react"; import { ActionIcon, Avatar, + Badge, Box, + Button, + Checkbox, + CloseButton, Group, + Image, Loader, NavLink, Paper, + Popover, ScrollArea, Select, Stack, Text, Textarea, + TextInput, Tooltip, } from "@mantine/core"; import { + IconAlertTriangle, IconArrowLeft, IconArrowUp, + IconDatabase, IconHistory, + IconPaperclip, + IconPhoto, IconPlayerStopFilled, IconPlus, + IconRefresh, IconRobot, IconUser, + IconWorld, } from "@tabler/icons-react"; import { clientApi } from "@homarr/api/client"; @@ -35,8 +48,28 @@ import classes from "./chat.module.css"; interface ChatMessage { role: "system" | "user" | "assistant"; content: string; + images?: string[]; } +type CompletionPart = { type: "text"; text: string } | { type: "image_url"; image_url: { url: string } }; +type CompletionMessage = { role: ChatMessage["role"]; content: string | CompletionPart[] }; + +interface ImageAttachment { + name: string; + dataUrl: string; +} + +interface WebAttachment { + url: string; + collectionName: string; + title: string; +} + +// Heuristic fallback for detecting vision-capable models when Open WebUI does +// not declare the capability explicitly. +const VISION_MODEL_PATTERN = + /vision|llava|bakllava|moondream|minicpm-?v|qwen2?-?vl|pixtral|cogvlm|internvl|gemma3|smolvlm|granite[\w-]*vision/i; + export function Chat({ options, integrationIds, isEditMode }: WidgetComponentProps<"openWebUi">) { const t = useI18n(); const integrationId = integrationIds.at(0); @@ -46,16 +79,26 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro const [input, setInput] = useState(""); const [streamingText, setStreamingText] = useState(""); const [isStreaming, setIsStreaming] = useState(false); - const [pendingMessages, setPendingMessages] = useState([]); + const [pendingMessages, setPendingMessages] = useState([]); + const [pendingCollections, setPendingCollections] = useState([]); const [activeChatId, setActiveChatId] = useState(null); const [selectedChatId, setSelectedChatId] = useState(null); const [error, setError] = useState(null); const [historyOpened, setHistoryOpened] = useState(false); + // Attachments grounding the conversation. Images are per-message; knowledge + // collections and web pages are sticky context kept until removed. + const [attachments, setAttachments] = useState([]); + const [knowledgeIds, setKnowledgeIds] = useState([]); + const [webItems, setWebItems] = useState([]); + const [webInput, setWebInput] = useState(""); + const [attachOpened, setAttachOpened] = useState(false); + const messagesRef = useRef([]); const streamingTextRef = useRef(""); const viewportRef = useRef(null); const composerRef = useRef(null); + const fileInputRef = useRef(null); const [composerHeight, setComposerHeight] = useState(72); const utils = clientApi.useUtils(); @@ -65,6 +108,11 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro { enabled: Boolean(integrationId), refetchOnWindowFocus: false, retry: false }, ); + const { data: knowledge = [] } = clientApi.widget.openWebUi.getKnowledge.useQuery( + { integrationId: integrationId ?? "" }, + { enabled: Boolean(integrationId), refetchOnWindowFocus: false, retry: false }, + ); + const { data: chats = [] } = clientApi.widget.openWebUi.getChats.useQuery( { integrationId: integrationId ?? "" }, { enabled: Boolean(integrationId) && options.showHistory, refetchOnWindowFocus: false, retry: false }, @@ -77,6 +125,7 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro const createChat = clientApi.widget.openWebUi.createChat.useMutation(); const updateChat = clientApi.widget.openWebUi.updateChat.useMutation(); + const processWeb = clientApi.widget.openWebUi.processWeb.useMutation(); // Default to the first available model once the list loads. useEffect(() => { @@ -113,7 +162,12 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro }, [messages, streamingText, composerHeight]); clientApi.widget.openWebUi.sendMessage.useSubscription( - { integrationId: integrationId ?? "", model: model ?? "", messages: pendingMessages }, + { + integrationId: integrationId ?? "", + model: model ?? "", + messages: pendingMessages, + collections: pendingCollections.length > 0 ? pendingCollections : undefined, + }, { enabled: isStreaming && pendingMessages.length > 0, onData: (event) => { @@ -146,12 +200,18 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro setStreamingText(""); setIsStreaming(false); setPendingMessages([]); + setPendingCollections([]); }; const persistChat = async (finalMessages: ChatMessage[]) => { if (!integrationId || !model) return; + // Native history stores plain text; image data URLs are dropped on persist. const title = finalMessages.find((message) => message.role === "user")?.content.slice(0, 80) ?? "New chat"; - const payload = { title, models: [model], messages: finalMessages }; + const payload = { + title, + models: [model], + messages: finalMessages.map(({ role, content }) => ({ role, content })), + }; try { if (activeChatId) { await updateChat.mutateAsync({ integrationId, chatId: activeChatId, chat: payload }); @@ -167,26 +227,56 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro } }; - const handleSend = () => { - const trimmed = input.trim(); - if (!trimmed || !model || !integrationId || isStreaming) return; + // Kick off a completion for the given conversation (ending with a user message). + // Shared by sending a new message and retrying after a failure. + const startCompletion = (baseMessages: ChatMessage[]) => { + if (!model || !integrationId) return; setError(null); - const userMessage: ChatMessage = { role: "user", content: trimmed }; + const systemPrompt = options.systemPrompt.trim(); + const payloadMessages: ChatMessage[] = systemPrompt + ? [{ role: "system", content: systemPrompt }, ...baseMessages] + : baseMessages; + + // Knowledge ids double as their collection name; web pages bring their own. + const collections = [...knowledgeIds, ...webItems.map((item) => item.collectionName)]; + + streamingTextRef.current = ""; + setStreamingText(""); + setPendingMessages(toCompletionMessages(payloadMessages)); + setPendingCollections(collections); + setIsStreaming(true); + }; + + const handleSend = () => { + const trimmed = input.trim(); + if ((!trimmed && attachments.length === 0) || !model || !integrationId || isStreaming) return; + + const userMessage: ChatMessage = { + role: "user", + content: trimmed, + images: attachments.length > 0 ? attachments.map((attachment) => attachment.dataUrl) : undefined, + }; const nextMessages = [...messagesRef.current, userMessage]; setMessages(nextMessages); messagesRef.current = nextMessages; setInput(""); + setAttachments([]); - const systemPrompt = options.systemPrompt.trim(); - const payloadMessages: ChatMessage[] = systemPrompt - ? [{ role: "system", content: systemPrompt }, ...nextMessages] - : nextMessages; + startCompletion(nextMessages); + }; - streamingTextRef.current = ""; - setStreamingText(""); - setPendingMessages(payloadMessages); - setIsStreaming(true); + // Retry the last turn after a failure: drop any partial assistant reply and + // re-run the completion from the most recent user message. + const handleRetry = () => { + if (isStreaming) return; + const lastUserIndex = messagesRef.current.findLastIndex((message) => message.role === "user"); + if (lastUserIndex === -1) return; + + const truncated = messagesRef.current.slice(0, lastUserIndex + 1); + setMessages(truncated); + messagesRef.current = truncated; + startCompletion(truncated); }; const handleNewChat = () => { @@ -199,10 +289,56 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro setStreamingText(""); setIsStreaming(false); setPendingMessages([]); + setPendingCollections([]); + setAttachments([]); + setKnowledgeIds([]); + setWebItems([]); setHistoryOpened(false); }; + const handleFilesSelected = async (fileList: FileList | null) => { + if (!fileList) return; + const images = await Promise.all( + Array.from(fileList) + .filter((file) => file.type.startsWith("image/")) + .map(async (file) => ({ name: file.name, dataUrl: await readFileAsDataUrl(file) })), + ); + setAttachments((previous) => [...previous, ...images]); + }; + + const addWebUrl = async () => { + if (!integrationId) return; + const raw = webInput.trim(); + if (!raw) return; + const url = /^https?:\/\//i.test(raw) ? raw : `https://${raw}`; + setWebInput(""); + setError(null); + try { + const document = await processWeb.mutateAsync({ integrationId, url }); + setWebItems((previous) => + previous.some((item) => item.collectionName === document.collectionName) + ? previous + : [...previous, { url, collectionName: document.collectionName, title: document.title }], + ); + } catch { + setError(t("widget.openWebUi.webError")); + } + }; + const modelData = useMemo(() => models.map((m) => ({ value: m.id, label: m.name ?? m.id })), [models]); + const knowledgeName = (id: string) => knowledge.find((item) => item.id === id)?.name ?? id; + const hasAttachments = attachments.length > 0 || knowledgeIds.length > 0 || webItems.length > 0; + + // Whether the selected model can actually see attached images. Prefer Open + // WebUI's declared capability; fall back to a name heuristic when it's absent. + const selectedModel = useMemo(() => models.find((item) => item.id === model), [models, model]); + const modelSupportsVision = useMemo(() => { + if (!selectedModel) return true; + const capability = selectedModel.info?.meta?.capabilities?.vision; + if (typeof capability === "boolean") return capability; + return VISION_MODEL_PATTERN.test(selectedModel.id) || VISION_MODEL_PATTERN.test(selectedModel.name ?? ""); + }, [selectedModel]); + const showVisionWarning = attachments.length > 0 && !modelSupportsVision; return ( @@ -215,18 +351,92 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro ) : null} {messages.map((message, index) => ( - + ))} {isStreaming ? : null} {error ? ( - - {error} - + + + {error} + + + ) : null} + {hasAttachments ? ( + + {attachments.map((attachment, index) => ( + + {attachment.name} + setAttachments((previous) => previous.filter((_, i) => i !== index))} + aria-label={t("widget.openWebUi.removeAttachment")} + /> + + ))} + {knowledgeIds.map((id) => ( + } + rightSection={ + setKnowledgeIds((previous) => previous.filter((value) => value !== id))} + aria-label={t("widget.openWebUi.removeAttachment")} + /> + } + > + {knowledgeName(id)} + + ))} + {webItems.map((item) => ( + } + rightSection={ + + setWebItems((previous) => + previous.filter((value) => value.collectionName !== item.collectionName), + ) + } + aria-label={t("widget.openWebUi.removeAttachment")} + /> + } + > + {hostnameOf(item.url)} + + ))} + + ) : null} + + {showVisionWarning ? ( + + + + {t("widget.openWebUi.visionWarning")} + + + ) : null} +