diff --git a/apps/docs/docs/widgets/open-webui/index.mdx b/apps/docs/docs/widgets/open-webui/index.mdx index 88602c95c..8f5baaf7d 100644 --- a/apps/docs/docs/widgets/open-webui/index.mdx +++ b/apps/docs/docs/widgets/open-webui/index.mdx @@ -13,7 +13,7 @@ import { openWebUiIntegration } from "@site/docs/integrations/open-webui"; -The AI Chat widget lets you chat with AI models served by Open WebUI directly from your dashboard. Responses stream in token-by-token, you can pick any model your Open WebUI instance exposes (Ollama, OpenAI, and other configured providers), and you can browse and reopen your Open WebUI chat history. You can also ground a conversation by attaching images, knowledge bases, or web pages. +The AI Chat widget lets you chat with AI models served by Open WebUI directly from your dashboard. Responses stream in token-by-token, you can pick any model your Open WebUI instance exposes (Ollama, OpenAI, and other configured providers), and you can browse and reopen your Open WebUI chat history. A full attachment menu lets you ground a conversation on images, uploaded or stored files, web pages, knowledge bases, notes, and previous chats. ### Supported Integrations @@ -40,9 +40,12 @@ The AI Chat widget lets you chat with AI models served by Open WebUI directly fr | **Model picker** | Lists every model exposed by Open WebUI's `/api/models` endpoint, selectable from the chat input. The first model is selected by default. | | **Streaming responses** | Assistant replies stream in live as they are generated. | | **Chat history** | Browse and reopen previous chats. New conversations are saved back to Open WebUI. | -| **Image attachments** | Attach images to a message for vision-capable models (sent inline as multimodal content). | -| **Knowledge bases** | Ground replies on any RAG knowledge collection (`/api/v1/knowledge/`) your account owns. | -| **Web pages** | Attach a URL and Open WebUI fetches and embeds the page as context for the reply. | +| **Image attachments** | Attach or capture images for vision-capable models (sent inline as multimodal content). A hint warns when the model has no vision support. | +| **File attachments** | Upload new documents or attach files already stored in Open WebUI; their content is retrieved as context. | +| **Web pages** | Attach a URL; Open WebUI ingests the page and the relevant parts are retrieved as context. | +| **Knowledge bases** | Ground replies on any RAG knowledge collection your account owns, with an expandable picker to inspect a base's files. | +| **Notes** | Attach the contents of your Open WebUI notes. | +| **Reference chats** | Bring a previous conversation in as context for the current one. | ### Notes diff --git a/packages/api/src/router/widgets/open-webui.ts b/packages/api/src/router/widgets/open-webui.ts index 1247dba62..7087ae6be 100644 --- a/packages/api/src/router/widgets/open-webui.ts +++ b/packages/api/src/router/widgets/open-webui.ts @@ -6,8 +6,11 @@ import { createIntegrationAsync } from "@homarr/integrations"; import { openWebUiChatRequestHandler, openWebUiChatsRequestHandler, + openWebUiFilesRequestHandler, + openWebUiKnowledgeFilesRequestHandler, openWebUiKnowledgeRequestHandler, openWebUiModelsRequestHandler, + openWebUiNotesRequestHandler, } from "@homarr/request-handler/open-webui"; import { createOneIntegrationMiddleware } from "../../middlewares/integration"; @@ -57,26 +60,30 @@ const groundMessagesAsync = async ( client: { queryCollectionsAsync: (names: string[], query: string, k?: number) => Promise }, messages: CompletionMessage[], collections: string[], + contextTexts: string[], ): Promise => { - if (collections.length === 0) return messages; + if (collections.length === 0 && contextTexts.length === 0) return messages; const lastUserMessage = messages.findLast((message) => message.role === "user"); const query = lastUserMessage ? messageText(lastUserMessage) : ""; let documents: string[] = []; - try { - documents = await client.queryCollectionsAsync(collections, query, 4); - } catch (error) { - logger.warn("Open WebUI context retrieval failed; continuing without it", { error }); + if (collections.length > 0) { + try { + documents = await client.queryCollectionsAsync(collections, query, 4); + } catch (error) { + logger.warn("Open WebUI context retrieval failed; continuing without it", { error }); + } } - if (documents.length === 0) return messages; - const context = documents.join("\n\n"); + const blocks = [...documents, ...contextTexts].filter((block) => block.trim() !== ""); + if (blocks.length === 0) return messages; + const contextMessage: CompletionMessage = { role: "system", content: - "Use the following context from the user's attached sources (web pages and knowledge bases) " + - `to answer. If the answer is not in the context, say so.\n\n${context}`, + "Use the following context from the user's attached sources (web pages, knowledge bases, " + + `files, notes and chats) to answer. If the answer is not in the context, say so.\n\n${blocks.join("\n\n")}`, }; return [contextMessage, ...messages]; }; @@ -100,6 +107,29 @@ export const openWebUiRouter = createTRPCRouter({ return data; }), + getFiles: publicProcedure.concat(createOneIntegrationMiddleware("query", "openWebUi")).query(async ({ ctx }) => { + const handler = openWebUiFilesRequestHandler.handler(ctx.integration, {}); + const { data } = await handler.getCachedOrUpdatedDataAsync({ forceUpdate: false }); + return data; + }), + + getNotes: publicProcedure.concat(createOneIntegrationMiddleware("query", "openWebUi")).query(async ({ ctx }) => { + const handler = openWebUiNotesRequestHandler.handler(ctx.integration, {}); + const { data } = await handler.getCachedOrUpdatedDataAsync({ forceUpdate: false }); + return data; + }), + + getKnowledgeFiles: publicProcedure + .input(z.object({ knowledgeId: z.string() })) + .concat(createOneIntegrationMiddleware("query", "openWebUi")) + .query(async ({ ctx, input }) => { + const handler = openWebUiKnowledgeFilesRequestHandler.handler(ctx.integration, { + knowledgeId: input.knowledgeId, + }); + const { data } = await handler.getCachedOrUpdatedDataAsync({ forceUpdate: false }); + return data; + }), + // Ingest a web page so its content can ground later messages. Returns the // collection to retrieve from plus a display title. processWeb: protectedProcedure @@ -110,6 +140,16 @@ export const openWebUiRouter = createTRPCRouter({ return await client.processWebAsync(input.url); }), + // Upload a file (e.g. a document) so it can be retrieved from later. Returns + // the file id; its vector collection is `file-{id}`. + uploadFile: protectedProcedure + .input(z.object({ filename: z.string(), contentBase64: z.string(), contentType: z.string() })) + .concat(createOneIntegrationMiddleware("interact", "openWebUi")) + .mutation(async ({ ctx, input }) => { + const client = await createIntegrationAsync(ctx.integration); + return await client.uploadFileAsync(input.filename, input.contentBase64, input.contentType); + }), + getChat: publicProcedure .input(z.object({ chatId: z.string() })) .concat(createOneIntegrationMiddleware("query", "openWebUi")) @@ -152,9 +192,11 @@ export const openWebUiRouter = createTRPCRouter({ z.object({ model: z.string(), messages: z.array(completionMessageSchema), - // Collections to ground the reply on (web pages + knowledge bases). We - // retrieve relevant chunks and inject them as context before streaming. + // Collections to ground the reply on (web pages, knowledge bases, files). + // We retrieve relevant chunks and inject them before streaming. collections: z.array(z.string()).optional(), + // Verbatim context blocks (notes, referenced chats) injected as-is. + contextTexts: z.array(z.string()).optional(), }), ) .concat(createOneIntegrationMiddleware("query", "openWebUi")) @@ -165,7 +207,12 @@ export const openWebUiRouter = createTRPCRouter({ void (async () => { try { const client = await createIntegrationAsync(ctx.integration); - const messages = await groundMessagesAsync(client, input.messages, input.collections ?? []); + const messages = await groundMessagesAsync( + client, + input.messages, + input.collections ?? [], + input.contextTexts ?? [], + ); await client.streamChatCompletionAsync( { model: input.model, messages }, (delta) => emit.next({ type: "delta", content: delta }), diff --git a/packages/integrations/src/open-webui/open-webui-integration.ts b/packages/integrations/src/open-webui/open-webui-integration.ts index ff96db259..cfce4b9e6 100644 --- a/packages/integrations/src/open-webui/open-webui-integration.ts +++ b/packages/integrations/src/open-webui/open-webui-integration.ts @@ -1,3 +1,5 @@ +import { FormData } from "undici"; + import { ResponseError } from "@homarr/common/server"; import { fetchWithTrustedCertificatesAsync } from "@homarr/core/infrastructure/http"; import { createLogger } from "@homarr/core/infrastructure/logs"; @@ -10,8 +12,10 @@ import type { OpenWebUiChatListItem, OpenWebUiChatPayload, OpenWebUiCompletionMessage, + OpenWebUiFileSummary, OpenWebUiKnowledge, OpenWebUiModel, + OpenWebUiNote, OpenWebUiWebDocument, } from "./open-webui-types"; import { @@ -19,9 +23,13 @@ import { openWebUiChatSchema, openWebUiCollectionQueryResponseSchema, openWebUiCompletionChunkSchema, + openWebUiFileListSchema, + openWebUiKnowledgeDetailSchema, openWebUiKnowledgeListSchema, openWebUiModelsResponseSchema, + openWebUiNoteListSchema, openWebUiProcessWebResponseSchema, + openWebUiUploadedFileSchema, } from "./open-webui-types"; const logger = createLogger({ module: "open-webui-integration" }); @@ -70,6 +78,86 @@ export class OpenWebUiIntegration extends Integration { return Array.isArray(parsed) ? parsed : parsed.items; } + /** + * List the user's uploaded files (for the "Attach Files" picker). + */ + public async listFilesAsync(): Promise { + const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/files/"), { + headers: this.getAuthHeaders(), + }); + + if (!response.ok) { + throw new ResponseError(response); + } + + const parsed = openWebUiFileListSchema.parse(await response.json()); + const files = Array.isArray(parsed) ? parsed : parsed.items; + return files.map((file) => ({ id: file.id, name: file.meta?.name ?? file.filename ?? file.id })); + } + + /** + * List the files contained in a knowledge base (for the expandable picker). + */ + public async getKnowledgeFilesAsync(knowledgeId: string): Promise { + const response = await fetchWithTrustedCertificatesAsync(this.url(`/api/v1/knowledge/${knowledgeId}`), { + headers: this.getAuthHeaders(), + }); + + if (!response.ok) { + throw new ResponseError(response); + } + + const parsed = openWebUiKnowledgeDetailSchema.parse(await response.json()); + return (parsed.files ?? []).map((file) => ({ id: file.id, name: file.meta?.name ?? file.filename ?? file.id })); + } + + /** + * List the user's notes (for the "Attach Notes" picker). + */ + public async listNotesAsync(): Promise { + const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/notes/"), { + headers: this.getAuthHeaders(), + }); + + if (!response.ok) { + throw new ResponseError(response); + } + + const parsed = openWebUiNoteListSchema.parse(await response.json()); + const notes = Array.isArray(parsed) ? parsed : parsed.items; + return notes.map((note) => ({ + id: note.id, + title: note.title ?? "Untitled note", + content: note.data?.content?.md ?? "", + })); + } + + /** + * Upload a file to Open WebUI so it can be retrieved from later. Returns the + * new file id (its vector collection is `file-{id}`). + */ + public async uploadFileAsync( + filename: string, + base64Content: string, + contentType: string, + ): Promise { + const form = new FormData(); + form.append("file", new Blob([Buffer.from(base64Content, "base64")], { type: contentType }), filename); + + const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/files/"), { + method: "POST", + headers: this.getAuthHeaders(), + body: form, + }); + + if (!response.ok) { + throw new ResponseError(response); + } + + const parsed = openWebUiUploadedFileSchema.parse(await response.json()); + return { id: parsed.id, name: parsed.meta?.name ?? parsed.filename ?? filename }; + } + /** * Ingest a web page into a temporary vector collection and return a handle to * retrieve from it. Open WebUI does not crawl URLs at completion time, so we diff --git a/packages/integrations/src/open-webui/open-webui-types.ts b/packages/integrations/src/open-webui/open-webui-types.ts index 8955a2728..0dcee73c3 100644 --- a/packages/integrations/src/open-webui/open-webui-types.ts +++ b/packages/integrations/src/open-webui/open-webui-types.ts @@ -86,6 +86,61 @@ export const openWebUiKnowledgeListSchema = z.union([ z.object({ items: z.array(openWebUiKnowledgeSchema) }), ]); +// A file stored in Open WebUI (uploaded directly or part of a knowledge base). +export const openWebUiFileSchema = z.object({ + id: z.string(), + filename: z.string().nullish(), + meta: z.object({ name: z.string().nullish(), content_type: z.string().nullish() }).nullish(), +}); + +export type OpenWebUiFile = z.infer; + +export const openWebUiFileListSchema = z.union([ + z.array(openWebUiFileSchema), + z.object({ items: z.array(openWebUiFileSchema) }), +]); + +// POST /api/v1/files/ — upload response (we only need the new file id + name). +export const openWebUiUploadedFileSchema = z.object({ + id: z.string(), + filename: z.string().nullish(), + meta: z.object({ name: z.string().nullish() }).nullish(), +}); + +// GET /api/v1/knowledge/{id} — knowledge base with its files. +export const openWebUiKnowledgeDetailSchema = z.object({ + id: z.string(), + name: z.string().nullish(), + files: z.array(openWebUiFileSchema).nullish(), +}); + +export type OpenWebUiKnowledgeDetail = z.infer; + +// GET /api/v1/notes/ — user notes (markdown content under data.content.md). +export const openWebUiNoteSchema = z.object({ + id: z.string(), + title: z.string().nullish(), + data: z.object({ content: z.object({ md: z.string().nullish() }).nullish() }).nullish(), +}); + +export const openWebUiNoteListSchema = z.union([ + z.array(openWebUiNoteSchema), + z.object({ items: z.array(openWebUiNoteSchema) }), +]); + +// A note flattened for the widget: id, title and its markdown body. +export interface OpenWebUiNote { + id: string; + title: string; + content: string; +} + +// A file flattened for the widget: id and a display name. +export interface OpenWebUiFileSummary { + id: string; + name: string; +} + // POST /api/v1/retrieval/process/web — ingests a URL into a temporary vector // collection. We retrieve from that collection at send time and inject the // results as context (Open WebUI does not fetch URLs at completion time). diff --git a/packages/request-handler/src/open-webui.ts b/packages/request-handler/src/open-webui.ts index b7306eb58..81162cd66 100644 --- a/packages/request-handler/src/open-webui.ts +++ b/packages/request-handler/src/open-webui.ts @@ -4,8 +4,10 @@ import { createIntegrationAsync } from "@homarr/integrations"; import type { OpenWebUiChat, OpenWebUiChatListItem, + OpenWebUiFileSummary, OpenWebUiKnowledge, OpenWebUiModel, + OpenWebUiNote, } from "@homarr/integrations/types"; import { createCachedIntegrationRequestHandler } from "./lib/cached-integration-request-handler"; @@ -36,6 +38,45 @@ export const openWebUiKnowledgeRequestHandler = createCachedIntegrationRequestHa queryKey: "openWebUiKnowledge", }); +export const openWebUiKnowledgeFilesRequestHandler = createCachedIntegrationRequestHandler< + OpenWebUiFileSummary[], + "openWebUi", + { knowledgeId: string } +>({ + async requestAsync(integration, input) { + const integrationInstance = await createIntegrationAsync(integration); + return await integrationInstance.getKnowledgeFilesAsync(input.knowledgeId); + }, + cacheDuration: dayjs.duration(1, "minute"), + queryKey: "openWebUiKnowledgeFiles", +}); + +export const openWebUiFilesRequestHandler = createCachedIntegrationRequestHandler< + OpenWebUiFileSummary[], + "openWebUi", + Record +>({ + async requestAsync(integration) { + const integrationInstance = await createIntegrationAsync(integration); + return await integrationInstance.listFilesAsync(); + }, + cacheDuration: dayjs.duration(30, "seconds"), + queryKey: "openWebUiFiles", +}); + +export const openWebUiNotesRequestHandler = createCachedIntegrationRequestHandler< + OpenWebUiNote[], + "openWebUi", + Record +>({ + async requestAsync(integration) { + const integrationInstance = await createIntegrationAsync(integration); + return await integrationInstance.listNotesAsync(); + }, + cacheDuration: dayjs.duration(30, "seconds"), + queryKey: "openWebUiNotes", +}); + export const openWebUiChatsRequestHandler = createCachedIntegrationRequestHandler< OpenWebUiChatListItem[], "openWebUi", diff --git a/packages/translation/src/lang/en.json b/packages/translation/src/lang/en.json index a1f011ac8..247299e39 100644 --- a/packages/translation/src/lang/en.json +++ b/packages/translation/src/lang/en.json @@ -3565,13 +3565,24 @@ "retry": "Retry", "visionWarning": "This model may not support images", "attach": "Attach", + "uploadFiles": "Upload Files", + "capture": "Capture", "attachImage": "Attach image", "attachKnowledge": "Knowledge", "noKnowledge": "No knowledge bases in Open WebUI", "attachWebpage": "Webpage", + "attachFiles": "Files", + "attachNotes": "Notes", + "referenceChats": "Reference Chats", + "noFiles": "No files in Open WebUI", + "noNotes": "No notes in Open WebUI", + "noChats": "No previous chats", "addUrl": "Add URL", "removeAttachment": "Remove attachment", "webError": "Could not load that web page", + "uploadError": "Could not upload the file", + "captureError": "Could not access the camera", + "chatError": "Could not load that chat", "error": { "internalServerError": "Failed to reach Open WebUI" } diff --git a/packages/widgets/src/open-webui/chat.module.css b/packages/widgets/src/open-webui/chat.module.css index ba8d7a7a6..3aa564197 100644 --- a/packages/widgets/src/open-webui/chat.module.css +++ b/packages/widgets/src/open-webui/chat.module.css @@ -38,6 +38,24 @@ max-width: 85%; } +.menuRowLabel { + flex: 1; + text-align: left; + font-weight: 400; +} + +.attachmentBadge { + max-width: 180px; + text-transform: none; +} + +.captureVideo { + width: 100%; + max-height: 50vh; + border-radius: var(--mantine-radius-md); + background-color: black; +} + .thumb { position: relative; line-height: 0; diff --git a/packages/widgets/src/open-webui/chat.tsx b/packages/widgets/src/open-webui/chat.tsx index 04f22403f..2a701cf27 100644 --- a/packages/widgets/src/open-webui/chat.tsx +++ b/packages/widgets/src/open-webui/chat.tsx @@ -12,6 +12,7 @@ import { Group, Image, Loader, + Modal, NavLink, Paper, Popover, @@ -27,10 +28,15 @@ import { IconAlertTriangle, IconArrowLeft, IconArrowUp, + IconCamera, + IconChevronDown, + IconChevronRight, IconDatabase, + IconFile, + IconFileText, IconHistory, + IconMessage, IconPaperclip, - IconPhoto, IconPlayerStopFilled, IconPlus, IconRefresh, @@ -58,12 +64,28 @@ interface ImageAttachment { name: string; dataUrl: string; } - interface WebAttachment { url: string; collectionName: string; title: string; } +interface FileAttachment { + id: string; + name: string; +} +interface NoteAttachment { + id: string; + title: string; + content: string; +} +interface ChatAttachment { + id: string; + title: string; + transcript: string; +} + +// Which sub-view the attach popover is showing. +type AttachView = "root" | "webpage" | "files" | "notes" | "knowledge" | "chats"; // Heuristic fallback for detecting vision-capable models when Open WebUI does // not declare the capability explicitly. @@ -81,24 +103,33 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro const [isStreaming, setIsStreaming] = useState(false); const [pendingMessages, setPendingMessages] = useState([]); const [pendingCollections, setPendingCollections] = useState([]); + const [pendingContextTexts, setPendingContextTexts] = useState([]); const [activeChatId, setActiveChatId] = useState(null); const [selectedChatId, setSelectedChatId] = useState(null); const [error, setError] = useState(null); const [historyOpened, setHistoryOpened] = useState(false); - // Attachments grounding the conversation. Images are per-message; knowledge - // collections and web pages are sticky context kept until removed. + // Attachments. Images are inline per-message (vision); the rest are sticky + // context kept for the conversation until removed. const [attachments, setAttachments] = useState([]); const [knowledgeIds, setKnowledgeIds] = useState([]); const [webItems, setWebItems] = useState([]); + const [fileItems, setFileItems] = useState([]); + const [noteItems, setNoteItems] = useState([]); + const [chatItems, setChatItems] = useState([]); const [webInput, setWebInput] = useState(""); const [attachOpened, setAttachOpened] = useState(false); + const [attachView, setAttachView] = useState("root"); + + const [captureOpened, setCaptureOpened] = useState(false); const messagesRef = useRef([]); const streamingTextRef = useRef(""); const viewportRef = useRef(null); const composerRef = useRef(null); const fileInputRef = useRef(null); + const videoRef = useRef(null); + const captureStreamRef = useRef(null); const [composerHeight, setComposerHeight] = useState(72); const utils = clientApi.useUtils(); @@ -110,12 +141,22 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro const { data: knowledge = [] } = clientApi.widget.openWebUi.getKnowledge.useQuery( { integrationId: integrationId ?? "" }, - { enabled: Boolean(integrationId), refetchOnWindowFocus: false, retry: false }, + { enabled: Boolean(integrationId) && attachOpened, refetchOnWindowFocus: false, retry: false }, + ); + + const { data: files = [] } = clientApi.widget.openWebUi.getFiles.useQuery( + { integrationId: integrationId ?? "" }, + { enabled: Boolean(integrationId) && attachOpened, refetchOnWindowFocus: false, retry: false }, + ); + + const { data: notes = [] } = clientApi.widget.openWebUi.getNotes.useQuery( + { integrationId: integrationId ?? "" }, + { enabled: Boolean(integrationId) && attachOpened, refetchOnWindowFocus: false, retry: false }, ); const { data: chats = [] } = clientApi.widget.openWebUi.getChats.useQuery( { integrationId: integrationId ?? "" }, - { enabled: Boolean(integrationId) && options.showHistory, refetchOnWindowFocus: false, retry: false }, + { enabled: Boolean(integrationId), refetchOnWindowFocus: false, retry: false }, ); const { data: loadedChat } = clientApi.widget.openWebUi.getChat.useQuery( @@ -126,6 +167,7 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro const createChat = clientApi.widget.openWebUi.createChat.useMutation(); const updateChat = clientApi.widget.openWebUi.updateChat.useMutation(); const processWeb = clientApi.widget.openWebUi.processWeb.useMutation(); + const uploadFile = clientApi.widget.openWebUi.uploadFile.useMutation(); // Default to the first available model once the list loads. useEffect(() => { @@ -161,12 +203,21 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro viewportRef.current?.scrollTo({ top: viewportRef.current.scrollHeight, behavior: "smooth" }); }, [messages, streamingText, composerHeight]); + // Reset the popover to the root view whenever it closes. + useEffect(() => { + if (!attachOpened) setAttachView("root"); + }, [attachOpened]); + + // Stop the camera stream on unmount. + useEffect(() => () => stopCaptureStream(), []); + clientApi.widget.openWebUi.sendMessage.useSubscription( { integrationId: integrationId ?? "", model: model ?? "", messages: pendingMessages, collections: pendingCollections.length > 0 ? pendingCollections : undefined, + contextTexts: pendingContextTexts.length > 0 ? pendingContextTexts : undefined, }, { enabled: isStreaming && pendingMessages.length > 0, @@ -201,6 +252,7 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro setIsStreaming(false); setPendingMessages([]); setPendingCollections([]); + setPendingContextTexts([]); }; const persistChat = async (finalMessages: ChatMessage[]) => { @@ -219,9 +271,7 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro const chat = await createChat.mutateAsync({ integrationId, ...payload }); setActiveChatId(chat.id); } - if (options.showHistory) { - await utils.widget.openWebUi.getChats.invalidate({ integrationId }); - } + await utils.widget.openWebUi.getChats.invalidate({ integrationId }); } catch { // Persistence is best-effort; the conversation remains usable in-memory. } @@ -238,13 +288,23 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro ? [{ role: "system", content: systemPrompt }, ...baseMessages] : baseMessages; - // Knowledge ids double as their collection name; web pages bring their own. - const collections = [...knowledgeIds, ...webItems.map((item) => item.collectionName)]; + // Retrieval collections: knowledge bases (id), web pages and files. + const collections = [ + ...knowledgeIds, + ...webItems.map((item) => item.collectionName), + ...fileItems.map((item) => `file-${item.id}`), + ]; + // Verbatim context: notes and referenced chats. + const contextTexts = [ + ...noteItems.map((note) => `Note "${note.title}":\n${note.content}`), + ...chatItems.map((chat) => `Previous chat "${chat.title}":\n${chat.transcript}`), + ]; streamingTextRef.current = ""; setStreamingText(""); setPendingMessages(toCompletionMessages(payloadMessages)); setPendingCollections(collections); + setPendingContextTexts(contextTexts); setIsStreaming(true); }; @@ -290,20 +350,41 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro setIsStreaming(false); setPendingMessages([]); setPendingCollections([]); + setPendingContextTexts([]); setAttachments([]); setKnowledgeIds([]); setWebItems([]); + setFileItems([]); + setNoteItems([]); + setChatItems([]); setHistoryOpened(false); }; + // Upload Files: images become inline vision attachments; everything else is + // uploaded to Open WebUI and grounded via retrieval. const handleFilesSelected = async (fileList: FileList | null) => { - if (!fileList) return; - const images = await Promise.all( - Array.from(fileList) - .filter((file) => file.type.startsWith("image/")) - .map(async (file) => ({ name: file.name, dataUrl: await readFileAsDataUrl(file) })), - ); - setAttachments((previous) => [...previous, ...images]); + if (!fileList || !integrationId) return; + for (const file of Array.from(fileList)) { + try { + const dataUrl = await readFileAsDataUrl(file); + if (file.type.startsWith("image/")) { + setAttachments((previous) => [...previous, { name: file.name, dataUrl }]); + } else { + const contentBase64 = dataUrl.split(",")[1] ?? ""; + const uploaded = await uploadFile.mutateAsync({ + integrationId, + filename: file.name, + contentBase64, + contentType: file.type || "application/octet-stream", + }); + setFileItems((previous) => + previous.some((item) => item.id === uploaded.id) ? previous : [...previous, uploaded], + ); + } + } catch { + setError(t("widget.openWebUi.uploadError")); + } + } }; const addWebUrl = async () => { @@ -320,22 +401,105 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro ? previous : [...previous, { url, collectionName: document.collectionName, title: document.title }], ); + setAttachOpened(false); } catch { setError(t("widget.openWebUi.webError")); } }; + const toggleFile = (file: FileAttachment) => + setFileItems((previous) => + previous.some((item) => item.id === file.id) + ? previous.filter((item) => item.id !== file.id) + : [...previous, file], + ); + + const toggleNote = (note: NoteAttachment) => + setNoteItems((previous) => + previous.some((item) => item.id === note.id) + ? previous.filter((item) => item.id !== note.id) + : [...previous, note], + ); + + const toggleKnowledge = (id: string) => + setKnowledgeIds((previous) => + previous.includes(id) ? previous.filter((value) => value !== id) : [...previous, id], + ); + + const toggleChatReference = async (chat: { id: string; title?: string | null }) => { + if (!integrationId) return; + if (chatItems.some((item) => item.id === chat.id)) { + setChatItems((previous) => previous.filter((item) => item.id !== chat.id)); + return; + } + try { + const full = await utils.widget.openWebUi.getChat.fetch({ integrationId, chatId: chat.id }); + const transcript = extractChatMessages(full.chat) + .map((message) => `${message.role}: ${message.content}`) + .join("\n"); + setChatItems((previous) => [...previous, { id: chat.id, title: chat.title ?? chat.id, transcript }]); + } catch { + setError(t("widget.openWebUi.chatError")); + } + }; + + // ---- Camera capture ---- + const stopCaptureStream = () => { + captureStreamRef.current?.getTracks().forEach((track) => track.stop()); + captureStreamRef.current = null; + }; + + const openCapture = async () => { + setAttachOpened(false); + try { + const stream = await navigator.mediaDevices.getUserMedia({ video: true }); + captureStreamRef.current = stream; + setCaptureOpened(true); + } catch { + setError(t("widget.openWebUi.captureError")); + } + }; + + const closeCapture = () => { + stopCaptureStream(); + setCaptureOpened(false); + }; + + useEffect(() => { + if (captureOpened && videoRef.current && captureStreamRef.current) { + videoRef.current.srcObject = captureStreamRef.current; + } + }, [captureOpened]); + + const takePhoto = () => { + const video = videoRef.current; + if (!video) return; + const canvas = document.createElement("canvas"); + canvas.width = video.videoWidth; + canvas.height = video.videoHeight; + canvas.getContext("2d")?.drawImage(video, 0, 0); + setAttachments((previous) => [ + ...previous, + { name: `capture-${Date.now()}.png`, dataUrl: canvas.toDataURL("image/png") }, + ]); + closeCapture(); + }; + // Drop the noisy ":latest" tag from the label (common for Ollama models); the // value keeps the full id so the right model is still sent. const modelData = useMemo( () => models.map((m) => ({ value: m.id, label: (m.name ?? m.id).replace(/:latest$/i, "") })), [models], ); - const knowledgeName = (id: string) => knowledge.find((item) => item.id === id)?.name ?? id; - const hasAttachments = attachments.length > 0 || knowledgeIds.length > 0 || webItems.length > 0; + const hasAttachments = + attachments.length > 0 || + knowledgeIds.length > 0 || + webItems.length > 0 || + fileItems.length > 0 || + noteItems.length > 0 || + chatItems.length > 0; - // Whether the selected model can actually see attached images. Prefer Open - // WebUI's declared capability; fall back to a name heuristic when it's absent. + // Whether the selected model can actually see attached images. const selectedModel = useMemo(() => models.find((item) => item.id === model), [models, model]); const modelSupportsVision = useMemo(() => { if (!selectedModel) return true; @@ -345,6 +509,19 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro }, [selectedModel]); const showVisionWarning = attachments.length > 0 && !modelSupportsVision; + const attachViewTitle = + attachView === "webpage" + ? t("widget.openWebUi.attachWebpage") + : attachView === "files" + ? t("widget.openWebUi.attachFiles") + : attachView === "notes" + ? t("widget.openWebUi.attachNotes") + : attachView === "knowledge" + ? t("widget.openWebUi.attachKnowledge") + : attachView === "chats" + ? t("widget.openWebUi.referenceChats") + : ""; + return ( @@ -393,42 +570,56 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro ))} {knowledgeIds.map((id) => ( - } - rightSection={ - setKnowledgeIds((previous) => previous.filter((value) => value !== id))} - aria-label={t("widget.openWebUi.removeAttachment")} - /> - } - > - {knowledgeName(id)} - + icon={} + label={knowledge.find((item) => item.id === id)?.name ?? id} + onRemove={() => toggleKnowledge(id)} + removeLabel={t("widget.openWebUi.removeAttachment")} + /> ))} {webItems.map((item) => ( - } - rightSection={ - - setWebItems((previous) => - previous.filter((value) => value.collectionName !== item.collectionName), - ) - } - aria-label={t("widget.openWebUi.removeAttachment")} - /> + icon={} + label={hostnameOf(item.url)} + onRemove={() => + setWebItems((previous) => previous.filter((value) => value.collectionName !== item.collectionName)) } - > - {hostnameOf(item.url)} - + removeLabel={t("widget.openWebUi.removeAttachment")} + /> + ))} + {fileItems.map((item) => ( + } + label={item.name} + onRemove={() => toggleFile(item)} + removeLabel={t("widget.openWebUi.removeAttachment")} + /> + ))} + {noteItems.map((item) => ( + } + label={item.title} + onRemove={() => toggleNote(item)} + removeLabel={t("widget.openWebUi.removeAttachment")} + /> + ))} + {chatItems.map((item) => ( + } + label={item.title} + onRemove={() => setChatItems((previous) => previous.filter((value) => value.id !== item.id))} + removeLabel={t("widget.openWebUi.removeAttachment")} + /> ))} ) : null} @@ -475,7 +666,7 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro onChange={setAttachOpened} position="top-start" withinPortal={false} - width={260} + width={280} shadow="md" > @@ -483,6 +674,7 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro setAttachOpened((value) => !value)} aria-label={t("widget.openWebUi.attach")} > @@ -490,68 +682,154 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro - - - - - - - {t("widget.openWebUi.attachKnowledge")} - - {knowledge.length > 0 ? ( - - - {knowledge.map((item) => ( - - ))} - - - ) : ( - - {t("widget.openWebUi.noKnowledge")} - - )} + + {attachView === "root" ? ( + + } + label={t("widget.openWebUi.uploadFiles")} + onClick={() => fileInputRef.current?.click()} + /> + } + label={t("widget.openWebUi.capture")} + onClick={() => void openCapture()} + /> + } + label={t("widget.openWebUi.attachWebpage")} + hasSub + onClick={() => setAttachView("webpage")} + /> + } + label={t("widget.openWebUi.attachFiles")} + hasSub + onClick={() => setAttachView("files")} + /> + } + label={t("widget.openWebUi.attachNotes")} + hasSub + onClick={() => setAttachView("notes")} + /> + } + label={t("widget.openWebUi.attachKnowledge")} + hasSub + onClick={() => setAttachView("knowledge")} + /> + } + label={t("widget.openWebUi.referenceChats")} + hasSub + onClick={() => setAttachView("chats")} + /> - + ) : ( - - {t("widget.openWebUi.attachWebpage")} - - - setWebInput(event.currentTarget.value)} - onKeyDown={(event) => { - if (event.key === "Enter") { - event.preventDefault(); - void addWebUrl(); - } - }} - /> + void addWebUrl()} - aria-label={t("widget.openWebUi.addUrl")} + size="sm" + onClick={() => setAttachView("root")} + aria-label={t("widget.openWebUi.back")} > - + + + {attachViewTitle} + + + {attachView === "webpage" ? ( + + setWebInput(event.currentTarget.value)} + onKeyDown={(event) => { + if (event.key === "Enter") { + event.preventDefault(); + void addWebUrl(); + } + }} + /> + void addWebUrl()} + aria-label={t("widget.openWebUi.addUrl")} + > + + + + ) : null} + + {attachView === "files" ? ( + + {files.map((file) => ( + item.id === file.id)} + onChange={() => toggleFile(file)} + /> + ))} + + ) : null} + + {attachView === "notes" ? ( + + {notes.map((note) => ( + item.id === note.id)} + onChange={() => toggleNote({ id: note.id, title: note.title, content: note.content })} + /> + ))} + + ) : null} + + {attachView === "knowledge" ? ( + + {knowledge.map((item) => ( + toggleKnowledge(item.id)} + isFileSelected={(id) => fileItems.some((file) => file.id === id)} + onToggleFile={toggleFile} + /> + ))} + + ) : null} + + {attachView === "chats" ? ( + + {chats.map((chat) => ( + item.id === chat.id)} + onChange={() => void toggleChatReference(chat)} + /> + ))} + + ) : null} - + )} {options.showHistory ? ( @@ -613,10 +891,9 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro { void handleFilesSelected(event.currentTarget.files); event.currentTarget.value = ""; @@ -624,6 +901,28 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro /> + + + + + + + + + + {options.showHistory && historyOpened ? ( @@ -736,6 +1035,129 @@ const extractChatMessages = (chat: RawChat | null | undefined): ChatMessage[] => .filter((message): message is ChatMessage => message !== null); }; +function MenuRow({ + icon, + label, + hasSub, + onClick, +}: { + icon: React.ReactNode; + label: string; + hasSub?: boolean; + onClick: () => void; +}) { + return ( + + ); +} + +function PickerList({ count, emptyLabel, children }: { count: number; emptyLabel: string; children: React.ReactNode }) { + if (count === 0) { + return ( + + {emptyLabel} + + ); + } + return ( + + + {children} + + + ); +} + +function KnowledgeRow({ + integrationId, + knowledgeId, + name, + selected, + onToggle, + isFileSelected, + onToggleFile, +}: { + integrationId: string; + knowledgeId: string; + name: string; + selected: boolean; + onToggle: () => void; + isFileSelected: (id: string) => boolean; + onToggleFile: (file: FileAttachment) => void; +}) { + const [expanded, setExpanded] = useState(false); + const { data: files = [], isLoading } = clientApi.widget.openWebUi.getKnowledgeFiles.useQuery( + { integrationId, knowledgeId }, + { enabled: expanded && Boolean(integrationId), refetchOnWindowFocus: false, retry: false }, + ); + + return ( + + + + setExpanded((value) => !value)}> + {expanded ? : } + + + {expanded ? ( + + {isLoading ? ( + + ) : ( + files.map((file) => ( + onToggleFile(file)} + /> + )) + )} + + ) : null} + + ); +} + +function AttachmentBadge({ + color, + icon, + label, + onRemove, + removeLabel, +}: { + color: string; + icon: React.ReactNode; + label: string; + onRemove: () => void; + removeLabel: string; +}) { + return ( + } + className={classes.attachmentBadge} + > + {label} + + ); +} + function MessageBubble({ from, content, images }: { from: ChatMessage["role"]; content: string; images?: string[] }) { const isUser = from === "user"; const avatar = (