From 8e0f8cd46a7864b56104d7747dd33ad7311aa970 Mon Sep 17 00:00:00 2001 From: Lok3sh-Kumar Date: Sat, 27 Jun 2026 14:17:59 +0530 Subject: [PATCH] refactor(open-webui): split chat widget into hooks and components Break the monolithic chat.tsx into focused hooks (use-attachments, use-chat-stream, use-voice-recording, use-camera-capture) and presentational components (chat-components), with shared types and utils extracted to chat-types/chat-utils. Review fixes applied on top: - require auth for the streaming completion (protectedProcedure) so anonymous board viewers cannot spend the owner's LLM credits - rename image-only state to images and the picker query options object to pickerQueryOptions for clarity - extract AttachPopover, clearPending(), composerEmpty and releaseAudioResources() to remove duplication --- packages/api/src/router/widgets/open-webui.ts | 36 +- .../src/open-webui/open-webui-integration.ts | 264 ++-- .../src/open-webui/chat-components.tsx | 484 +++++++ packages/widgets/src/open-webui/chat-types.ts | 41 + packages/widgets/src/open-webui/chat-utils.ts | 83 ++ .../widgets/src/open-webui/chat.module.css | 39 + packages/widgets/src/open-webui/chat.tsx | 1142 ++--------------- .../widgets/src/open-webui/use-attachments.ts | 225 ++++ .../src/open-webui/use-camera-capture.ts | 69 + .../widgets/src/open-webui/use-chat-stream.ts | 233 ++++ .../src/open-webui/use-voice-recording.ts | 136 ++ 11 files changed, 1563 insertions(+), 1189 deletions(-) create mode 100644 packages/widgets/src/open-webui/chat-components.tsx create mode 100644 packages/widgets/src/open-webui/chat-types.ts create mode 100644 packages/widgets/src/open-webui/chat-utils.ts create mode 100644 packages/widgets/src/open-webui/use-attachments.ts create mode 100644 packages/widgets/src/open-webui/use-camera-capture.ts create mode 100644 packages/widgets/src/open-webui/use-chat-stream.ts create mode 100644 packages/widgets/src/open-webui/use-voice-recording.ts diff --git a/packages/api/src/router/widgets/open-webui.ts b/packages/api/src/router/widgets/open-webui.ts index fe15d1312..5c0f7cd87 100644 --- a/packages/api/src/router/widgets/open-webui.ts +++ b/packages/api/src/router/widgets/open-webui.ts @@ -55,6 +55,15 @@ const messageText = (message: CompletionMessage): string => .map((part) => part.text) .join(" "); +// Number of chunks to retrieve per collection when grounding a reply. +const RETRIEVAL_K = 4; + +// Instruction prepended to the retrieved context so the model knows how to use +// it. Model-facing, so intentionally not localized. +const RETRIEVAL_INSTRUCTION = + "Use the following context from the user's attached sources (web pages, knowledge bases, " + + "files, notes and chats) to answer. If the answer is not in the context, say so."; + // Retrieve relevant chunks from the attached collections and prepend them as a // system context message so the model can ground its reply on them. const groundMessagesAsync = async ( @@ -65,13 +74,19 @@ const groundMessagesAsync = async ( ): Promise => { if (collections.length === 0 && contextTexts.length === 0) return messages; - const lastUserMessage = messages.findLast((message) => message.role === "user"); - const query = lastUserMessage ? messageText(lastUserMessage) : ""; + // Use the most recent user message that has text. The latest turn can be + // image-only (no text), in which case we fall back to the prior question so + // attached collections still get a meaningful retrieval query. + const query = + messages + .filter((message) => message.role === "user") + .map(messageText) + .findLast((text) => text.trim() !== "") ?? ""; let documents: string[] = []; if (collections.length > 0) { try { - documents = await client.queryCollectionsAsync(collections, query, 4); + documents = await client.queryCollectionsAsync(collections, query, RETRIEVAL_K); } catch (error) { logger.warn("Open WebUI context retrieval failed; continuing without it", { error }); } @@ -82,9 +97,7 @@ const groundMessagesAsync = async ( const contextMessage: CompletionMessage = { role: "system", - content: - "Use the following context from the user's attached sources (web pages, knowledge bases, " + - `files, notes and chats) to answer. If the answer is not in the context, say so.\n\n${blocks.join("\n\n")}`, + content: `${RETRIEVAL_INSTRUCTION}\n\n${blocks.join("\n\n")}`, }; return [contextMessage, ...messages]; }; @@ -205,9 +218,12 @@ export const openWebUiRouter = createTRPCRouter({ }), // Streaming chat completion. Delivered over the WebSocket subscription link so - // tokens can be emitted as they arrive. Uses "query" access (same level as - // viewing the widget) to match the other integration subscriptions in the app. - sendMessage: publicProcedure + // tokens can be emitted as they arrive. A completion spends the owner's upstream + // LLM credits, so it must not be reachable by anonymous viewers of a public + // board: protectedProcedure requires a session (the WS context resolves it from + // the session-token cookie) and the "interact" middleware checks integration + // access, matching processWeb/uploadFile/transcribe above. + sendMessage: protectedProcedure .input( z.object({ model: z.string(), @@ -219,7 +235,7 @@ export const openWebUiRouter = createTRPCRouter({ contextTexts: z.array(z.string()).optional(), }), ) - .concat(createOneIntegrationMiddleware("query", "openWebUi")) + .concat(createOneIntegrationMiddleware("interact", "openWebUi")) .subscription(({ ctx, input }) => { return observable((emit) => { const controller = new AbortController(); diff --git a/packages/integrations/src/open-webui/open-webui-integration.ts b/packages/integrations/src/open-webui/open-webui-integration.ts index 90d6bfa25..197612aca 100644 --- a/packages/integrations/src/open-webui/open-webui-integration.ts +++ b/packages/integrations/src/open-webui/open-webui-integration.ts @@ -35,6 +35,10 @@ import { const logger = createLogger({ module: "open-webui-integration" }); +// The (undici) request options accepted by the shared fetch helper. Derived so +// the request helpers stay in sync with the underlying fetch types. +type RequestOptions = NonNullable[1]>; + export class OpenWebUiIntegration extends Integration { protected async testingAsync(input: IntegrationTestingInput): Promise { const response = await input.fetchAsync(this.url("/api/models"), { @@ -52,30 +56,15 @@ export class OpenWebUiIntegration extends Integration { * List the models served by Open WebUI (aggregates all configured providers). */ public async getModelsAsync(): Promise { - const response = await fetchWithTrustedCertificatesAsync(this.url("/api/models"), { - headers: this.getAuthHeaders(), - }); - - if (!response.ok) { - throw new ResponseError(response); - } - - return openWebUiModelsResponseSchema.parse(await response.json()).data; + const parsed = await this.getJsonAsync("/api/models", openWebUiModelsResponseSchema); + return parsed.data; } /** * List the knowledge bases (RAG collections) the user can ground chats on. */ public async getKnowledgeAsync(): Promise { - const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/knowledge/"), { - headers: this.getAuthHeaders(), - }); - - if (!response.ok) { - throw new ResponseError(response); - } - - const parsed = openWebUiKnowledgeListSchema.parse(await response.json()); + const parsed = await this.getJsonAsync("/api/v1/knowledge/", openWebUiKnowledgeListSchema); return Array.isArray(parsed) ? parsed : parsed.items; } @@ -83,16 +72,7 @@ export class OpenWebUiIntegration extends Integration { * List the user's uploaded files (for the "Attach Files" picker). */ public async listFilesAsync(): Promise { - const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/files/"), { - headers: this.getAuthHeaders(), - }); - - if (!response.ok) { - throw new ResponseError(response); - } - - const parsed = openWebUiFileListSchema.parse(await response.json()); - const files = Array.isArray(parsed) ? parsed : parsed.items; + const files = await this.fetchFilesAsync(); return files.map((file) => ({ id: file.id, name: file.meta?.name ?? file.filename ?? file.id })); } @@ -102,34 +82,23 @@ export class OpenWebUiIntegration extends Integration { * we read the files list and filter by the base's collection id instead. */ public async getKnowledgeFilesAsync(knowledgeId: string): Promise { - const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/files/"), { - headers: this.getAuthHeaders(), - }); - - if (!response.ok) { - throw new ResponseError(response); - } - - const parsed = openWebUiFileListSchema.parse(await response.json()); - const files = Array.isArray(parsed) ? parsed : parsed.items; + const files = await this.fetchFilesAsync(); return files .filter((file) => file.meta?.collection_name === knowledgeId) .map((file) => ({ id: file.id, name: file.meta?.name ?? file.filename ?? file.id })); } + /** Fetch the raw file list, normalizing the paginated and bare-array shapes. */ + private async fetchFilesAsync() { + const parsed = await this.getJsonAsync("/api/v1/files/", openWebUiFileListSchema); + return Array.isArray(parsed) ? parsed : parsed.items; + } + /** * List the user's notes (for the "Attach Notes" picker). */ public async listNotesAsync(): Promise { - const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/notes/"), { - headers: this.getAuthHeaders(), - }); - - if (!response.ok) { - throw new ResponseError(response); - } - - const parsed = openWebUiNoteListSchema.parse(await response.json()); + const parsed = await this.getJsonAsync("/api/v1/notes/", openWebUiNoteListSchema); const notes = Array.isArray(parsed) ? parsed : parsed.items; return notes.map((note) => ({ id: note.id, @@ -143,15 +112,10 @@ export class OpenWebUiIntegration extends Integration { * truncates the body, so we fetch the note directly when attaching it. */ public async getNoteAsync(noteId: string): Promise { - const response = await fetchWithTrustedCertificatesAsync(this.url(`/api/v1/notes/${noteId}`), { - headers: this.getAuthHeaders(), - }); - - if (!response.ok) { - throw new ResponseError(response); - } - - const parsed = openWebUiNoteSchema.parse(await response.json()); + const parsed = await this.getJsonAsync( + `/api/v1/notes/${encodeURIComponent(noteId)}`, + openWebUiNoteSchema, + ); return { id: parsed.id, title: parsed.title ?? "Untitled note", content: parsed.data?.content?.md ?? "" }; } @@ -162,17 +126,8 @@ export class OpenWebUiIntegration extends Integration { const form = new FormData(); form.append("file", new Blob([Buffer.from(base64Content, "base64")], { type: contentType }), filename); - const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/audio/transcriptions"), { - method: "POST", - headers: this.getAuthHeaders(), - body: form, - }); - - if (!response.ok) { - throw new ResponseError(response); - } - - return openWebUiTranscriptionSchema.parse(await response.json()).text; + const parsed = await this.postFormAsync("/api/v1/audio/transcriptions", form, openWebUiTranscriptionSchema); + return parsed.text; } /** @@ -187,17 +142,7 @@ export class OpenWebUiIntegration extends Integration { const form = new FormData(); form.append("file", new Blob([Buffer.from(base64Content, "base64")], { type: contentType }), filename); - const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/files/"), { - method: "POST", - headers: this.getAuthHeaders(), - body: form, - }); - - if (!response.ok) { - throw new ResponseError(response); - } - - const parsed = openWebUiUploadedFileSchema.parse(await response.json()); + const parsed = await this.postFormAsync("/api/v1/files/", form, openWebUiUploadedFileSchema); return { id: parsed.id, name: parsed.meta?.name ?? parsed.filename ?? filename }; } @@ -207,17 +152,11 @@ export class OpenWebUiIntegration extends Integration { * embed the page here and inject retrieved chunks ourselves at send time. */ public async processWebAsync(url: string): Promise { - const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/retrieval/process/web"), { - method: "POST", - headers: { ...this.getAuthHeaders(), "Content-Type": "application/json" }, - body: JSON.stringify({ url, collection_name: "" }), - }); - - if (!response.ok) { - throw new ResponseError(response); - } - - const parsed = openWebUiProcessWebResponseSchema.parse(await response.json()); + const parsed = await this.postJsonAsync( + "/api/v1/retrieval/process/web", + { url, collection_name: "" }, + openWebUiProcessWebResponseSchema, + ); return { collectionName: parsed.collection_name, title: parsed.file?.meta?.name ?? url }; } @@ -230,17 +169,11 @@ export class OpenWebUiIntegration extends Integration { return []; } - const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/retrieval/query/collection"), { - method: "POST", - headers: { ...this.getAuthHeaders(), "Content-Type": "application/json" }, - body: JSON.stringify({ collection_names: collectionNames, query, k }), - }); - - if (!response.ok) { - throw new ResponseError(response); - } - - const parsed = openWebUiCollectionQueryResponseSchema.parse(await response.json()); + const parsed = await this.postJsonAsync( + "/api/v1/retrieval/query/collection", + { collection_names: collectionNames, query, k }, + openWebUiCollectionQueryResponseSchema, + ); return (parsed.documents ?? []).flat(); } @@ -248,64 +181,32 @@ export class OpenWebUiIntegration extends Integration { * List the user's chats (native Open WebUI history). */ public async listChatsAsync(): Promise { - const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/chats/"), { - headers: this.getAuthHeaders(), - }); - - if (!response.ok) { - throw new ResponseError(response); - } - - return openWebUiChatListItemSchema.array().parse(await response.json()); + return await this.getJsonAsync("/api/v1/chats/", openWebUiChatListItemSchema.array()); } /** * Get a single chat including its messages. */ public async getChatAsync(chatId: string): Promise { - const response = await fetchWithTrustedCertificatesAsync(this.url(`/api/v1/chats/${chatId}`), { - headers: this.getAuthHeaders(), - }); - - if (!response.ok) { - throw new ResponseError(response); - } - - return openWebUiChatSchema.parse(await response.json()); + return await this.getJsonAsync(`/api/v1/chats/${encodeURIComponent(chatId)}`, openWebUiChatSchema); } /** * Create a new chat in the native history. */ public async createChatAsync(payload: OpenWebUiChatPayload): Promise { - const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/chats/new"), { - method: "POST", - headers: { ...this.getAuthHeaders(), "Content-Type": "application/json" }, - body: JSON.stringify({ chat: payload }), - }); - - if (!response.ok) { - throw new ResponseError(response); - } - - return openWebUiChatSchema.parse(await response.json()); + return await this.postJsonAsync("/api/v1/chats/new", { chat: payload }, openWebUiChatSchema); } /** * Update an existing chat in the native history. */ public async updateChatAsync(chatId: string, payload: OpenWebUiChatPayload): Promise { - const response = await fetchWithTrustedCertificatesAsync(this.url(`/api/v1/chats/${chatId}`), { - method: "POST", - headers: { ...this.getAuthHeaders(), "Content-Type": "application/json" }, - body: JSON.stringify({ chat: payload }), - }); - - if (!response.ok) { - throw new ResponseError(response); - } - - return openWebUiChatSchema.parse(await response.json()); + return await this.postJsonAsync( + `/api/v1/chats/${encodeURIComponent(chatId)}`, + { chat: payload }, + openWebUiChatSchema, + ); } /** @@ -318,14 +219,14 @@ export class OpenWebUiIntegration extends Integration { onDelta: (delta: string) => void, signal: AbortSignal, ): Promise { - const response = await fetchWithTrustedCertificatesAsync(this.url("/api/chat/completions"), { + const response = await this.requestAsync("/api/chat/completions", { method: "POST", - headers: { ...this.getAuthHeaders(), "Content-Type": "application/json" }, + headers: { "Content-Type": "application/json" }, body: JSON.stringify({ model: params.model, messages: params.messages, stream: true }), signal, }); - if (!response.ok || !response.body) { + if (!response.body) { throw new ResponseError(response); } @@ -343,16 +244,20 @@ export class OpenWebUiIntegration extends Integration { try { const parsed = openWebUiCompletionChunkSchema.parse(JSON.parse(payload)); - const delta = parsed.choices?.[0]?.delta?.content; - if (delta) { - onDelta(delta); + const choice = parsed.choices?.[0]; + if (choice?.delta?.content) { + onDelta(choice.delta.content); } + // Some providers end the stream with a finish_reason chunk and no + // explicit [DONE] sentinel; treat that as a clean completion too. + if (choice?.finish_reason) return true; } catch { logger.debug("Failed to parse Open WebUI completion chunk", { line: trimmed }); } return false; }; + let completed = false; try { while (!signal.aborted) { const { done, value } = await reader.read(); @@ -362,14 +267,25 @@ export class OpenWebUiIntegration extends Integration { const lines = buffer.split("\n"); buffer = lines.pop() ?? ""; for (const line of lines) { - if (processLine(line)) return; + if (processLine(line)) { + completed = true; + break; + } } + if (completed) break; } } finally { reader.cancel().catch(() => { // ignore cancellation errors during teardown }); } + + // The reader ended without a [DONE]/finish_reason marker: the upstream + // connection was cut mid-stream. Surface it instead of presenting the + // partial reply as a complete answer. + if (!signal.aborted && !completed) { + throw new Error("Open WebUI stream ended before completion"); + } } private getAuthHeaders(): Record { @@ -377,4 +293,58 @@ export class OpenWebUiIntegration extends Integration { Authorization: `Bearer ${this.getSecretValue("apiKey")}`, }; } + + /** + * Issue an authenticated request against the Open WebUI API and throw a + * {@link ResponseError} on a non-2xx status. Centralizes the auth header, + * trusted-certificate fetch and error handling shared by every endpoint. + */ + private async requestAsync( + path: `/${string}`, + init?: Omit & { headers?: Record }, + ) { + const response = await fetchWithTrustedCertificatesAsync(this.url(path), { + ...init, + headers: { ...this.getAuthHeaders(), ...init?.headers }, + }); + + if (!response.ok) { + throw new ResponseError(response); + } + + return response; + } + + /** GET an endpoint and validate the JSON body against `schema`. */ + private async getJsonAsync( + path: `/${string}`, + schema: { parse: (data: unknown) => TOutput }, + ): Promise { + const response = await this.requestAsync(path); + return schema.parse(await response.json()); + } + + /** POST a JSON body and validate the JSON response against `schema`. */ + private async postJsonAsync( + path: `/${string}`, + body: unknown, + schema: { parse: (data: unknown) => TOutput }, + ): Promise { + const response = await this.requestAsync(path, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + return schema.parse(await response.json()); + } + + /** POST a multipart form (file upload) and validate the JSON response. */ + private async postFormAsync( + path: `/${string}`, + form: FormData, + schema: { parse: (data: unknown) => TOutput }, + ): Promise { + const response = await this.requestAsync(path, { method: "POST", body: form }); + return schema.parse(await response.json()); + } } diff --git a/packages/widgets/src/open-webui/chat-components.tsx b/packages/widgets/src/open-webui/chat-components.tsx new file mode 100644 index 000000000..32b62d21d --- /dev/null +++ b/packages/widgets/src/open-webui/chat-components.tsx @@ -0,0 +1,484 @@ +import { useEffect, useRef, useState } from "react"; +import { + ActionIcon, + Avatar, + Badge, + Box, + Button, + Checkbox, + CloseButton, + Group, + Image, + Loader, + Paper, + Popover, + ScrollArea, + Stack, + Text, + TextInput, + Tooltip, +} from "@mantine/core"; +import { + IconArrowLeft, + IconCamera, + IconChevronDown, + IconChevronRight, + IconDatabase, + IconFile, + IconFileText, + IconMessage, + IconPaperclip, + IconPlus, + IconRobot, + IconUser, + IconWorld, +} from "@tabler/icons-react"; +import ReactMarkdown from "react-markdown"; + +import { clientApi } from "@homarr/api/client"; +import { useI18n } from "@homarr/translation/client"; + +import type { AttachView, ChatMessage, FileAttachment } from "./chat-types"; +import type { useAttachments } from "./use-attachments"; +import classes from "./chat.module.css"; + +export function MenuRow({ + icon, + label, + hasSub, + onClick, +}: { + icon: React.ReactNode; + label: string; + hasSub?: boolean; + onClick: () => void; +}) { + return ( + + ); +} + +export function PickerList({ + count, + emptyLabel, + children, +}: { + count: number; + emptyLabel: string; + children: React.ReactNode; +}) { + if (count === 0) { + return ( + + {emptyLabel} + + ); + } + return ( + + + {children} + + + ); +} + +export function KnowledgeRow({ + integrationId, + knowledgeId, + name, + selected, + onToggle, + isFileSelected, + onToggleFile, + emptyFilesLabel, +}: { + integrationId: string; + knowledgeId: string; + name: string; + selected: boolean; + onToggle: () => void; + isFileSelected: (id: string) => boolean; + onToggleFile: (file: FileAttachment) => void; + emptyFilesLabel: string; +}) { + // Default to expanded so a base's documents are visible without an extra click. + const [expanded, setExpanded] = useState(true); + const { data: files = [], isLoading } = clientApi.widget.openWebUi.getKnowledgeFiles.useQuery( + { integrationId, knowledgeId }, + { enabled: expanded && Boolean(integrationId), refetchOnWindowFocus: false, retry: false }, + ); + + return ( + + + + setExpanded((value) => !value)} + aria-label={name} + > + {expanded ? : } + + + {expanded ? ( + + {isLoading ? ( + + ) : files.length === 0 ? ( + + {emptyFilesLabel} + + ) : ( + files.map((file) => ( + onToggleFile(file)} + /> + )) + )} + + ) : null} + + ); +} + +export function AttachmentBadge({ + color, + icon, + label, + onRemove, + removeLabel, +}: { + color: string; + icon: React.ReactNode; + label: string; + onRemove: () => void; + removeLabel: string; +}) { + return ( + } + className={classes.attachmentBadge} + > + {label} + + ); +} + +// The "attach" popover: a root menu of sources and the per-source sub-views +// (web URL input, file / note / knowledge / chat pickers) used to ground a reply. +export function AttachPopover({ + attachments, + chats, + integrationId, + onUploadClick, + onCapture, +}: { + attachments: ReturnType; + chats: { id: string; title?: string | null }[]; + integrationId: string | undefined; + onUploadClick: () => void; + onCapture: () => void; +}) { + const t = useI18n(); + + const attachViewTitles: Record, string> = { + webpage: t("widget.openWebUi.attachWebpage"), + files: t("widget.openWebUi.attachFiles"), + notes: t("widget.openWebUi.attachNotes"), + knowledge: t("widget.openWebUi.attachKnowledge"), + chats: t("widget.openWebUi.referenceChats"), + }; + const attachViewTitle = attachments.attachView === "root" ? "" : attachViewTitles[attachments.attachView]; + + return ( + + + + attachments.openAttach(!attachments.attachOpened)} + aria-label={t("widget.openWebUi.attach")} + > + + + + + + {attachments.attachView === "root" ? ( + + } + label={t("widget.openWebUi.uploadFiles")} + onClick={onUploadClick} + /> + } label={t("widget.openWebUi.capture")} onClick={onCapture} /> + } + label={t("widget.openWebUi.attachWebpage")} + hasSub + onClick={() => attachments.setAttachView("webpage")} + /> + } + label={t("widget.openWebUi.attachFiles")} + hasSub + onClick={() => attachments.setAttachView("files")} + /> + } + label={t("widget.openWebUi.attachNotes")} + hasSub + onClick={() => attachments.setAttachView("notes")} + /> + } + label={t("widget.openWebUi.attachKnowledge")} + hasSub + onClick={() => attachments.setAttachView("knowledge")} + /> + } + label={t("widget.openWebUi.referenceChats")} + hasSub + onClick={() => attachments.setAttachView("chats")} + /> + + ) : ( + + + attachments.setAttachView("root")} + aria-label={t("widget.openWebUi.back")} + > + + + + {attachViewTitle} + + + + {attachments.attachView === "webpage" ? ( + + attachments.setWebInput(event.currentTarget.value)} + onKeyDown={(event) => { + if (event.key === "Enter") { + event.preventDefault(); + void attachments.addWebUrl(); + } + }} + /> + void attachments.addWebUrl()} + aria-label={t("widget.openWebUi.addUrl")} + > + + + + ) : null} + + {attachments.attachView === "files" ? ( + + {attachments.files.map((file) => ( + item.id === file.id)} + onChange={() => attachments.toggleFile(file)} + /> + ))} + + ) : null} + + {attachments.attachView === "notes" ? ( + + {attachments.notes.map((note) => ( + item.id === note.id)} + onChange={() => void attachments.toggleNote({ id: note.id, title: note.title })} + /> + ))} + + ) : null} + + {attachments.attachView === "knowledge" ? ( + + {attachments.knowledge.map((item) => ( + attachments.toggleKnowledge(item.id)} + isFileSelected={(id) => attachments.fileItems.some((file) => file.id === id)} + onToggleFile={attachments.toggleFile} + emptyFilesLabel={t("widget.openWebUi.noFiles")} + /> + ))} + + ) : null} + + {attachments.attachView === "chats" ? ( + + {chats.map((chat) => ( + item.id === chat.id)} + onChange={() => void attachments.toggleChatReference(chat)} + /> + ))} + + ) : null} + + )} + + + ); +} + +// Live microphone waveform driven by a Web Audio analyser node. +export function RecordingWaveform({ analyser }: { analyser: AnalyserNode | null }) { + const canvasRef = useRef(null); + + useEffect(() => { + const canvas = canvasRef.current; + if (!analyser || !canvas) return; + const ctx = canvas.getContext("2d"); + if (!ctx) return; + + const data = new Uint8Array(analyser.frequencyBinCount); + const color = getComputedStyle(canvas).color; + let raf = 0; + + const render = () => { + raf = requestAnimationFrame(render); + analyser.getByteFrequencyData(data); + const { width, height } = canvas; + ctx.clearRect(0, 0, width, height); + ctx.fillStyle = color; + const barCount = 56; + const step = Math.max(1, Math.floor(data.length / barCount)); + const slot = width / barCount; + for (let index = 0; index < barCount; index++) { + const value = (data[index * step] ?? 0) / 255; + const barHeight = Math.max(2, value * height); + ctx.fillRect(index * slot + slot * 0.3, (height - barHeight) / 2, slot * 0.4, barHeight); + } + }; + + render(); + return () => cancelAnimationFrame(raf); + }, [analyser]); + + return ; +} + +function TypingDots() { + return ( + + + + + + ); +} + +export function MessageBubble({ + from, + content, + images, + loading, +}: { + from: ChatMessage["role"]; + content: string; + images?: string[]; + loading?: boolean; +}) { + const isUser = from === "user"; + const avatar = ( + + {isUser ? : } + + ); + return ( + + {isUser ? null : avatar} + + {images && images.length > 0 ? ( + + {images.map((src, index) => ( + + ))} + + ) : null} + {loading && !content ? : null} + {content ? ( + isUser ? ( + + {content} + + ) : ( +
+ {content} +
+ ) + ) : null} + + {isUser ? avatar : null} + + ); +} diff --git a/packages/widgets/src/open-webui/chat-types.ts b/packages/widgets/src/open-webui/chat-types.ts new file mode 100644 index 000000000..b386da929 --- /dev/null +++ b/packages/widgets/src/open-webui/chat-types.ts @@ -0,0 +1,41 @@ +export interface ChatMessage { + role: "system" | "user" | "assistant"; + content: string; + images?: string[]; +} + +export type CompletionPart = { type: "text"; text: string } | { type: "image_url"; image_url: { url: string } }; +export type CompletionMessage = { role: ChatMessage["role"]; content: string | CompletionPart[] }; + +export interface ImageAttachment { + name: string; + dataUrl: string; +} +export interface WebAttachment { + url: string; + collectionName: string; + title: string; +} +export interface FileAttachment { + id: string; + name: string; +} +export interface NoteAttachment { + id: string; + title: string; + content: string; +} +export interface ChatAttachment { + id: string; + title: string; + transcript: string; +} + +// Which sub-view the attach popover is showing. +export type AttachView = "root" | "webpage" | "files" | "notes" | "knowledge" | "chats"; + +// Retrieval collections + verbatim context blocks injected before a completion. +export interface Grounding { + collections: string[]; + contextTexts: string[]; +} diff --git a/packages/widgets/src/open-webui/chat-utils.ts b/packages/widgets/src/open-webui/chat-utils.ts new file mode 100644 index 000000000..0f29c9508 --- /dev/null +++ b/packages/widgets/src/open-webui/chat-utils.ts @@ -0,0 +1,83 @@ +import type { ChatMessage, CompletionMessage } from "./chat-types"; + +// Heuristic fallback for detecting vision-capable models when Open WebUI does +// not declare the capability explicitly. +export const VISION_MODEL_PATTERN = + /vision|llava|bakllava|moondream|minicpm-?v|qwen2?-?vl|pixtral|cogvlm|internvl|gemma3|smolvlm|granite[\w-]*vision/i; + +export const readBlobAsDataUrl = (blob: Blob): Promise => + new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.addEventListener("load", () => resolve(reader.result as string)); + reader.addEventListener("error", () => reject(reader.error ?? new Error("Failed to read file"))); + reader.readAsDataURL(blob); + }); + +export const hostnameOf = (url: string): string => { + try { + return new URL(url).hostname || url; + } catch { + return url; + } +}; + +export const formatSeconds = (total: number): string => { + const minutes = Math.floor(total / 60); + const seconds = total % 60; + return `${minutes}:${String(seconds).padStart(2, "0")}`; +}; + +// Convert displayed messages to the OpenAI completion format, inlining any +// attached images on the message that carries them (vision content parts). +export const toCompletionMessages = (messages: ChatMessage[]): CompletionMessage[] => + messages.map((message) => + message.images && message.images.length > 0 + ? { + role: message.role, + content: [ + ...(message.content ? [{ type: "text" as const, text: message.content }] : []), + ...message.images.map((url) => ({ type: "image_url" as const, image_url: { url } })), + ], + } + : { role: message.role, content: message.content }, + ); + +type RawMessage = { role?: string | null; content?: string | null }; +type HistoryMessage = RawMessage & { parentId?: string | null }; +interface RawChat { + messages?: RawMessage[] | null; + history?: { currentId?: string | null; messages?: Record | null } | null; +} + +const toChatMessage = (role: string | null | undefined, content: string | null | undefined): ChatMessage | null => + (role === "system" || role === "user" || role === "assistant") && typeof content === "string" + ? { role, content } + : null; + +/** + * Reconstructs the ordered conversation from an Open WebUI chat. Chats created in + * the Open WebUI UI store the active branch as a tree under `chat.history` + * (walk from `currentId` up through `parentId`); `chat.messages` can be stale, so + * the history tree takes precedence and we fall back to the flat list. + */ +export const extractChatMessages = (chat: RawChat | null | undefined): ChatMessage[] => { + const history = chat?.history; + if (history?.messages && history.currentId) { + const map = history.messages; + const ordered: ChatMessage[] = []; + const seen = new Set(); + let cursor: string | null | undefined = history.currentId; + while (cursor && !seen.has(cursor)) { + seen.add(cursor); + const node: HistoryMessage | undefined = map[cursor]; + if (!node) break; + const message = toChatMessage(node.role, node.content); + if (message) ordered.unshift(message); + cursor = node.parentId; + } + if (ordered.length > 0) return ordered; + } + return (chat?.messages ?? []) + .map((message) => toChatMessage(message.role, message.content)) + .filter((message): message is ChatMessage => message !== null); +}; diff --git a/packages/widgets/src/open-webui/chat.module.css b/packages/widgets/src/open-webui/chat.module.css index 50381fd89..938d997cb 100644 --- a/packages/widgets/src/open-webui/chat.module.css +++ b/packages/widgets/src/open-webui/chat.module.css @@ -29,6 +29,45 @@ text-overflow: ellipsis; } +/* Markdown-rendered assistant replies: keep block spacing tight inside the bubble. */ +.markdown { + font-size: var(--mantine-font-size-sm); + word-break: break-word; +} + +.markdown > :first-child { + margin-top: 0; +} + +.markdown > :last-child { + margin-bottom: 0; +} + +.markdown p, +.markdown ul, +.markdown ol, +.markdown pre { + margin-block: var(--mantine-spacing-xs); +} + +.markdown pre { + overflow-x: auto; + padding: var(--mantine-spacing-xs); + border-radius: var(--mantine-radius-sm); + background-color: var(--mantine-color-default); +} + +.markdown code { + font-family: var(--mantine-font-family-monospace); + font-size: 0.85em; +} + +.markdown :not(pre) > code { + padding: 0.1em 0.3em; + border-radius: var(--mantine-radius-sm); + background-color: var(--mantine-color-default); +} + .avatar { flex-shrink: 0; margin-top: 2px; diff --git a/packages/widgets/src/open-webui/chat.tsx b/packages/widgets/src/open-webui/chat.tsx index 64413ec9c..b049feeef 100644 --- a/packages/widgets/src/open-webui/chat.tsx +++ b/packages/widgets/src/open-webui/chat.tsx @@ -3,11 +3,8 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { ActionIcon, - Avatar, - Badge, Box, Button, - Checkbox, CloseButton, Group, Image, @@ -15,13 +12,11 @@ import { Modal, NavLink, Paper, - Popover, ScrollArea, Select, Stack, Text, Textarea, - TextInput, Tooltip, } from "@mantine/core"; import { @@ -30,20 +25,16 @@ import { IconArrowUp, IconCamera, IconCheck, - IconChevronDown, - IconChevronRight, IconDatabase, IconFile, IconFileText, IconHistory, IconMessage, IconMicrophone, - IconPaperclip, IconPlayerStopFilled, IconPlus, IconRefresh, IconRobot, - IconUser, IconWorld, IconX, } from "@tabler/icons-react"; @@ -52,134 +43,74 @@ import { clientApi } from "@homarr/api/client"; import { useI18n } from "@homarr/translation/client"; import type { WidgetComponentProps } from "../definition"; +import { AttachmentBadge, AttachPopover, MessageBubble, RecordingWaveform } from "./chat-components"; import classes from "./chat.module.css"; - -interface ChatMessage { - role: "system" | "user" | "assistant"; - content: string; - images?: string[]; -} - -type CompletionPart = { type: "text"; text: string } | { type: "image_url"; image_url: { url: string } }; -type CompletionMessage = { role: ChatMessage["role"]; content: string | CompletionPart[] }; - -interface ImageAttachment { - name: string; - dataUrl: string; -} -interface WebAttachment { - url: string; - collectionName: string; - title: string; -} -interface FileAttachment { - id: string; - name: string; -} -interface NoteAttachment { - id: string; - title: string; - content: string; -} -interface ChatAttachment { - id: string; - title: string; - transcript: string; -} - -// Which sub-view the attach popover is showing. -type AttachView = "root" | "webpage" | "files" | "notes" | "knowledge" | "chats"; - -// Heuristic fallback for detecting vision-capable models when Open WebUI does -// not declare the capability explicitly. -const VISION_MODEL_PATTERN = - /vision|llava|bakllava|moondream|minicpm-?v|qwen2?-?vl|pixtral|cogvlm|internvl|gemma3|smolvlm|granite[\w-]*vision/i; +import { VISION_MODEL_PATTERN, formatSeconds, hostnameOf } from "./chat-utils"; +import { useAttachments } from "./use-attachments"; +import { useCameraCapture } from "./use-camera-capture"; +import { useChatStream } from "./use-chat-stream"; +import { useVoiceRecording } from "./use-voice-recording"; export function Chat({ options, integrationIds, isEditMode }: WidgetComponentProps<"openWebUi">) { const t = useI18n(); const integrationId = integrationIds.at(0); const [model, setModel] = useState(null); - const [messages, setMessages] = useState([]); const [input, setInput] = useState(""); - const [streamingText, setStreamingText] = useState(""); - const [isStreaming, setIsStreaming] = useState(false); - const [pendingMessages, setPendingMessages] = useState([]); - const [pendingCollections, setPendingCollections] = useState([]); - const [pendingContextTexts, setPendingContextTexts] = useState([]); - const [activeChatId, setActiveChatId] = useState(null); - const [selectedChatId, setSelectedChatId] = useState(null); const [error, setError] = useState(null); const [historyOpened, setHistoryOpened] = useState(false); + const [composerHeight, setComposerHeight] = useState(72); - // Attachments. Images are inline per-message (vision); the rest are sticky - // context kept for the conversation until removed. - const [attachments, setAttachments] = useState([]); - const [knowledgeIds, setKnowledgeIds] = useState([]); - const [webItems, setWebItems] = useState([]); - const [fileItems, setFileItems] = useState([]); - const [noteItems, setNoteItems] = useState([]); - const [chatItems, setChatItems] = useState([]); - const [webInput, setWebInput] = useState(""); - const [attachOpened, setAttachOpened] = useState(false); - const [attachView, setAttachView] = useState("root"); - - const [captureOpened, setCaptureOpened] = useState(false); - const [isRecording, setIsRecording] = useState(false); - const [recordingSeconds, setRecordingSeconds] = useState(0); - const [analyser, setAnalyser] = useState(null); - - const messagesRef = useRef([]); - const streamingTextRef = useRef(""); const viewportRef = useRef(null); const composerRef = useRef(null); const fileInputRef = useRef(null); - const captureStreamRef = useRef(null); - const mediaRecorderRef = useRef(null); - const audioChunksRef = useRef([]); - const audioStreamRef = useRef(null); - const audioContextRef = useRef(null); - const recordingTimerRef = useRef(null); - const recordingCancelledRef = useRef(false); - const [composerHeight, setComposerHeight] = useState(72); - - const utils = clientApi.useUtils(); const { data: models = [], isLoading: modelsLoading } = clientApi.widget.openWebUi.getModels.useQuery( { integrationId: integrationId ?? "" }, { enabled: Boolean(integrationId), refetchOnWindowFocus: false, retry: false }, ); - const { data: knowledge = [] } = clientApi.widget.openWebUi.getKnowledge.useQuery( - { integrationId: integrationId ?? "" }, - { enabled: Boolean(integrationId) && attachOpened, refetchOnWindowFocus: false, retry: false }, - ); - - const { data: files = [] } = clientApi.widget.openWebUi.getFiles.useQuery( - { integrationId: integrationId ?? "" }, - { enabled: Boolean(integrationId) && attachOpened, refetchOnWindowFocus: false, retry: false }, - ); - - const { data: notes = [] } = clientApi.widget.openWebUi.getNotes.useQuery( - { integrationId: integrationId ?? "" }, - { enabled: Boolean(integrationId) && attachOpened, refetchOnWindowFocus: false, retry: false }, - ); - const { data: chats = [] } = clientApi.widget.openWebUi.getChats.useQuery( { integrationId: integrationId ?? "" }, { enabled: Boolean(integrationId), refetchOnWindowFocus: false, retry: false }, ); - const { data: loadedChat } = clientApi.widget.openWebUi.getChat.useQuery( - { integrationId: integrationId ?? "", chatId: selectedChatId ?? "" }, - { enabled: Boolean(integrationId) && Boolean(selectedChatId) }, - ); + const attachments = useAttachments({ + integrationId, + onError: setError, + errorMessages: { + upload: t("widget.openWebUi.uploadError"), + web: t("widget.openWebUi.webError"), + note: t("widget.openWebUi.noteError"), + chat: t("widget.openWebUi.chatError"), + }, + }); - const createChat = clientApi.widget.openWebUi.createChat.useMutation(); - const updateChat = clientApi.widget.openWebUi.updateChat.useMutation(); - const processWeb = clientApi.widget.openWebUi.processWeb.useMutation(); - const uploadFile = clientApi.widget.openWebUi.uploadFile.useMutation(); - const transcribe = clientApi.widget.openWebUi.transcribe.useMutation(); + const closeHistory = useCallback(() => setHistoryOpened(false), []); + const stream = useChatStream({ + integrationId, + model, + setModel, + systemPrompt: options.systemPrompt, + buildGrounding: attachments.buildGrounding, + setError, + onChatLoaded: closeHistory, + newChatTitle: t("widget.openWebUi.newChat"), + }); + + const voice = useVoiceRecording({ + integrationId, + onError: setError, + onTranscribed: (text) => setInput((previous) => (previous ? `${previous} ${text}` : text)), + micErrorMessage: t("widget.openWebUi.micError"), + transcribeErrorMessage: t("widget.openWebUi.transcribeError"), + }); + + const camera = useCameraCapture({ + onError: setError, + onCapture: (attachment) => attachments.setImages((previous) => [...previous, attachment]), + captureErrorMessage: t("widget.openWebUi.captureError"), + }); // Default to the first available model once the list loads. useEffect(() => { @@ -187,19 +118,6 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro setModel(models[0]?.id ?? null); }, [model, models]); - // Hydrate the conversation when a history chat finishes loading. - useEffect(() => { - if (!loadedChat) return; - const loadedMessages = extractChatMessages(loadedChat.chat); - setMessages(loadedMessages); - messagesRef.current = loadedMessages; - setActiveChatId(loadedChat.id); - if (loadedChat.chat?.models?.[0]) { - setModel(loadedChat.chat.models[0]); - } - setHistoryOpened(false); - }, [loadedChat]); - // Track the floating composer's height so messages can pad below it. useEffect(() => { const element = composerRef.current; @@ -213,404 +131,37 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro // Auto-scroll to the bottom as messages or streamed tokens arrive. useEffect(() => { viewportRef.current?.scrollTo({ top: viewportRef.current.scrollHeight, behavior: "smooth" }); - }, [messages, streamingText, composerHeight]); + }, [stream.messages, stream.streamingText, composerHeight]); - // Reset the popover to the root view whenever it closes. - useEffect(() => { - if (!attachOpened) setAttachView("root"); - }, [attachOpened]); - - // Stop the camera and microphone streams on unmount. - useEffect( - () => () => { - stopCaptureStream(); - if (recordingTimerRef.current !== null) window.clearInterval(recordingTimerRef.current); - audioStreamRef.current?.getTracks().forEach((track) => track.stop()); - void audioContextRef.current?.close().catch(() => undefined); - }, - [], - ); - - clientApi.widget.openWebUi.sendMessage.useSubscription( - { - integrationId: integrationId ?? "", - model: model ?? "", - messages: pendingMessages, - collections: pendingCollections.length > 0 ? pendingCollections : undefined, - contextTexts: pendingContextTexts.length > 0 ? pendingContextTexts : undefined, - }, - { - enabled: isStreaming && pendingMessages.length > 0, - onData: (event) => { - if (event.type === "delta") { - streamingTextRef.current += event.content; - setStreamingText(streamingTextRef.current); - } else if (event.type === "done") { - finalizeAssistantMessage(); - } else if (event.type === "error") { - setError(event.message); - finalizeAssistantMessage(); - } - }, - onError: (err) => { - setError(err.message); - finalizeAssistantMessage(); - }, - }, - ); - - const finalizeAssistantMessage = () => { - const finalText = streamingTextRef.current; - if (finalText.length > 0) { - const finalMessages = [...messagesRef.current, { role: "assistant" as const, content: finalText }]; - setMessages(finalMessages); - messagesRef.current = finalMessages; - void persistChat(finalMessages); - } - streamingTextRef.current = ""; - setStreamingText(""); - setIsStreaming(false); - setPendingMessages([]); - setPendingCollections([]); - setPendingContextTexts([]); - }; - - const persistChat = async (finalMessages: ChatMessage[]) => { - if (!integrationId || !model) return; - // Native history stores plain text; image data URLs are dropped on persist. - const title = finalMessages.find((message) => message.role === "user")?.content.slice(0, 80) ?? "New chat"; - const payload = { - title, - models: [model], - messages: finalMessages.map(({ role, content }) => ({ role, content })), - }; - try { - if (activeChatId) { - await updateChat.mutateAsync({ integrationId, chatId: activeChatId, chat: payload }); - } else { - const chat = await createChat.mutateAsync({ integrationId, ...payload }); - setActiveChatId(chat.id); - } - await utils.widget.openWebUi.getChats.invalidate({ integrationId }); - } catch { - // Persistence is best-effort; the conversation remains usable in-memory. - } - }; - - // Kick off a completion for the given conversation (ending with a user message). - // Shared by sending a new message and retrying after a failure. - const startCompletion = (baseMessages: ChatMessage[]) => { - if (!model || !integrationId) return; - setError(null); - - const systemPrompt = options.systemPrompt.trim(); - const payloadMessages: ChatMessage[] = systemPrompt - ? [{ role: "system", content: systemPrompt }, ...baseMessages] - : baseMessages; - - // Retrieval collections: knowledge bases (id), web pages and files. - const collections = [ - ...knowledgeIds, - ...webItems.map((item) => item.collectionName), - ...fileItems.map((item) => `file-${item.id}`), - ]; - // Verbatim context: notes and referenced chats. - const contextTexts = [ - ...noteItems.map((note) => `Note "${note.title}":\n${note.content}`), - ...chatItems.map((chat) => `Previous chat "${chat.title}":\n${chat.transcript}`), - ]; - - streamingTextRef.current = ""; - setStreamingText(""); - setPendingMessages(toCompletionMessages(payloadMessages)); - setPendingCollections(collections); - setPendingContextTexts(contextTexts); - setIsStreaming(true); - }; + // Nothing to send yet: no typed text and no inline image attachments. + const composerEmpty = !input.trim() && attachments.images.length === 0; const handleSend = () => { const trimmed = input.trim(); - if ((!trimmed && attachments.length === 0) || !model || !integrationId || isStreaming) return; + if (composerEmpty || !model || !integrationId || stream.isStreaming) return; - const userMessage: ChatMessage = { + stream.send({ role: "user", content: trimmed, - images: attachments.length > 0 ? attachments.map((attachment) => attachment.dataUrl) : undefined, - }; - const nextMessages = [...messagesRef.current, userMessage]; - setMessages(nextMessages); - messagesRef.current = nextMessages; + images: attachments.images.length > 0 ? attachments.images.map((item) => item.dataUrl) : undefined, + }); setInput(""); - setAttachments([]); - - startCompletion(nextMessages); - }; - - // Retry the last turn after a failure: drop any partial assistant reply and - // re-run the completion from the most recent user message. - const handleRetry = () => { - if (isStreaming) return; - const lastUserIndex = messagesRef.current.findLastIndex((message) => message.role === "user"); - if (lastUserIndex === -1) return; - - const truncated = messagesRef.current.slice(0, lastUserIndex + 1); - setMessages(truncated); - messagesRef.current = truncated; - startCompletion(truncated); + attachments.setImages([]); }; const handleNewChat = () => { - setMessages([]); - messagesRef.current = []; - setActiveChatId(null); - setSelectedChatId(null); + stream.reset(); + attachments.reset(); setError(null); - streamingTextRef.current = ""; - setStreamingText(""); - setIsStreaming(false); - setPendingMessages([]); - setPendingCollections([]); - setPendingContextTexts([]); - setAttachments([]); - setKnowledgeIds([]); - setWebItems([]); - setFileItems([]); - setNoteItems([]); - setChatItems([]); setHistoryOpened(false); }; - // Upload Files: images become inline vision attachments; everything else is - // uploaded to Open WebUI and grounded via retrieval. - const handleFilesSelected = async (fileList: FileList | null) => { - if (!fileList || !integrationId) return; - for (const file of Array.from(fileList)) { - try { - const dataUrl = await readFileAsDataUrl(file); - if (file.type.startsWith("image/")) { - setAttachments((previous) => [...previous, { name: file.name, dataUrl }]); - } else { - const contentBase64 = dataUrl.split(",")[1] ?? ""; - const uploaded = await uploadFile.mutateAsync({ - integrationId, - filename: file.name, - contentBase64, - contentType: file.type || "application/octet-stream", - }); - setFileItems((previous) => - previous.some((item) => item.id === uploaded.id) ? previous : [...previous, uploaded], - ); - } - } catch { - setError(t("widget.openWebUi.uploadError")); - } - } - }; - - const addWebUrl = async () => { - if (!integrationId) return; - const raw = webInput.trim(); - if (!raw) return; - const url = /^https?:\/\//i.test(raw) ? raw : `https://${raw}`; - setWebInput(""); - setError(null); - try { - const document = await processWeb.mutateAsync({ integrationId, url }); - setWebItems((previous) => - previous.some((item) => item.collectionName === document.collectionName) - ? previous - : [...previous, { url, collectionName: document.collectionName, title: document.title }], - ); - setAttachOpened(false); - } catch { - setError(t("widget.openWebUi.webError")); - } - }; - - const toggleFile = (file: FileAttachment) => - setFileItems((previous) => - previous.some((item) => item.id === file.id) - ? previous.filter((item) => item.id !== file.id) - : [...previous, file], - ); - - // The list endpoint truncates note bodies, so fetch the full note on attach. - const toggleNote = async (note: { id: string; title: string }) => { - if (!integrationId) return; - if (noteItems.some((item) => item.id === note.id)) { - setNoteItems((previous) => previous.filter((item) => item.id !== note.id)); - return; - } - try { - const full = await utils.widget.openWebUi.getNote.fetch({ integrationId, noteId: note.id }); - setNoteItems((previous) => [...previous, { id: full.id, title: full.title, content: full.content }]); - } catch { - setError(t("widget.openWebUi.noteError")); - } - }; - - const toggleKnowledge = (id: string) => - setKnowledgeIds((previous) => - previous.includes(id) ? previous.filter((value) => value !== id) : [...previous, id], - ); - - const toggleChatReference = async (chat: { id: string; title?: string | null }) => { - if (!integrationId) return; - if (chatItems.some((item) => item.id === chat.id)) { - setChatItems((previous) => previous.filter((item) => item.id !== chat.id)); - return; - } - try { - const full = await utils.widget.openWebUi.getChat.fetch({ integrationId, chatId: chat.id }); - const transcript = extractChatMessages(full.chat) - .map((message) => `${message.role}: ${message.content}`) - .join("\n"); - setChatItems((previous) => [...previous, { id: chat.id, title: chat.title ?? chat.id, transcript }]); - } catch { - setError(t("widget.openWebUi.chatError")); - } - }; - - // ---- Camera capture ---- - const stopCaptureStream = () => { - captureStreamRef.current?.getTracks().forEach((track) => track.stop()); - captureStreamRef.current = null; - }; - - const openCapture = async () => { - setAttachOpened(false); - try { - const stream = await navigator.mediaDevices.getUserMedia({ video: true }); - captureStreamRef.current = stream; - setCaptureOpened(true); - } catch { - setError(t("widget.openWebUi.captureError")); - } - }; - - const closeCapture = () => { - stopCaptureStream(); - setCaptureOpened(false); - }; - - // Attach the live stream the moment the