From 46454cc0f03a2a0842645b993a0aa93b006553a3 Mon Sep 17 00:00:00 2001 From: Lok3sh-Kumar Date: Sat, 27 Jun 2026 14:16:42 +0530 Subject: [PATCH] feat(open-webui): voice messages, fix camera capture and full notes - Voice messages: record from the microphone and transcribe to the input via Open WebUI's /api/v1/audio/transcriptions (mic button in composer). - Camera capture: attach the stream via a ref callback so the preview is no longer a black frame (the ref could be read before the video mounted). - Notes: fetch the full note body on attach. The list endpoint truncates content to 1000 chars, so attached notes were cut off; getNote returns the complete markdown. --- apps/docs/docs/widgets/open-webui/index.mdx | 1 + packages/api/src/router/widgets/open-webui.ts | 20 +++ .../src/open-webui/open-webui-integration.ts | 39 ++++++ .../src/open-webui/open-webui-types.ts | 3 + packages/request-handler/src/open-webui.ts | 13 ++ packages/translation/src/lang/en.json | 5 + packages/widgets/src/open-webui/chat.tsx | 127 +++++++++++++++--- 7 files changed, 189 insertions(+), 19 deletions(-) diff --git a/apps/docs/docs/widgets/open-webui/index.mdx b/apps/docs/docs/widgets/open-webui/index.mdx index 8f5baaf7d..80e0ec25b 100644 --- a/apps/docs/docs/widgets/open-webui/index.mdx +++ b/apps/docs/docs/widgets/open-webui/index.mdx @@ -46,6 +46,7 @@ The AI Chat widget lets you chat with AI models served by Open WebUI directly fr | **Knowledge bases** | Ground replies on any RAG knowledge collection your account owns, with an expandable picker to inspect a base's files. | | **Notes** | Attach the contents of your Open WebUI notes. | | **Reference chats** | Bring a previous conversation in as context for the current one. | +| **Voice messages** | Record from your microphone and have Open WebUI transcribe it into the message box (requires STT configured in Open WebUI). | ### Notes diff --git a/packages/api/src/router/widgets/open-webui.ts b/packages/api/src/router/widgets/open-webui.ts index 7087ae6be..fe15d1312 100644 --- a/packages/api/src/router/widgets/open-webui.ts +++ b/packages/api/src/router/widgets/open-webui.ts @@ -10,6 +10,7 @@ import { openWebUiKnowledgeFilesRequestHandler, openWebUiKnowledgeRequestHandler, openWebUiModelsRequestHandler, + openWebUiNoteRequestHandler, openWebUiNotesRequestHandler, } from "@homarr/request-handler/open-webui"; @@ -119,6 +120,15 @@ export const openWebUiRouter = createTRPCRouter({ return data; }), + getNote: publicProcedure + .input(z.object({ noteId: z.string() })) + .concat(createOneIntegrationMiddleware("query", "openWebUi")) + .query(async ({ ctx, input }) => { + const handler = openWebUiNoteRequestHandler.handler(ctx.integration, { noteId: input.noteId }); + const { data } = await handler.getCachedOrUpdatedDataAsync({ forceUpdate: false }); + return data; + }), + getKnowledgeFiles: publicProcedure .input(z.object({ knowledgeId: z.string() })) .concat(createOneIntegrationMiddleware("query", "openWebUi")) @@ -150,6 +160,16 @@ export const openWebUiRouter = createTRPCRouter({ return await client.uploadFileAsync(input.filename, input.contentBase64, input.contentType); }), + // Transcribe recorded audio to text for voice messages. + transcribe: protectedProcedure + .input(z.object({ filename: z.string(), contentBase64: z.string(), contentType: z.string() })) + .concat(createOneIntegrationMiddleware("interact", "openWebUi")) + .mutation(async ({ ctx, input }) => { + const client = await createIntegrationAsync(ctx.integration); + const text = await client.transcribeAudioAsync(input.filename, input.contentBase64, input.contentType); + return { text }; + }), + getChat: publicProcedure .input(z.object({ chatId: z.string() })) .concat(createOneIntegrationMiddleware("query", "openWebUi")) diff --git a/packages/integrations/src/open-webui/open-webui-integration.ts b/packages/integrations/src/open-webui/open-webui-integration.ts index cfce4b9e6..943d9b2c3 100644 --- a/packages/integrations/src/open-webui/open-webui-integration.ts +++ b/packages/integrations/src/open-webui/open-webui-integration.ts @@ -28,7 +28,9 @@ import { openWebUiKnowledgeListSchema, openWebUiModelsResponseSchema, openWebUiNoteListSchema, + openWebUiNoteSchema, openWebUiProcessWebResponseSchema, + openWebUiTranscriptionSchema, openWebUiUploadedFileSchema, } from "./open-webui-types"; @@ -132,6 +134,43 @@ export class OpenWebUiIntegration extends Integration { })); } + /** + * Get a single note including its full markdown content. The list endpoint + * truncates the body, so we fetch the note directly when attaching it. + */ + public async getNoteAsync(noteId: string): Promise { + const response = await fetchWithTrustedCertificatesAsync(this.url(`/api/v1/notes/${noteId}`), { + headers: this.getAuthHeaders(), + }); + + if (!response.ok) { + throw new ResponseError(response); + } + + const parsed = openWebUiNoteSchema.parse(await response.json()); + return { id: parsed.id, title: parsed.title ?? "Untitled note", content: parsed.data?.content?.md ?? "" }; + } + + /** + * Transcribe recorded audio to text (speech-to-text) for voice messages. + */ + public async transcribeAudioAsync(filename: string, base64Content: string, contentType: string): Promise { + const form = new FormData(); + form.append("file", new Blob([Buffer.from(base64Content, "base64")], { type: contentType }), filename); + + const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/audio/transcriptions"), { + method: "POST", + headers: this.getAuthHeaders(), + body: form, + }); + + if (!response.ok) { + throw new ResponseError(response); + } + + return openWebUiTranscriptionSchema.parse(await response.json()).text; + } + /** * Upload a file to Open WebUI so it can be retrieved from later. Returns the * new file id (its vector collection is `file-{id}`). diff --git a/packages/integrations/src/open-webui/open-webui-types.ts b/packages/integrations/src/open-webui/open-webui-types.ts index 0dcee73c3..81c9dd18f 100644 --- a/packages/integrations/src/open-webui/open-webui-types.ts +++ b/packages/integrations/src/open-webui/open-webui-types.ts @@ -135,6 +135,9 @@ export interface OpenWebUiNote { content: string; } +// POST /api/v1/audio/transcriptions — speech-to-text result. +export const openWebUiTranscriptionSchema = z.object({ text: z.string() }); + // A file flattened for the widget: id and a display name. export interface OpenWebUiFileSummary { id: string; diff --git a/packages/request-handler/src/open-webui.ts b/packages/request-handler/src/open-webui.ts index 81162cd66..6edbc548f 100644 --- a/packages/request-handler/src/open-webui.ts +++ b/packages/request-handler/src/open-webui.ts @@ -77,6 +77,19 @@ export const openWebUiNotesRequestHandler = createCachedIntegrationRequestHandle queryKey: "openWebUiNotes", }); +export const openWebUiNoteRequestHandler = createCachedIntegrationRequestHandler< + OpenWebUiNote, + "openWebUi", + { noteId: string } +>({ + async requestAsync(integration, input) { + const integrationInstance = await createIntegrationAsync(integration); + return await integrationInstance.getNoteAsync(input.noteId); + }, + cacheDuration: dayjs.duration(30, "seconds"), + queryKey: "openWebUiNote", +}); + export const openWebUiChatsRequestHandler = createCachedIntegrationRequestHandler< OpenWebUiChatListItem[], "openWebUi", diff --git a/packages/translation/src/lang/en.json b/packages/translation/src/lang/en.json index 247299e39..fd995549c 100644 --- a/packages/translation/src/lang/en.json +++ b/packages/translation/src/lang/en.json @@ -3583,6 +3583,11 @@ "uploadError": "Could not upload the file", "captureError": "Could not access the camera", "chatError": "Could not load that chat", + "noteError": "Could not load that note", + "record": "Record voice message", + "stopRecording": "Stop recording", + "micError": "Could not access the microphone", + "transcribeError": "Could not transcribe the recording", "error": { "internalServerError": "Failed to reach Open WebUI" } diff --git a/packages/widgets/src/open-webui/chat.tsx b/packages/widgets/src/open-webui/chat.tsx index 771e91359..4060f02b1 100644 --- a/packages/widgets/src/open-webui/chat.tsx +++ b/packages/widgets/src/open-webui/chat.tsx @@ -1,6 +1,6 @@ "use client"; -import { useEffect, useMemo, useRef, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { ActionIcon, Avatar, @@ -36,6 +36,7 @@ import { IconFileText, IconHistory, IconMessage, + IconMicrophone, IconPaperclip, IconPlayerStopFilled, IconPlus, @@ -122,14 +123,17 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro const [attachView, setAttachView] = useState("root"); const [captureOpened, setCaptureOpened] = useState(false); + const [isRecording, setIsRecording] = useState(false); const messagesRef = useRef([]); const streamingTextRef = useRef(""); const viewportRef = useRef(null); const composerRef = useRef(null); const fileInputRef = useRef(null); - const videoRef = useRef(null); const captureStreamRef = useRef(null); + const mediaRecorderRef = useRef(null); + const audioChunksRef = useRef([]); + const audioStreamRef = useRef(null); const [composerHeight, setComposerHeight] = useState(72); const utils = clientApi.useUtils(); @@ -168,6 +172,7 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro const updateChat = clientApi.widget.openWebUi.updateChat.useMutation(); const processWeb = clientApi.widget.openWebUi.processWeb.useMutation(); const uploadFile = clientApi.widget.openWebUi.uploadFile.useMutation(); + const transcribe = clientApi.widget.openWebUi.transcribe.useMutation(); // Default to the first available model once the list loads. useEffect(() => { @@ -208,8 +213,14 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro if (!attachOpened) setAttachView("root"); }, [attachOpened]); - // Stop the camera stream on unmount. - useEffect(() => () => stopCaptureStream(), []); + // Stop the camera and microphone streams on unmount. + useEffect( + () => () => { + stopCaptureStream(); + audioStreamRef.current?.getTracks().forEach((track) => track.stop()); + }, + [], + ); clientApi.widget.openWebUi.sendMessage.useSubscription( { @@ -414,12 +425,20 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro : [...previous, file], ); - const toggleNote = (note: NoteAttachment) => - setNoteItems((previous) => - previous.some((item) => item.id === note.id) - ? previous.filter((item) => item.id !== note.id) - : [...previous, note], - ); + // The list endpoint truncates note bodies, so fetch the full note on attach. + const toggleNote = async (note: { id: string; title: string }) => { + if (!integrationId) return; + if (noteItems.some((item) => item.id === note.id)) { + setNoteItems((previous) => previous.filter((item) => item.id !== note.id)); + return; + } + try { + const full = await utils.widget.openWebUi.getNote.fetch({ integrationId, noteId: note.id }); + setNoteItems((previous) => [...previous, { id: full.id, title: full.title, content: full.content }]); + } catch { + setError(t("widget.openWebUi.noteError")); + } + }; const toggleKnowledge = (id: string) => setKnowledgeIds((previous) => @@ -465,11 +484,16 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro setCaptureOpened(false); }; - useEffect(() => { - if (captureOpened && videoRef.current && captureStreamRef.current) { - videoRef.current.srcObject = captureStreamRef.current; + // Attach the live stream the moment the