feat(open-webui): voice messages, fix camera capture and full notes

- Voice messages: record from the microphone and transcribe to the input
  via Open WebUI's /api/v1/audio/transcriptions (mic button in composer).
- Camera capture: attach the stream via a ref callback so the preview is
  no longer a black frame (the ref could be read before the video mounted).
- Notes: fetch the full note body on attach. The list endpoint truncates
  content to 1000 chars, so attached notes were cut off; getNote returns
  the complete markdown.

apps/docs/docs/widgets/open-webui/index.mdx

@@ -46,6 +46,7 @@ The AI Chat widget lets you chat with AI models served by Open WebUI directly fr
 | **Knowledge bases**     | Ground replies on any RAG knowledge collection your account owns, with an expandable picker to inspect a base's files.                     |
 | **Notes**               | Attach the contents of your Open WebUI notes.                                                                                              |
 | **Reference chats**     | Bring a previous conversation in as context for the current one.                                                                          |
+| **Voice messages**      | Record from your microphone and have Open WebUI transcribe it into the message box (requires STT configured in Open WebUI).                |
 
 ### Notes
 

packages/api/src/router/widgets/open-webui.ts

@@ -10,6 +10,7 @@ import {
   openWebUiKnowledgeFilesRequestHandler,
   openWebUiKnowledgeRequestHandler,
   openWebUiModelsRequestHandler,
+  openWebUiNoteRequestHandler,
   openWebUiNotesRequestHandler,
 } from "@homarr/request-handler/open-webui";
 
@@ -119,6 +120,15 @@ export const openWebUiRouter = createTRPCRouter({
     return data;
   }),
 
+  getNote: publicProcedure
+    .input(z.object({ noteId: z.string() }))
+    .concat(createOneIntegrationMiddleware("query", "openWebUi"))
+    .query(async ({ ctx, input }) => {
+      const handler = openWebUiNoteRequestHandler.handler(ctx.integration, { noteId: input.noteId });
+      const { data } = await handler.getCachedOrUpdatedDataAsync({ forceUpdate: false });
+      return data;
+    }),
+
   getKnowledgeFiles: publicProcedure
     .input(z.object({ knowledgeId: z.string() }))
     .concat(createOneIntegrationMiddleware("query", "openWebUi"))
@@ -150,6 +160,16 @@ export const openWebUiRouter = createTRPCRouter({
       return await client.uploadFileAsync(input.filename, input.contentBase64, input.contentType);
     }),
 
+  // Transcribe recorded audio to text for voice messages.
+  transcribe: protectedProcedure
+    .input(z.object({ filename: z.string(), contentBase64: z.string(), contentType: z.string() }))
+    .concat(createOneIntegrationMiddleware("interact", "openWebUi"))
+    .mutation(async ({ ctx, input }) => {
+      const client = await createIntegrationAsync(ctx.integration);
+      const text = await client.transcribeAudioAsync(input.filename, input.contentBase64, input.contentType);
+      return { text };
+    }),
+
   getChat: publicProcedure
     .input(z.object({ chatId: z.string() }))
     .concat(createOneIntegrationMiddleware("query", "openWebUi"))

packages/integrations/src/open-webui/open-webui-integration.ts

@@ -28,7 +28,9 @@ import {
   openWebUiKnowledgeListSchema,
   openWebUiModelsResponseSchema,
   openWebUiNoteListSchema,
+  openWebUiNoteSchema,
   openWebUiProcessWebResponseSchema,
+  openWebUiTranscriptionSchema,
   openWebUiUploadedFileSchema,
 } from "./open-webui-types";
 
@@ -132,6 +134,43 @@ export class OpenWebUiIntegration extends Integration {
     }));
   }
 
+  /**
+   * Get a single note including its full markdown content. The list endpoint
+   * truncates the body, so we fetch the note directly when attaching it.
+   */
+  public async getNoteAsync(noteId: string): Promise<OpenWebUiNote> {
+    const response = await fetchWithTrustedCertificatesAsync(this.url(`/api/v1/notes/${noteId}`), {
+      headers: this.getAuthHeaders(),
+    });
+
+    if (!response.ok) {
+      throw new ResponseError(response);
+    }
+
+    const parsed = openWebUiNoteSchema.parse(await response.json());
+    return { id: parsed.id, title: parsed.title ?? "Untitled note", content: parsed.data?.content?.md ?? "" };
+  }
+
+  /**
+   * Transcribe recorded audio to text (speech-to-text) for voice messages.
+   */
+  public async transcribeAudioAsync(filename: string, base64Content: string, contentType: string): Promise<string> {
+    const form = new FormData();
+    form.append("file", new Blob([Buffer.from(base64Content, "base64")], { type: contentType }), filename);
+
+    const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/audio/transcriptions"), {
+      method: "POST",
+      headers: this.getAuthHeaders(),
+      body: form,
+    });
+
+    if (!response.ok) {
+      throw new ResponseError(response);
+    }
+
+    return openWebUiTranscriptionSchema.parse(await response.json()).text;
+  }
+
   /**
    * Upload a file to Open WebUI so it can be retrieved from later. Returns the
    * new file id (its vector collection is `file-{id}`).

packages/integrations/src/open-webui/open-webui-types.ts

@@ -135,6 +135,9 @@ export interface OpenWebUiNote {
   content: string;
 }
 
+// POST /api/v1/audio/transcriptions — speech-to-text result.
+export const openWebUiTranscriptionSchema = z.object({ text: z.string() });
+
 // A file flattened for the widget: id and a display name.
 export interface OpenWebUiFileSummary {
   id: string;

packages/request-handler/src/open-webui.ts

@@ -77,6 +77,19 @@ export const openWebUiNotesRequestHandler = createCachedIntegrationRequestHandle
   queryKey: "openWebUiNotes",
 });
 
+export const openWebUiNoteRequestHandler = createCachedIntegrationRequestHandler<
+  OpenWebUiNote,
+  "openWebUi",
+  { noteId: string }
+>({
+  async requestAsync(integration, input) {
+    const integrationInstance = await createIntegrationAsync(integration);
+    return await integrationInstance.getNoteAsync(input.noteId);
+  },
+  cacheDuration: dayjs.duration(30, "seconds"),
+  queryKey: "openWebUiNote",
+});
+
 export const openWebUiChatsRequestHandler = createCachedIntegrationRequestHandler<
   OpenWebUiChatListItem[],
   "openWebUi",

packages/translation/src/lang/en.json

@@ -3583,6 +3583,11 @@
       "uploadError": "Could not upload the file",
       "captureError": "Could not access the camera",
       "chatError": "Could not load that chat",
+      "noteError": "Could not load that note",
+      "record": "Record voice message",
+      "stopRecording": "Stop recording",
+      "micError": "Could not access the microphone",
+      "transcribeError": "Could not transcribe the recording",
       "error": {
         "internalServerError": "Failed to reach Open WebUI"
       }

packages/widgets/src/open-webui/chat.tsx

@@ -1,6 +1,6 @@
 "use client";
 
-import { useEffect, useMemo, useRef, useState } from "react";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import {
   ActionIcon,
   Avatar,
@@ -36,6 +36,7 @@ import {
   IconFileText,
   IconHistory,
   IconMessage,
+  IconMicrophone,
   IconPaperclip,
   IconPlayerStopFilled,
   IconPlus,
@@ -122,14 +123,17 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
   const [attachView, setAttachView] = useState<AttachView>("root");
 
   const [captureOpened, setCaptureOpened] = useState(false);
+  const [isRecording, setIsRecording] = useState(false);
 
   const messagesRef = useRef<ChatMessage[]>([]);
   const streamingTextRef = useRef("");
   const viewportRef = useRef<HTMLDivElement>(null);
   const composerRef = useRef<HTMLDivElement>(null);
   const fileInputRef = useRef<HTMLInputElement>(null);
-  const videoRef = useRef<HTMLVideoElement>(null);
   const captureStreamRef = useRef<MediaStream | null>(null);
+  const mediaRecorderRef = useRef<MediaRecorder | null>(null);
+  const audioChunksRef = useRef<Blob[]>([]);
+  const audioStreamRef = useRef<MediaStream | null>(null);
   const [composerHeight, setComposerHeight] = useState(72);
 
   const utils = clientApi.useUtils();
@@ -168,6 +172,7 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
   const updateChat = clientApi.widget.openWebUi.updateChat.useMutation();
   const processWeb = clientApi.widget.openWebUi.processWeb.useMutation();
   const uploadFile = clientApi.widget.openWebUi.uploadFile.useMutation();
+  const transcribe = clientApi.widget.openWebUi.transcribe.useMutation();
 
   // Default to the first available model once the list loads.
   useEffect(() => {
@@ -208,8 +213,14 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
     if (!attachOpened) setAttachView("root");
   }, [attachOpened]);
 
-  // Stop the camera stream on unmount.
-  useEffect(() => () => stopCaptureStream(), []);
+  // Stop the camera and microphone streams on unmount.
+  useEffect(
+    () => () => {
+      stopCaptureStream();
+      audioStreamRef.current?.getTracks().forEach((track) => track.stop());
+    },
+    [],
+  );
 
   clientApi.widget.openWebUi.sendMessage.useSubscription(
     {
@@ -414,12 +425,20 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
         : [...previous, file],
     );
 
-  const toggleNote = (note: NoteAttachment) =>
-    setNoteItems((previous) =>
-      previous.some((item) => item.id === note.id)
-        ? previous.filter((item) => item.id !== note.id)
-        : [...previous, note],
-    );
+  // The list endpoint truncates note bodies, so fetch the full note on attach.
+  const toggleNote = async (note: { id: string; title: string }) => {
+    if (!integrationId) return;
+    if (noteItems.some((item) => item.id === note.id)) {
+      setNoteItems((previous) => previous.filter((item) => item.id !== note.id));
+      return;
+    }
+    try {
+      const full = await utils.widget.openWebUi.getNote.fetch({ integrationId, noteId: note.id });
+      setNoteItems((previous) => [...previous, { id: full.id, title: full.title, content: full.content }]);
+    } catch {
+      setError(t("widget.openWebUi.noteError"));
+    }
+  };
 
   const toggleKnowledge = (id: string) =>
     setKnowledgeIds((previous) =>
@@ -465,11 +484,16 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
     setCaptureOpened(false);
   };
 
-  useEffect(() => {
-    if (captureOpened && videoRef.current && captureStreamRef.current) {
-      videoRef.current.srcObject = captureStreamRef.current;
+  // Attach the live stream the moment the <video> mounts (avoids a ref race
+  // where an effect runs before the modal's video element exists).
+  const videoRef = useRef<HTMLVideoElement | null>(null);
+  const setVideoNode = useCallback((node: HTMLVideoElement | null) => {
+    videoRef.current = node;
+    if (node && captureStreamRef.current) {
+      node.srcObject = captureStreamRef.current;
+      void node.play().catch(() => undefined);
     }
-  }, [captureOpened]);
+  }, []);
 
   const takePhoto = () => {
     const video = videoRef.current;
@@ -485,6 +509,54 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
     closeCapture();
   };
 
+  // ---- Voice recording (speech-to-text) ----
+  const startRecording = async () => {
+    try {
+      const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+      audioStreamRef.current = stream;
+      const recorder = new MediaRecorder(stream);
+      audioChunksRef.current = [];
+      recorder.addEventListener("dataavailable", (event) => {
+        if (event.data.size > 0) audioChunksRef.current.push(event.data);
+      });
+      recorder.addEventListener("stop", () => void transcribeRecording());
+      mediaRecorderRef.current = recorder;
+      recorder.start();
+      setIsRecording(true);
+      setError(null);
+    } catch {
+      setError(t("widget.openWebUi.micError"));
+    }
+  };
+
+  const stopRecording = () => {
+    mediaRecorderRef.current?.stop();
+    setIsRecording(false);
+  };
+
+  const transcribeRecording = async () => {
+    audioStreamRef.current?.getTracks().forEach((track) => track.stop());
+    audioStreamRef.current = null;
+    const chunks = audioChunksRef.current;
+    audioChunksRef.current = [];
+    if (chunks.length === 0 || !integrationId) return;
+
+    const blob = new Blob(chunks, { type: chunks[0]?.type ?? "audio/webm" });
+    try {
+      const dataUrl = await readBlobAsDataUrl(blob);
+      const result = await transcribe.mutateAsync({
+        integrationId,
+        filename: "recording.webm",
+        contentBase64: dataUrl.split(",")[1] ?? "",
+        contentType: blob.type || "audio/webm",
+      });
+      const text = result.text.trim();
+      if (text) setInput((previous) => (previous ? `${previous} ${text}` : text));
+    } catch {
+      setError(t("widget.openWebUi.transcribeError"));
+    }
+  };
+
   // Drop the noisy ":latest" tag from the label (common for Ollama models); the
   // value keeps the full id so the right model is still sent.
   const modelData = useMemo(
@@ -607,7 +679,7 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
                 color="yellow"
                 icon={<IconFileText size={12} />}
                 label={item.title}
-                onRemove={() => toggleNote(item)}
+                onRemove={() => void toggleNote(item)}
                 removeLabel={t("widget.openWebUi.removeAttachment")}
               />
             ))}
@@ -792,7 +864,7 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
                             size="xs"
                             label={note.title}
                             checked={noteItems.some((item) => item.id === note.id)}
-                            onChange={() => toggleNote({ id: note.id, title: note.title, content: note.content })}
+                            onChange={() => void toggleNote({ id: note.id, title: note.title })}
                           />
                         ))}
                       </PickerList>
@@ -844,6 +916,20 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
                 </ActionIcon>
               </Tooltip>
             ) : null}
+            <Tooltip
+              label={isRecording ? t("widget.openWebUi.stopRecording") : t("widget.openWebUi.record")}
+              withinPortal={false}
+            >
+              <ActionIcon
+                variant={isRecording ? "filled" : "subtle"}
+                color={isRecording ? "red" : "gray"}
+                loading={transcribe.isPending}
+                onClick={() => (isRecording ? stopRecording() : void startRecording())}
+                aria-label={t("widget.openWebUi.record")}
+              >
+                {isRecording ? <IconPlayerStopFilled size={16} /> : <IconMicrophone size={18} />}
+              </ActionIcon>
+            </Tooltip>
           </Group>
           <Group gap="xs" wrap="nowrap">
             <Select
@@ -904,9 +990,10 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
       <Modal opened={captureOpened} onClose={closeCapture} title={t("widget.openWebUi.capture")} centered>
         <Stack gap="sm">
           <video
-            ref={videoRef}
+            ref={setVideoNode}
             autoPlay
             playsInline
+            muted
             aria-label={t("widget.openWebUi.capture")}
             className={classes.captureVideo}
           >
@@ -964,14 +1051,16 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
   );
 }
 
-const readFileAsDataUrl = (file: File): Promise<string> =>
+const readBlobAsDataUrl = (blob: Blob): Promise<string> =>
   new Promise((resolve, reject) => {
     const reader = new FileReader();
     reader.addEventListener("load", () => resolve(reader.result as string));
     reader.addEventListener("error", () => reject(reader.error ?? new Error("Failed to read file")));
-    reader.readAsDataURL(file);
+    reader.readAsDataURL(blob);
   });
 
+const readFileAsDataUrl = readBlobAsDataUrl;
+
 const hostnameOf = (url: string): string => {
   try {
     return new URL(url).hostname || url;