feat(open-webui): add image, knowledge and webpage attachments with retry

Extend the AI Chat widget with three ways to ground a conversation:

- Images: attach images to a message, sent inline as OpenAI multimodal
  content parts for vision-capable models. A hint warns when the selected
  model likely has no vision support.
- Knowledge bases: pick RAG collections exposed by /api/v1/knowledge/.
- Web pages: ingest a URL via /api/v1/retrieval/process/web.

Open WebUI does not run RAG from the chat-completions `files` parameter,
so attached collections (web pages and knowledge bases) are grounded by
retrieving the most relevant chunks at send time
(/api/v1/retrieval/query/collection) and injecting them as system context.

Also add a Retry action to re-run the last turn after a model failure, and
handle the paginated `{ items, total }` shape of the knowledge endpoint on
newer Open WebUI versions.

packages/widgets/src/open-webui/chat.module.css

@@ -38,6 +38,20 @@
   max-width: 85%;
 }
 
+.thumb {
+  position: relative;
+  line-height: 0;
+}
+
+.thumbRemove {
+  position: absolute;
+  top: -6px;
+  right: -6px;
+  background-color: var(--mantine-color-body);
+  border-radius: var(--mantine-radius-xl);
+  box-shadow: var(--mantine-shadow-xs);
+}
+
 .historyOverlay {
   position: absolute;
   inset: 0;

packages/widgets/src/open-webui/chat.tsx

@@ -4,26 +4,39 @@ import { useEffect, useMemo, useRef, useState } from "react";
 import {
   ActionIcon,
   Avatar,
+  Badge,
   Box,
+  Button,
+  Checkbox,
+  CloseButton,
   Group,
+  Image,
   Loader,
   NavLink,
   Paper,
+  Popover,
   ScrollArea,
   Select,
   Stack,
   Text,
   Textarea,
+  TextInput,
   Tooltip,
 } from "@mantine/core";
 import {
+  IconAlertTriangle,
   IconArrowLeft,
   IconArrowUp,
+  IconDatabase,
   IconHistory,
+  IconPaperclip,
+  IconPhoto,
   IconPlayerStopFilled,
   IconPlus,
+  IconRefresh,
   IconRobot,
   IconUser,
+  IconWorld,
 } from "@tabler/icons-react";
 
 import { clientApi } from "@homarr/api/client";
@@ -35,8 +48,28 @@ import classes from "./chat.module.css";
 interface ChatMessage {
   role: "system" | "user" | "assistant";
   content: string;
+  images?: string[];
 }
 
+type CompletionPart = { type: "text"; text: string } | { type: "image_url"; image_url: { url: string } };
+type CompletionMessage = { role: ChatMessage["role"]; content: string | CompletionPart[] };
+
+interface ImageAttachment {
+  name: string;
+  dataUrl: string;
+}
+
+interface WebAttachment {
+  url: string;
+  collectionName: string;
+  title: string;
+}
+
+// Heuristic fallback for detecting vision-capable models when Open WebUI does
+// not declare the capability explicitly.
+const VISION_MODEL_PATTERN =
+  /vision|llava|bakllava|moondream|minicpm-?v|qwen2?-?vl|pixtral|cogvlm|internvl|gemma3|smolvlm|granite[\w-]*vision/i;
+
 export function Chat({ options, integrationIds, isEditMode }: WidgetComponentProps<"openWebUi">) {
   const t = useI18n();
   const integrationId = integrationIds.at(0);
@@ -46,16 +79,26 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
   const [input, setInput] = useState("");
   const [streamingText, setStreamingText] = useState("");
   const [isStreaming, setIsStreaming] = useState(false);
-  const [pendingMessages, setPendingMessages] = useState<ChatMessage[]>([]);
+  const [pendingMessages, setPendingMessages] = useState<CompletionMessage[]>([]);
+  const [pendingCollections, setPendingCollections] = useState<string[]>([]);
   const [activeChatId, setActiveChatId] = useState<string | null>(null);
   const [selectedChatId, setSelectedChatId] = useState<string | null>(null);
   const [error, setError] = useState<string | null>(null);
   const [historyOpened, setHistoryOpened] = useState(false);
 
+  // Attachments grounding the conversation. Images are per-message; knowledge
+  // collections and web pages are sticky context kept until removed.
+  const [attachments, setAttachments] = useState<ImageAttachment[]>([]);
+  const [knowledgeIds, setKnowledgeIds] = useState<string[]>([]);
+  const [webItems, setWebItems] = useState<WebAttachment[]>([]);
+  const [webInput, setWebInput] = useState("");
+  const [attachOpened, setAttachOpened] = useState(false);
+
   const messagesRef = useRef<ChatMessage[]>([]);
   const streamingTextRef = useRef("");
   const viewportRef = useRef<HTMLDivElement>(null);
   const composerRef = useRef<HTMLDivElement>(null);
+  const fileInputRef = useRef<HTMLInputElement>(null);
   const [composerHeight, setComposerHeight] = useState(72);
 
   const utils = clientApi.useUtils();
@@ -65,6 +108,11 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
     { enabled: Boolean(integrationId), refetchOnWindowFocus: false, retry: false },
   );
 
+  const { data: knowledge = [] } = clientApi.widget.openWebUi.getKnowledge.useQuery(
+    { integrationId: integrationId ?? "" },
+    { enabled: Boolean(integrationId), refetchOnWindowFocus: false, retry: false },
+  );
+
   const { data: chats = [] } = clientApi.widget.openWebUi.getChats.useQuery(
     { integrationId: integrationId ?? "" },
     { enabled: Boolean(integrationId) && options.showHistory, refetchOnWindowFocus: false, retry: false },
@@ -77,6 +125,7 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
 
   const createChat = clientApi.widget.openWebUi.createChat.useMutation();
   const updateChat = clientApi.widget.openWebUi.updateChat.useMutation();
+  const processWeb = clientApi.widget.openWebUi.processWeb.useMutation();
 
   // Default to the first available model once the list loads.
   useEffect(() => {
@@ -113,7 +162,12 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
   }, [messages, streamingText, composerHeight]);
 
   clientApi.widget.openWebUi.sendMessage.useSubscription(
-    { integrationId: integrationId ?? "", model: model ?? "", messages: pendingMessages },
+    {
+      integrationId: integrationId ?? "",
+      model: model ?? "",
+      messages: pendingMessages,
+      collections: pendingCollections.length > 0 ? pendingCollections : undefined,
+    },
     {
       enabled: isStreaming && pendingMessages.length > 0,
       onData: (event) => {
@@ -146,12 +200,18 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
     setStreamingText("");
     setIsStreaming(false);
     setPendingMessages([]);
+    setPendingCollections([]);
   };
 
   const persistChat = async (finalMessages: ChatMessage[]) => {
     if (!integrationId || !model) return;
+    // Native history stores plain text; image data URLs are dropped on persist.
     const title = finalMessages.find((message) => message.role === "user")?.content.slice(0, 80) ?? "New chat";
-    const payload = { title, models: [model], messages: finalMessages };
+    const payload = {
+      title,
+      models: [model],
+      messages: finalMessages.map(({ role, content }) => ({ role, content })),
+    };
     try {
       if (activeChatId) {
         await updateChat.mutateAsync({ integrationId, chatId: activeChatId, chat: payload });
@@ -167,26 +227,56 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
     }
   };
 
-  const handleSend = () => {
-    const trimmed = input.trim();
-    if (!trimmed || !model || !integrationId || isStreaming) return;
+  // Kick off a completion for the given conversation (ending with a user message).
+  // Shared by sending a new message and retrying after a failure.
+  const startCompletion = (baseMessages: ChatMessage[]) => {
+    if (!model || !integrationId) return;
     setError(null);
 
-    const userMessage: ChatMessage = { role: "user", content: trimmed };
+    const systemPrompt = options.systemPrompt.trim();
+    const payloadMessages: ChatMessage[] = systemPrompt
+      ? [{ role: "system", content: systemPrompt }, ...baseMessages]
+      : baseMessages;
+
+    // Knowledge ids double as their collection name; web pages bring their own.
+    const collections = [...knowledgeIds, ...webItems.map((item) => item.collectionName)];
+
+    streamingTextRef.current = "";
+    setStreamingText("");
+    setPendingMessages(toCompletionMessages(payloadMessages));
+    setPendingCollections(collections);
+    setIsStreaming(true);
+  };
+
+  const handleSend = () => {
+    const trimmed = input.trim();
+    if ((!trimmed && attachments.length === 0) || !model || !integrationId || isStreaming) return;
+
+    const userMessage: ChatMessage = {
+      role: "user",
+      content: trimmed,
+      images: attachments.length > 0 ? attachments.map((attachment) => attachment.dataUrl) : undefined,
+    };
     const nextMessages = [...messagesRef.current, userMessage];
     setMessages(nextMessages);
     messagesRef.current = nextMessages;
     setInput("");
+    setAttachments([]);
 
-    const systemPrompt = options.systemPrompt.trim();
-    const payloadMessages: ChatMessage[] = systemPrompt
-      ? [{ role: "system", content: systemPrompt }, ...nextMessages]
-      : nextMessages;
+    startCompletion(nextMessages);
+  };
 
-    streamingTextRef.current = "";
-    setStreamingText("");
-    setPendingMessages(payloadMessages);
-    setIsStreaming(true);
+  // Retry the last turn after a failure: drop any partial assistant reply and
+  // re-run the completion from the most recent user message.
+  const handleRetry = () => {
+    if (isStreaming) return;
+    const lastUserIndex = messagesRef.current.findLastIndex((message) => message.role === "user");
+    if (lastUserIndex === -1) return;
+
+    const truncated = messagesRef.current.slice(0, lastUserIndex + 1);
+    setMessages(truncated);
+    messagesRef.current = truncated;
+    startCompletion(truncated);
   };
 
   const handleNewChat = () => {
@@ -199,10 +289,56 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
     setStreamingText("");
     setIsStreaming(false);
     setPendingMessages([]);
+    setPendingCollections([]);
+    setAttachments([]);
+    setKnowledgeIds([]);
+    setWebItems([]);
     setHistoryOpened(false);
   };
 
+  const handleFilesSelected = async (fileList: FileList | null) => {
+    if (!fileList) return;
+    const images = await Promise.all(
+      Array.from(fileList)
+        .filter((file) => file.type.startsWith("image/"))
+        .map(async (file) => ({ name: file.name, dataUrl: await readFileAsDataUrl(file) })),
+    );
+    setAttachments((previous) => [...previous, ...images]);
+  };
+
+  const addWebUrl = async () => {
+    if (!integrationId) return;
+    const raw = webInput.trim();
+    if (!raw) return;
+    const url = /^https?:\/\//i.test(raw) ? raw : `https://${raw}`;
+    setWebInput("");
+    setError(null);
+    try {
+      const document = await processWeb.mutateAsync({ integrationId, url });
+      setWebItems((previous) =>
+        previous.some((item) => item.collectionName === document.collectionName)
+          ? previous
+          : [...previous, { url, collectionName: document.collectionName, title: document.title }],
+      );
+    } catch {
+      setError(t("widget.openWebUi.webError"));
+    }
+  };
+
   const modelData = useMemo(() => models.map((m) => ({ value: m.id, label: m.name ?? m.id })), [models]);
+  const knowledgeName = (id: string) => knowledge.find((item) => item.id === id)?.name ?? id;
+  const hasAttachments = attachments.length > 0 || knowledgeIds.length > 0 || webItems.length > 0;
+
+  // Whether the selected model can actually see attached images. Prefer Open
+  // WebUI's declared capability; fall back to a name heuristic when it's absent.
+  const selectedModel = useMemo(() => models.find((item) => item.id === model), [models, model]);
+  const modelSupportsVision = useMemo(() => {
+    if (!selectedModel) return true;
+    const capability = selectedModel.info?.meta?.capabilities?.vision;
+    if (typeof capability === "boolean") return capability;
+    return VISION_MODEL_PATTERN.test(selectedModel.id) || VISION_MODEL_PATTERN.test(selectedModel.name ?? "");
+  }, [selectedModel]);
+  const showVisionWarning = attachments.length > 0 && !modelSupportsVision;
 
   return (
     <Box className={classes.root} style={isEditMode ? { pointerEvents: "none" } : undefined}>
@@ -215,18 +351,92 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
             </Stack>
           ) : null}
           {messages.map((message, index) => (
-            <MessageBubble key={index} from={message.role} content={message.content} />
+            <MessageBubble key={index} from={message.role} content={message.content} images={message.images} />
           ))}
           {isStreaming ? <MessageBubble from="assistant" content={streamingText || "…"} /> : null}
           {error ? (
-            <Text size="xs" c="red.6">
-              {error}
-            </Text>
+            <Group gap="xs" justify="center" wrap="nowrap">
+              <Text size="xs" c="red.6">
+                {error}
+              </Text>
+              <Button
+                size="compact-xs"
+                variant="light"
+                color="red"
+                leftSection={<IconRefresh size={14} />}
+                onClick={handleRetry}
+              >
+                {t("widget.openWebUi.retry")}
+              </Button>
+            </Group>
           ) : null}
         </Stack>
       </ScrollArea>
 
       <Paper ref={composerRef} className={classes.composer} withBorder radius="md" p="xs">
+        {hasAttachments ? (
+          <Group gap={6} mb={6}>
+            {attachments.map((attachment, index) => (
+              <Box key={`${attachment.name}-${index}`} className={classes.thumb}>
+                <Image src={attachment.dataUrl} alt={attachment.name} w={48} h={48} radius="sm" fit="cover" />
+                <CloseButton
+                  size="xs"
+                  className={classes.thumbRemove}
+                  onClick={() => setAttachments((previous) => previous.filter((_, i) => i !== index))}
+                  aria-label={t("widget.openWebUi.removeAttachment")}
+                />
+              </Box>
+            ))}
+            {knowledgeIds.map((id) => (
+              <Badge
+                key={id}
+                variant="light"
+                color="grape"
+                leftSection={<IconDatabase size={12} />}
+                rightSection={
+                  <CloseButton
+                    size="xs"
+                    onClick={() => setKnowledgeIds((previous) => previous.filter((value) => value !== id))}
+                    aria-label={t("widget.openWebUi.removeAttachment")}
+                  />
+                }
+              >
+                {knowledgeName(id)}
+              </Badge>
+            ))}
+            {webItems.map((item) => (
+              <Badge
+                key={item.collectionName}
+                variant="light"
+                color="blue"
+                leftSection={<IconWorld size={12} />}
+                rightSection={
+                  <CloseButton
+                    size="xs"
+                    onClick={() =>
+                      setWebItems((previous) =>
+                        previous.filter((value) => value.collectionName !== item.collectionName),
+                      )
+                    }
+                    aria-label={t("widget.openWebUi.removeAttachment")}
+                  />
+                }
+              >
+                {hostnameOf(item.url)}
+              </Badge>
+            ))}
+          </Group>
+        ) : null}
+
+        {showVisionWarning ? (
+          <Group gap={4} mb={6} wrap="nowrap">
+            <IconAlertTriangle size={14} color="var(--mantine-color-yellow-6)" />
+            <Text size="xs" c="yellow.7">
+              {t("widget.openWebUi.visionWarning")}
+            </Text>
+          </Group>
+        ) : null}
+
         <Textarea
           variant="unstyled"
           size="sm"
@@ -255,6 +465,86 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
                 <IconPlus size={18} />
               </ActionIcon>
             </Tooltip>
+            <Popover
+              opened={attachOpened}
+              onChange={setAttachOpened}
+              position="top-start"
+              withinPortal={false}
+              width={260}
+              shadow="md"
+            >
+              <Popover.Target>
+                <Tooltip label={t("widget.openWebUi.attach")} withinPortal={false}>
+                  <ActionIcon
+                    variant="subtle"
+                    color="gray"
+                    onClick={() => setAttachOpened((value) => !value)}
+                    aria-label={t("widget.openWebUi.attach")}
+                  >
+                    <IconPaperclip size={18} />
+                  </ActionIcon>
+                </Tooltip>
+              </Popover.Target>
+              <Popover.Dropdown>
+                <Stack gap="sm">
+                  <Button
+                    variant="light"
+                    color="gray"
+                    size="xs"
+                    leftSection={<IconPhoto size={16} />}
+                    justify="flex-start"
+                    onClick={() => fileInputRef.current?.click()}
+                  >
+                    {t("widget.openWebUi.attachImage")}
+                  </Button>
+
+                  {knowledge.length > 0 ? (
+                    <Stack gap={4}>
+                      <Text size="xs" fw={600} c="dimmed">
+                        {t("widget.openWebUi.attachKnowledge")}
+                      </Text>
+                      <Checkbox.Group value={knowledgeIds} onChange={setKnowledgeIds}>
+                        <Stack gap={4}>
+                          {knowledge.map((item) => (
+                            <Checkbox key={item.id} value={item.id} label={item.name ?? item.id} size="xs" />
+                          ))}
+                        </Stack>
+                      </Checkbox.Group>
+                    </Stack>
+                  ) : null}
+
+                  <Stack gap={4}>
+                    <Text size="xs" fw={600} c="dimmed">
+                      {t("widget.openWebUi.attachWebpage")}
+                    </Text>
+                    <Group gap={4} wrap="nowrap">
+                      <TextInput
+                        size="xs"
+                        flex={1}
+                        placeholder="https://…"
+                        value={webInput}
+                        onChange={(event) => setWebInput(event.currentTarget.value)}
+                        onKeyDown={(event) => {
+                          if (event.key === "Enter") {
+                            event.preventDefault();
+                            void addWebUrl();
+                          }
+                        }}
+                      />
+                      <ActionIcon
+                        variant="light"
+                        color="gray"
+                        loading={processWeb.isPending}
+                        onClick={() => void addWebUrl()}
+                        aria-label={t("widget.openWebUi.addUrl")}
+                      >
+                        <IconPlus size={16} />
+                      </ActionIcon>
+                    </Group>
+                  </Stack>
+                </Stack>
+              </Popover.Dropdown>
+            </Popover>
             {options.showHistory ? (
               <Tooltip label={t("widget.openWebUi.history")} withinPortal={false}>
                 <ActionIcon
@@ -301,7 +591,7 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
                   radius="xl"
                   variant="filled"
                   onClick={handleSend}
-                  disabled={!input.trim() || !model}
+                  disabled={(!input.trim() && attachments.length === 0) || !model}
                   aria-label={t("widget.openWebUi.send")}
                 >
                   <IconArrowUp size={18} />
@@ -310,6 +600,19 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
             )}
           </Group>
         </Group>
+
+        <input
+          ref={fileInputRef}
+          type="file"
+          accept="image/*"
+          multiple
+          hidden
+          aria-label={t("widget.openWebUi.attachImage")}
+          onChange={(event) => {
+            void handleFilesSelected(event.currentTarget.files);
+            event.currentTarget.value = "";
+          }}
+        />
       </Paper>
 
       {options.showHistory && historyOpened ? (
@@ -353,6 +656,37 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
   );
 }
 
+const readFileAsDataUrl = (file: File): Promise<string> =>
+  new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.addEventListener("load", () => resolve(reader.result as string));
+    reader.addEventListener("error", () => reject(reader.error ?? new Error("Failed to read file")));
+    reader.readAsDataURL(file);
+  });
+
+const hostnameOf = (url: string): string => {
+  try {
+    return new URL(url).hostname || url;
+  } catch {
+    return url;
+  }
+};
+
+// Convert displayed messages to the OpenAI completion format, inlining any
+// attached images on the message that carries them (vision content parts).
+const toCompletionMessages = (messages: ChatMessage[]): CompletionMessage[] =>
+  messages.map((message) =>
+    message.images && message.images.length > 0
+      ? {
+          role: message.role,
+          content: [
+            ...(message.content ? [{ type: "text" as const, text: message.content }] : []),
+            ...message.images.map((url) => ({ type: "image_url" as const, image_url: { url } })),
+          ],
+        }
+      : { role: message.role, content: message.content },
+  );
+
 type RawMessage = { role?: string | null; content?: string | null };
 type HistoryMessage = RawMessage & { parentId?: string | null };
 interface RawChat {
@@ -393,7 +727,7 @@ const extractChatMessages = (chat: RawChat | null | undefined): ChatMessage[] =>
     .filter((message): message is ChatMessage => message !== null);
 };
 
-function MessageBubble({ from, content }: { from: ChatMessage["role"]; content: string }) {
+function MessageBubble({ from, content, images }: { from: ChatMessage["role"]; content: string; images?: string[] }) {
   const isUser = from === "user";
   const avatar = (
     <Avatar size={22} radius="xl" variant="filled" color={isUser ? "gray" : "blue"} className={classes.avatar}>
@@ -410,9 +744,18 @@ function MessageBubble({ from, content }: { from: ChatMessage["role"]; content:
         p="xs"
         radius="md"
       >
-        <Text size="sm" style={{ whiteSpace: "pre-wrap", wordBreak: "break-word" }}>
-          {content}
-        </Text>
+        {images && images.length > 0 ? (
+          <Group gap={4} mb={content ? 6 : 0}>
+            {images.map((src, index) => (
+              <Image key={index} src={src} alt="" w={120} radius="sm" fit="contain" />
+            ))}
+          </Group>
+        ) : null}
+        {content ? (
+          <Text size="sm" style={{ whiteSpace: "pre-wrap", wordBreak: "break-word" }}>
+            {content}
+          </Text>
+        ) : null}
       </Paper>
       {isUser ? avatar : null}
     </Group>