feat(open-webui): add image, knowledge and webpage attachments with retry

Extend the AI Chat widget with three ways to ground a conversation:

- Images: attach images to a message, sent inline as OpenAI multimodal
  content parts for vision-capable models. A hint warns when the selected
  model likely has no vision support.
- Knowledge bases: pick RAG collections exposed by /api/v1/knowledge/.
- Web pages: ingest a URL via /api/v1/retrieval/process/web.

Open WebUI does not run RAG from the chat-completions `files` parameter,
so attached collections (web pages and knowledge bases) are grounded by
retrieving the most relevant chunks at send time
(/api/v1/retrieval/query/collection) and injecting them as system context.

Also add a Retry action to re-run the last turn after a model failure, and
handle the paginated `{ items, total }` shape of the knowledge endpoint on
newer Open WebUI versions.

packages/integrations/src/open-webui/open-webui-integration.ts

@@ -8,15 +8,20 @@ import type { TestingResult } from "../base/test-connection/test-connection-serv
 import type {
   OpenWebUiChat,
   OpenWebUiChatListItem,
-  OpenWebUiChatMessage,
   OpenWebUiChatPayload,
+  OpenWebUiCompletionMessage,
+  OpenWebUiKnowledge,
   OpenWebUiModel,
+  OpenWebUiWebDocument,
 } from "./open-webui-types";
 import {
   openWebUiChatListItemSchema,
   openWebUiChatSchema,
+  openWebUiCollectionQueryResponseSchema,
   openWebUiCompletionChunkSchema,
+  openWebUiKnowledgeListSchema,
   openWebUiModelsResponseSchema,
+  openWebUiProcessWebResponseSchema,
 } from "./open-webui-types";
 
 const logger = createLogger({ module: "open-webui-integration" });
@@ -49,6 +54,65 @@ export class OpenWebUiIntegration extends Integration {
     return openWebUiModelsResponseSchema.parse(await response.json()).data;
   }
 
+  /**
+   * List the knowledge bases (RAG collections) the user can ground chats on.
+   */
+  public async getKnowledgeAsync(): Promise<OpenWebUiKnowledge[]> {
+    const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/knowledge/"), {
+      headers: this.getAuthHeaders(),
+    });
+
+    if (!response.ok) {
+      throw new ResponseError(response);
+    }
+
+    const parsed = openWebUiKnowledgeListSchema.parse(await response.json());
+    return Array.isArray(parsed) ? parsed : parsed.items;
+  }
+
+  /**
+   * Ingest a web page into a temporary vector collection and return a handle to
+   * retrieve from it. Open WebUI does not crawl URLs at completion time, so we
+   * embed the page here and inject retrieved chunks ourselves at send time.
+   */
+  public async processWebAsync(url: string): Promise<OpenWebUiWebDocument> {
+    const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/retrieval/process/web"), {
+      method: "POST",
+      headers: { ...this.getAuthHeaders(), "Content-Type": "application/json" },
+      body: JSON.stringify({ url, collection_name: "" }),
+    });
+
+    if (!response.ok) {
+      throw new ResponseError(response);
+    }
+
+    const parsed = openWebUiProcessWebResponseSchema.parse(await response.json());
+    return { collectionName: parsed.collection_name, title: parsed.file?.meta?.name ?? url };
+  }
+
+  /**
+   * Retrieve the most relevant chunks for a query across the given collections
+   * (web pages and/or knowledge bases). Returns the chunk texts, best-effort.
+   */
+  public async queryCollectionsAsync(collectionNames: string[], query: string, k = 4): Promise<string[]> {
+    if (collectionNames.length === 0 || query.trim() === "") {
+      return [];
+    }
+
+    const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/retrieval/query/collection"), {
+      method: "POST",
+      headers: { ...this.getAuthHeaders(), "Content-Type": "application/json" },
+      body: JSON.stringify({ collection_names: collectionNames, query, k }),
+    });
+
+    if (!response.ok) {
+      throw new ResponseError(response);
+    }
+
+    const parsed = openWebUiCollectionQueryResponseSchema.parse(await response.json());
+    return (parsed.documents ?? []).flat();
+  }
+
   /**
    * List the user's chats (native Open WebUI history).
    */
@@ -119,7 +183,7 @@ export class OpenWebUiIntegration extends Integration {
    * content delta. Resolves when the stream completes ([DONE]) or is aborted.
    */
   public async streamChatCompletionAsync(
-    params: { model: string; messages: OpenWebUiChatMessage[] },
+    params: { model: string; messages: OpenWebUiCompletionMessage[] },
     onDelta: (delta: string) => void,
     signal: AbortSignal,
   ): Promise<void> {

packages/integrations/src/open-webui/open-webui-types.ts

@@ -10,6 +10,17 @@ export const openWebUiModelSchema = z.object({
   name: z.string().nullish(),
   object: z.string().nullish(),
   owned_by: z.string().nullish(),
+  // Open WebUI exposes per-model capabilities here when configured; `vision`
+  // tells us whether the model can accept image attachments.
+  info: z
+    .object({
+      meta: z
+        .object({
+          capabilities: z.object({ vision: z.boolean().nullish() }).nullish(),
+        })
+        .nullish(),
+    })
+    .nullish(),
 });
 
 export const openWebUiModelsResponseSchema = z.object({
@@ -44,6 +55,57 @@ export const openWebUiChatMessageSchema = z.object({
 
 export type OpenWebUiChatMessage = z.infer<typeof openWebUiChatMessageSchema>;
 
+// Multimodal content part (OpenAI vision format). Used only for outbound
+// completion requests so users can attach images to a message.
+export const openWebUiContentPartSchema = z.discriminatedUnion("type", [
+  z.object({ type: z.literal("text"), text: z.string() }),
+  z.object({ type: z.literal("image_url"), image_url: z.object({ url: z.string() }) }),
+]);
+
+// A completion message can carry plain text or multimodal parts (text + images).
+export const openWebUiCompletionMessageSchema = z.object({
+  role: z.enum(["system", "user", "assistant"]),
+  content: z.union([z.string(), z.array(openWebUiContentPartSchema)]),
+});
+
+export type OpenWebUiCompletionMessage = z.infer<typeof openWebUiCompletionMessageSchema>;
+
+// GET /api/v1/knowledge/ — knowledge bases (RAG collections owned by the user).
+// Newer Open WebUI versions paginate the list as `{ items, total }`; older ones
+// return a bare array. Accept both and normalise in the integration.
+export const openWebUiKnowledgeSchema = z.object({
+  id: z.string(),
+  name: z.string().nullish(),
+  description: z.string().nullish(),
+});
+
+export type OpenWebUiKnowledge = z.infer<typeof openWebUiKnowledgeSchema>;
+
+export const openWebUiKnowledgeListSchema = z.union([
+  z.array(openWebUiKnowledgeSchema),
+  z.object({ items: z.array(openWebUiKnowledgeSchema) }),
+]);
+
+// POST /api/v1/retrieval/process/web — ingests a URL into a temporary vector
+// collection. We retrieve from that collection at send time and inject the
+// results as context (Open WebUI does not fetch URLs at completion time).
+export const openWebUiProcessWebResponseSchema = z.object({
+  collection_name: z.string(),
+  filename: z.string().nullish(),
+  file: z.object({ meta: z.object({ name: z.string().nullish() }).nullish() }).nullish(),
+});
+
+// POST /api/v1/retrieval/query/collection — top-k chunks per collection.
+export const openWebUiCollectionQueryResponseSchema = z.object({
+  documents: z.array(z.array(z.string())).nullish(),
+});
+
+// A web page ingested for grounding: the collection to retrieve from + a label.
+export interface OpenWebUiWebDocument {
+  collectionName: string;
+  title: string;
+}
+
 // GET /api/v1/chats/ — list of the user's chats (native Open WebUI shape).
 export const openWebUiChatListItemSchema = z.object({
   id: z.string(),