feat(open-webui): add image, knowledge and webpage attachments with retry

Extend the AI Chat widget with three ways to ground a conversation:

- Images: attach images to a message, sent inline as OpenAI multimodal
  content parts for vision-capable models. A hint warns when the selected
  model likely has no vision support.
- Knowledge bases: pick RAG collections exposed by /api/v1/knowledge/.
- Web pages: ingest a URL via /api/v1/retrieval/process/web.

Open WebUI does not run RAG from the chat-completions `files` parameter,
so attached collections (web pages and knowledge bases) are grounded by
retrieving the most relevant chunks at send time
(/api/v1/retrieval/query/collection) and injecting them as system context.

Also add a Retry action to re-run the last turn after a model failure, and
handle the paginated `{ items, total }` shape of the knowledge endpoint on
newer Open WebUI versions.
This commit is contained in:
Lok3sh-Kumar committed 2026-06-27 14:15:08 +05:30
1 parent 005c2e4cd1
commit f61e7539d7
8 files changed
+624 -30

No files matched your search

+81 -2
View File
@@ -6,6 +6,7 @@ import { createIntegrationAsync } from "@homarr/integrations";
import {
openWebUiChatRequestHandler,
openWebUiChatsRequestHandler,
openWebUiKnowledgeRequestHandler,
openWebUiModelsRequestHandler,
} from "@homarr/request-handler/open-webui";
@@ -19,6 +20,18 @@ const messageSchema = z.object({
content: z.string(),
});
// Outbound completion messages may carry multimodal parts (text + images),
// so users can attach images to a message.
const contentPartSchema = z.discriminatedUnion("type", [
z.object({ type: z.literal("text"), text: z.string() }),
z.object({ type: z.literal("image_url"), image_url: z.object({ url: z.string() }) }),
]);
const completionMessageSchema = z.object({
role: z.enum(["system", "user", "assistant"]),
content: z.union([z.string(), z.array(contentPartSchema)]),
});
const chatPayloadSchema = z.object({
title: z.string(),
models: z.array(z.string()),
@@ -27,6 +40,47 @@ const chatPayloadSchema = z.object({
type StreamEvent = { type: "delta"; content: string } | { type: "done" } | { type: "error"; message: string };
type CompletionMessage = z.infer<typeof completionMessageSchema>;
// Extract the plain text of a message, flattening multimodal content parts.
const messageText = (message: CompletionMessage): string =>
typeof message.content === "string"
? message.content
: message.content
.filter((part): part is { type: "text"; text: string } => part.type === "text")
.map((part) => part.text)
.join(" ");
// Retrieve relevant chunks from the attached collections and prepend them as a
// system context message so the model can ground its reply on them.
const groundMessagesAsync = async (
client: { queryCollectionsAsync: (names: string[], query: string, k?: number) => Promise<string[]> },
messages: CompletionMessage[],
collections: string[],
): Promise<CompletionMessage[]> => {
if (collections.length === 0) return messages;
const lastUserMessage = messages.findLast((message) => message.role === "user");
const query = lastUserMessage ? messageText(lastUserMessage) : "";
let documents: string[] = [];
try {
documents = await client.queryCollectionsAsync(collections, query, 4);
} catch (error) {
logger.warn("Open WebUI context retrieval failed; continuing without it", { error });
}
if (documents.length === 0) return messages;
const context = documents.join("\n\n");
const contextMessage: CompletionMessage = {
role: "system",
content:
"Use the following context from the user's attached sources (web pages and knowledge bases) " +
`to answer. If the answer is not in the context, say so.\n\n${context}`,
};
return [contextMessage, ...messages];
};
export const openWebUiRouter = createTRPCRouter({
getModels: publicProcedure.concat(createOneIntegrationMiddleware("query", "openWebUi")).query(async ({ ctx }) => {
const handler = openWebUiModelsRequestHandler.handler(ctx.integration, {});
@@ -40,6 +94,22 @@ export const openWebUiRouter = createTRPCRouter({
return data;
}),
getKnowledge: publicProcedure.concat(createOneIntegrationMiddleware("query", "openWebUi")).query(async ({ ctx }) => {
const handler = openWebUiKnowledgeRequestHandler.handler(ctx.integration, {});
const { data } = await handler.getCachedOrUpdatedDataAsync({ forceUpdate: false });
return data;
}),
// Ingest a web page so its content can ground later messages. Returns the
// collection to retrieve from plus a display title.
processWeb: protectedProcedure
.input(z.object({ url: z.string().url() }))
.concat(createOneIntegrationMiddleware("interact", "openWebUi"))
.mutation(async ({ ctx, input }) => {
const client = await createIntegrationAsync(ctx.integration);
return await client.processWebAsync(input.url);
}),
getChat: publicProcedure
.input(z.object({ chatId: z.string() }))
.concat(createOneIntegrationMiddleware("query", "openWebUi"))
@@ -78,7 +148,15 @@ export const openWebUiRouter = createTRPCRouter({
// tokens can be emitted as they arrive. Uses "query" access (same level as
// viewing the widget) to match the other integration subscriptions in the app.
sendMessage: publicProcedure
.input(z.object({ model: z.string(), messages: z.array(messageSchema) }))
.input(
z.object({
model: z.string(),
messages: z.array(completionMessageSchema),
// Collections to ground the reply on (web pages + knowledge bases). We
// retrieve relevant chunks and inject them as context before streaming.
collections: z.array(z.string()).optional(),
}),
)
.concat(createOneIntegrationMiddleware("query", "openWebUi"))
.subscription(({ ctx, input }) => {
return observable<StreamEvent>((emit) => {
@@ -87,8 +165,9 @@ export const openWebUiRouter = createTRPCRouter({
void (async () => {
try {
const client = await createIntegrationAsync(ctx.integration);
const messages = await groundMessagesAsync(client, input.messages, input.collections ?? []);
await client.streamChatCompletionAsync(
{ model: input.model, messages: input.messages },
{ model: input.model, messages },
(delta) => emit.next({ type: "delta", content: delta }),
controller.signal,
);
@@ -8,15 +8,20 @@ import type { TestingResult } from "../base/test-connection/test-connection-serv
import type {
OpenWebUiChat,
OpenWebUiChatListItem,
OpenWebUiChatMessage,
OpenWebUiChatPayload,
OpenWebUiCompletionMessage,
OpenWebUiKnowledge,
OpenWebUiModel,
OpenWebUiWebDocument,
} from "./open-webui-types";
import {
openWebUiChatListItemSchema,
openWebUiChatSchema,
openWebUiCollectionQueryResponseSchema,
openWebUiCompletionChunkSchema,
openWebUiKnowledgeListSchema,
openWebUiModelsResponseSchema,
openWebUiProcessWebResponseSchema,
} from "./open-webui-types";
const logger = createLogger({ module: "open-webui-integration" });
@@ -49,6 +54,65 @@ export class OpenWebUiIntegration extends Integration {
return openWebUiModelsResponseSchema.parse(await response.json()).data;
}
/**
* List the knowledge bases (RAG collections) the user can ground chats on.
*/
public async getKnowledgeAsync(): Promise<OpenWebUiKnowledge[]> {
const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/knowledge/"), {
headers: this.getAuthHeaders(),
});
if (!response.ok) {
throw new ResponseError(response);
}
const parsed = openWebUiKnowledgeListSchema.parse(await response.json());
return Array.isArray(parsed) ? parsed : parsed.items;
}
/**
* Ingest a web page into a temporary vector collection and return a handle to
* retrieve from it. Open WebUI does not crawl URLs at completion time, so we
* embed the page here and inject retrieved chunks ourselves at send time.
*/
public async processWebAsync(url: string): Promise<OpenWebUiWebDocument> {
const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/retrieval/process/web"), {
method: "POST",
headers: { ...this.getAuthHeaders(), "Content-Type": "application/json" },
body: JSON.stringify({ url, collection_name: "" }),
});
if (!response.ok) {
throw new ResponseError(response);
}
const parsed = openWebUiProcessWebResponseSchema.parse(await response.json());
return { collectionName: parsed.collection_name, title: parsed.file?.meta?.name ?? url };
}
/**
* Retrieve the most relevant chunks for a query across the given collections
* (web pages and/or knowledge bases). Returns the chunk texts, best-effort.
*/
public async queryCollectionsAsync(collectionNames: string[], query: string, k = 4): Promise<string[]> {
if (collectionNames.length === 0 || query.trim() === "") {
return [];
}
const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/retrieval/query/collection"), {
method: "POST",
headers: { ...this.getAuthHeaders(), "Content-Type": "application/json" },
body: JSON.stringify({ collection_names: collectionNames, query, k }),
});
if (!response.ok) {
throw new ResponseError(response);
}
const parsed = openWebUiCollectionQueryResponseSchema.parse(await response.json());
return (parsed.documents ?? []).flat();
}
/**
* List the user's chats (native Open WebUI history).
*/
@@ -119,7 +183,7 @@ export class OpenWebUiIntegration extends Integration {
* content delta. Resolves when the stream completes ([DONE]) or is aborted.
*/
public async streamChatCompletionAsync(
params: { model: string; messages: OpenWebUiChatMessage[] },
params: { model: string; messages: OpenWebUiCompletionMessage[] },
onDelta: (delta: string) => void,
signal: AbortSignal,
): Promise<void> {
@@ -10,6 +10,17 @@ export const openWebUiModelSchema = z.object({
name: z.string().nullish(),
object: z.string().nullish(),
owned_by: z.string().nullish(),
// Open WebUI exposes per-model capabilities here when configured; `vision`
// tells us whether the model can accept image attachments.
info: z
.object({
meta: z
.object({
capabilities: z.object({ vision: z.boolean().nullish() }).nullish(),
})
.nullish(),
})
.nullish(),
});
export const openWebUiModelsResponseSchema = z.object({
@@ -44,6 +55,57 @@ export const openWebUiChatMessageSchema = z.object({
export type OpenWebUiChatMessage = z.infer<typeof openWebUiChatMessageSchema>;
// Multimodal content part (OpenAI vision format). Used only for outbound
// completion requests so users can attach images to a message.
export const openWebUiContentPartSchema = z.discriminatedUnion("type", [
z.object({ type: z.literal("text"), text: z.string() }),
z.object({ type: z.literal("image_url"), image_url: z.object({ url: z.string() }) }),
]);
// A completion message can carry plain text or multimodal parts (text + images).
export const openWebUiCompletionMessageSchema = z.object({
role: z.enum(["system", "user", "assistant"]),
content: z.union([z.string(), z.array(openWebUiContentPartSchema)]),
});
export type OpenWebUiCompletionMessage = z.infer<typeof openWebUiCompletionMessageSchema>;
// GET /api/v1/knowledge/ — knowledge bases (RAG collections owned by the user).
// Newer Open WebUI versions paginate the list as `{ items, total }`; older ones
// return a bare array. Accept both and normalise in the integration.
export const openWebUiKnowledgeSchema = z.object({
id: z.string(),
name: z.string().nullish(),
description: z.string().nullish(),
});
export type OpenWebUiKnowledge = z.infer<typeof openWebUiKnowledgeSchema>;
export const openWebUiKnowledgeListSchema = z.union([
z.array(openWebUiKnowledgeSchema),
z.object({ items: z.array(openWebUiKnowledgeSchema) }),
]);
// POST /api/v1/retrieval/process/web — ingests a URL into a temporary vector
// collection. We retrieve from that collection at send time and inject the
// results as context (Open WebUI does not fetch URLs at completion time).
export const openWebUiProcessWebResponseSchema = z.object({
collection_name: z.string(),
filename: z.string().nullish(),
file: z.object({ meta: z.object({ name: z.string().nullish() }).nullish() }).nullish(),
});
// POST /api/v1/retrieval/query/collection — top-k chunks per collection.
export const openWebUiCollectionQueryResponseSchema = z.object({
documents: z.array(z.array(z.string())).nullish(),
});
// A web page ingested for grounding: the collection to retrieve from + a label.
export interface OpenWebUiWebDocument {
collectionName: string;
title: string;
}
// GET /api/v1/chats/ — list of the user's chats (native Open WebUI shape).
export const openWebUiChatListItemSchema = z.object({
id: z.string(),
+19 -1
View File
@@ -1,7 +1,12 @@
import dayjs from "dayjs";
import { createIntegrationAsync } from "@homarr/integrations";
import type { OpenWebUiChat, OpenWebUiChatListItem, OpenWebUiModel } from "@homarr/integrations/types";
import type {
OpenWebUiChat,
OpenWebUiChatListItem,
OpenWebUiKnowledge,
OpenWebUiModel,
} from "@homarr/integrations/types";
import { createCachedIntegrationRequestHandler } from "./lib/cached-integration-request-handler";
@@ -18,6 +23,19 @@ export const openWebUiModelsRequestHandler = createCachedIntegrationRequestHandl
queryKey: "openWebUiModels",
});
export const openWebUiKnowledgeRequestHandler = createCachedIntegrationRequestHandler<
OpenWebUiKnowledge[],
"openWebUi",
Record<string, never>
>({
async requestAsync(integration) {
const integrationInstance = await createIntegrationAsync(integration);
return await integrationInstance.getKnowledgeAsync();
},
cacheDuration: dayjs.duration(5, "minutes"),
queryKey: "openWebUiKnowledge",
});
export const openWebUiChatsRequestHandler = createCachedIntegrationRequestHandler<
OpenWebUiChatListItem[],
"openWebUi",
+9
View File
@@ -3562,6 +3562,15 @@
"back": "Back",
"send": "Send",
"stop": "Stop",
"retry": "Retry",
"visionWarning": "This model may not support images",
"attach": "Attach",
"attachImage": "Attach image",
"attachKnowledge": "Knowledge",
"attachWebpage": "Webpage",
"addUrl": "Add URL",
"removeAttachment": "Remove attachment",
"webError": "Could not load that web page",
"error": {
"internalServerError": "Failed to reach Open WebUI"
}
@@ -38,6 +38,20 @@
max-width: 85%;
}
.thumb {
position: relative;
line-height: 0;
}
.thumbRemove {
position: absolute;
top: -6px;
right: -6px;
background-color: var(--mantine-color-body);
border-radius: var(--mantine-radius-xl);
box-shadow: var(--mantine-shadow-xs);
}
.historyOverlay {
position: absolute;
inset: 0;
+367 -24
View File
@@ -4,26 +4,39 @@ import { useEffect, useMemo, useRef, useState } from "react";
import {
ActionIcon,
Avatar,
Badge,
Box,
Button,
Checkbox,
CloseButton,
Group,
Image,
Loader,
NavLink,
Paper,
Popover,
ScrollArea,
Select,
Stack,
Text,
Textarea,
TextInput,
Tooltip,
} from "@mantine/core";
import {
IconAlertTriangle,
IconArrowLeft,
IconArrowUp,
IconDatabase,
IconHistory,
IconPaperclip,
IconPhoto,
IconPlayerStopFilled,
IconPlus,
IconRefresh,
IconRobot,
IconUser,
IconWorld,
} from "@tabler/icons-react";
import { clientApi } from "@homarr/api/client";
@@ -35,8 +48,28 @@ import classes from "./chat.module.css";
interface ChatMessage {
role: "system" | "user" | "assistant";
content: string;
images?: string[];
}
type CompletionPart = { type: "text"; text: string } | { type: "image_url"; image_url: { url: string } };
type CompletionMessage = { role: ChatMessage["role"]; content: string | CompletionPart[] };
interface ImageAttachment {
name: string;
dataUrl: string;
}
interface WebAttachment {
url: string;
collectionName: string;
title: string;
}
// Heuristic fallback for detecting vision-capable models when Open WebUI does
// not declare the capability explicitly.
const VISION_MODEL_PATTERN =
/vision|llava|bakllava|moondream|minicpm-?v|qwen2?-?vl|pixtral|cogvlm|internvl|gemma3|smolvlm|granite[\w-]*vision/i;
export function Chat({ options, integrationIds, isEditMode }: WidgetComponentProps<"openWebUi">) {
const t = useI18n();
const integrationId = integrationIds.at(0);
@@ -46,16 +79,26 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
const [input, setInput] = useState("");
const [streamingText, setStreamingText] = useState("");
const [isStreaming, setIsStreaming] = useState(false);
const [pendingMessages, setPendingMessages] = useState<ChatMessage[]>([]);
const [pendingMessages, setPendingMessages] = useState<CompletionMessage[]>([]);
const [pendingCollections, setPendingCollections] = useState<string[]>([]);
const [activeChatId, setActiveChatId] = useState<string | null>(null);
const [selectedChatId, setSelectedChatId] = useState<string | null>(null);
const [error, setError] = useState<string | null>(null);
const [historyOpened, setHistoryOpened] = useState(false);
// Attachments grounding the conversation. Images are per-message; knowledge
// collections and web pages are sticky context kept until removed.
const [attachments, setAttachments] = useState<ImageAttachment[]>([]);
const [knowledgeIds, setKnowledgeIds] = useState<string[]>([]);
const [webItems, setWebItems] = useState<WebAttachment[]>([]);
const [webInput, setWebInput] = useState("");
const [attachOpened, setAttachOpened] = useState(false);
const messagesRef = useRef<ChatMessage[]>([]);
const streamingTextRef = useRef("");
const viewportRef = useRef<HTMLDivElement>(null);
const composerRef = useRef<HTMLDivElement>(null);
const fileInputRef = useRef<HTMLInputElement>(null);
const [composerHeight, setComposerHeight] = useState(72);
const utils = clientApi.useUtils();
@@ -65,6 +108,11 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
{ enabled: Boolean(integrationId), refetchOnWindowFocus: false, retry: false },
);
const { data: knowledge = [] } = clientApi.widget.openWebUi.getKnowledge.useQuery(
{ integrationId: integrationId ?? "" },
{ enabled: Boolean(integrationId), refetchOnWindowFocus: false, retry: false },
);
const { data: chats = [] } = clientApi.widget.openWebUi.getChats.useQuery(
{ integrationId: integrationId ?? "" },
{ enabled: Boolean(integrationId) && options.showHistory, refetchOnWindowFocus: false, retry: false },
@@ -77,6 +125,7 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
const createChat = clientApi.widget.openWebUi.createChat.useMutation();
const updateChat = clientApi.widget.openWebUi.updateChat.useMutation();
const processWeb = clientApi.widget.openWebUi.processWeb.useMutation();
// Default to the first available model once the list loads.
useEffect(() => {
@@ -113,7 +162,12 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
}, [messages, streamingText, composerHeight]);
clientApi.widget.openWebUi.sendMessage.useSubscription(
{ integrationId: integrationId ?? "", model: model ?? "", messages: pendingMessages },
{
integrationId: integrationId ?? "",
model: model ?? "",
messages: pendingMessages,
collections: pendingCollections.length > 0 ? pendingCollections : undefined,
},
{
enabled: isStreaming && pendingMessages.length > 0,
onData: (event) => {
@@ -146,12 +200,18 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
setStreamingText("");
setIsStreaming(false);
setPendingMessages([]);
setPendingCollections([]);
};
const persistChat = async (finalMessages: ChatMessage[]) => {
if (!integrationId || !model) return;
// Native history stores plain text; image data URLs are dropped on persist.
const title = finalMessages.find((message) => message.role === "user")?.content.slice(0, 80) ?? "New chat";
const payload = { title, models: [model], messages: finalMessages };
const payload = {
title,
models: [model],
messages: finalMessages.map(({ role, content }) => ({ role, content })),
};
try {
if (activeChatId) {
await updateChat.mutateAsync({ integrationId, chatId: activeChatId, chat: payload });
@@ -167,26 +227,56 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
}
};
const handleSend = () => {
const trimmed = input.trim();
if (!trimmed || !model || !integrationId || isStreaming) return;
// Kick off a completion for the given conversation (ending with a user message).
// Shared by sending a new message and retrying after a failure.
const startCompletion = (baseMessages: ChatMessage[]) => {
if (!model || !integrationId) return;
setError(null);
const userMessage: ChatMessage = { role: "user", content: trimmed };
const systemPrompt = options.systemPrompt.trim();
const payloadMessages: ChatMessage[] = systemPrompt
? [{ role: "system", content: systemPrompt }, ...baseMessages]
: baseMessages;
// Knowledge ids double as their collection name; web pages bring their own.
const collections = [...knowledgeIds, ...webItems.map((item) => item.collectionName)];
streamingTextRef.current = "";
setStreamingText("");
setPendingMessages(toCompletionMessages(payloadMessages));
setPendingCollections(collections);
setIsStreaming(true);
};
const handleSend = () => {
const trimmed = input.trim();
if ((!trimmed && attachments.length === 0) || !model || !integrationId || isStreaming) return;
const userMessage: ChatMessage = {
role: "user",
content: trimmed,
images: attachments.length > 0 ? attachments.map((attachment) => attachment.dataUrl) : undefined,
};
const nextMessages = [...messagesRef.current, userMessage];
setMessages(nextMessages);
messagesRef.current = nextMessages;
setInput("");
setAttachments([]);
const systemPrompt = options.systemPrompt.trim();
const payloadMessages: ChatMessage[] = systemPrompt
? [{ role: "system", content: systemPrompt }, ...nextMessages]
: nextMessages;
startCompletion(nextMessages);
};
streamingTextRef.current = "";
setStreamingText("");
setPendingMessages(payloadMessages);
setIsStreaming(true);
// Retry the last turn after a failure: drop any partial assistant reply and
// re-run the completion from the most recent user message.
const handleRetry = () => {
if (isStreaming) return;
const lastUserIndex = messagesRef.current.findLastIndex((message) => message.role === "user");
if (lastUserIndex === -1) return;
const truncated = messagesRef.current.slice(0, lastUserIndex + 1);
setMessages(truncated);
messagesRef.current = truncated;
startCompletion(truncated);
};
const handleNewChat = () => {
@@ -199,10 +289,56 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
setStreamingText("");
setIsStreaming(false);
setPendingMessages([]);
setPendingCollections([]);
setAttachments([]);
setKnowledgeIds([]);
setWebItems([]);
setHistoryOpened(false);
};
const handleFilesSelected = async (fileList: FileList | null) => {
if (!fileList) return;
const images = await Promise.all(
Array.from(fileList)
.filter((file) => file.type.startsWith("image/"))
.map(async (file) => ({ name: file.name, dataUrl: await readFileAsDataUrl(file) })),
);
setAttachments((previous) => [...previous, ...images]);
};
const addWebUrl = async () => {
if (!integrationId) return;
const raw = webInput.trim();
if (!raw) return;
const url = /^https?:\/\//i.test(raw) ? raw : `https://${raw}`;
setWebInput("");
setError(null);
try {
const document = await processWeb.mutateAsync({ integrationId, url });
setWebItems((previous) =>
previous.some((item) => item.collectionName === document.collectionName)
? previous
: [...previous, { url, collectionName: document.collectionName, title: document.title }],
);
} catch {
setError(t("widget.openWebUi.webError"));
}
};
const modelData = useMemo(() => models.map((m) => ({ value: m.id, label: m.name ?? m.id })), [models]);
const knowledgeName = (id: string) => knowledge.find((item) => item.id === id)?.name ?? id;
const hasAttachments = attachments.length > 0 || knowledgeIds.length > 0 || webItems.length > 0;
// Whether the selected model can actually see attached images. Prefer Open
// WebUI's declared capability; fall back to a name heuristic when it's absent.
const selectedModel = useMemo(() => models.find((item) => item.id === model), [models, model]);
const modelSupportsVision = useMemo(() => {
if (!selectedModel) return true;
const capability = selectedModel.info?.meta?.capabilities?.vision;
if (typeof capability === "boolean") return capability;
return VISION_MODEL_PATTERN.test(selectedModel.id) || VISION_MODEL_PATTERN.test(selectedModel.name ?? "");
}, [selectedModel]);
const showVisionWarning = attachments.length > 0 && !modelSupportsVision;
return (
<Box className={classes.root} style={isEditMode ? { pointerEvents: "none" } : undefined}>
@@ -215,18 +351,92 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
</Stack>
) : null}
{messages.map((message, index) => (
<MessageBubble key={index} from={message.role} content={message.content} />
<MessageBubble key={index} from={message.role} content={message.content} images={message.images} />
))}
{isStreaming ? <MessageBubble from="assistant" content={streamingText || "…"} /> : null}
{error ? (
<Text size="xs" c="red.6">
{error}
</Text>
<Group gap="xs" justify="center" wrap="nowrap">
<Text size="xs" c="red.6">
{error}
</Text>
<Button
size="compact-xs"
variant="light"
color="red"
leftSection={<IconRefresh size={14} />}
onClick={handleRetry}
>
{t("widget.openWebUi.retry")}
</Button>
</Group>
) : null}
</Stack>
</ScrollArea>
<Paper ref={composerRef} className={classes.composer} withBorder radius="md" p="xs">
{hasAttachments ? (
<Group gap={6} mb={6}>
{attachments.map((attachment, index) => (
<Box key={`${attachment.name}-${index}`} className={classes.thumb}>
<Image src={attachment.dataUrl} alt={attachment.name} w={48} h={48} radius="sm" fit="cover" />
<CloseButton
size="xs"
className={classes.thumbRemove}
onClick={() => setAttachments((previous) => previous.filter((_, i) => i !== index))}
aria-label={t("widget.openWebUi.removeAttachment")}
/>
</Box>
))}
{knowledgeIds.map((id) => (
<Badge
key={id}
variant="light"
color="grape"
leftSection={<IconDatabase size={12} />}
rightSection={
<CloseButton
size="xs"
onClick={() => setKnowledgeIds((previous) => previous.filter((value) => value !== id))}
aria-label={t("widget.openWebUi.removeAttachment")}
/>
}
>
{knowledgeName(id)}
</Badge>
))}
{webItems.map((item) => (
<Badge
key={item.collectionName}
variant="light"
color="blue"
leftSection={<IconWorld size={12} />}
rightSection={
<CloseButton
size="xs"
onClick={() =>
setWebItems((previous) =>
previous.filter((value) => value.collectionName !== item.collectionName),
)
}
aria-label={t("widget.openWebUi.removeAttachment")}
/>
}
>
{hostnameOf(item.url)}
</Badge>
))}
</Group>
) : null}
{showVisionWarning ? (
<Group gap={4} mb={6} wrap="nowrap">
<IconAlertTriangle size={14} color="var(--mantine-color-yellow-6)" />
<Text size="xs" c="yellow.7">
{t("widget.openWebUi.visionWarning")}
</Text>
</Group>
) : null}
<Textarea
variant="unstyled"
size="sm"
@@ -255,6 +465,86 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
<IconPlus size={18} />
</ActionIcon>
</Tooltip>
<Popover
opened={attachOpened}
onChange={setAttachOpened}
position="top-start"
withinPortal={false}
width={260}
shadow="md"
>
<Popover.Target>
<Tooltip label={t("widget.openWebUi.attach")} withinPortal={false}>
<ActionIcon
variant="subtle"
color="gray"
onClick={() => setAttachOpened((value) => !value)}
aria-label={t("widget.openWebUi.attach")}
>
<IconPaperclip size={18} />
</ActionIcon>
</Tooltip>
</Popover.Target>
<Popover.Dropdown>
<Stack gap="sm">
<Button
variant="light"
color="gray"
size="xs"
leftSection={<IconPhoto size={16} />}
justify="flex-start"
onClick={() => fileInputRef.current?.click()}
>
{t("widget.openWebUi.attachImage")}
</Button>
{knowledge.length > 0 ? (
<Stack gap={4}>
<Text size="xs" fw={600} c="dimmed">
{t("widget.openWebUi.attachKnowledge")}
</Text>
<Checkbox.Group value={knowledgeIds} onChange={setKnowledgeIds}>
<Stack gap={4}>
{knowledge.map((item) => (
<Checkbox key={item.id} value={item.id} label={item.name ?? item.id} size="xs" />
))}
</Stack>
</Checkbox.Group>
</Stack>
) : null}
<Stack gap={4}>
<Text size="xs" fw={600} c="dimmed">
{t("widget.openWebUi.attachWebpage")}
</Text>
<Group gap={4} wrap="nowrap">
<TextInput
size="xs"
flex={1}
placeholder="https://…"
value={webInput}
onChange={(event) => setWebInput(event.currentTarget.value)}
onKeyDown={(event) => {
if (event.key === "Enter") {
event.preventDefault();
void addWebUrl();
}
}}
/>
<ActionIcon
variant="light"
color="gray"
loading={processWeb.isPending}
onClick={() => void addWebUrl()}
aria-label={t("widget.openWebUi.addUrl")}
>
<IconPlus size={16} />
</ActionIcon>
</Group>
</Stack>
</Stack>
</Popover.Dropdown>
</Popover>
{options.showHistory ? (
<Tooltip label={t("widget.openWebUi.history")} withinPortal={false}>
<ActionIcon
@@ -301,7 +591,7 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
radius="xl"
variant="filled"
onClick={handleSend}
disabled={!input.trim() || !model}
disabled={(!input.trim() && attachments.length === 0) || !model}
aria-label={t("widget.openWebUi.send")}
>
<IconArrowUp size={18} />
@@ -310,6 +600,19 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
)}
</Group>
</Group>
<input
ref={fileInputRef}
type="file"
accept="image/*"
multiple
hidden
aria-label={t("widget.openWebUi.attachImage")}
onChange={(event) => {
void handleFilesSelected(event.currentTarget.files);
event.currentTarget.value = "";
}}
/>
</Paper>
{options.showHistory && historyOpened ? (
@@ -353,6 +656,37 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
);
}
const readFileAsDataUrl = (file: File): Promise<string> =>
new Promise((resolve, reject) => {
const reader = new FileReader();
reader.addEventListener("load", () => resolve(reader.result as string));
reader.addEventListener("error", () => reject(reader.error ?? new Error("Failed to read file")));
reader.readAsDataURL(file);
});
const hostnameOf = (url: string): string => {
try {
return new URL(url).hostname || url;
} catch {
return url;
}
};
// Convert displayed messages to the OpenAI completion format, inlining any
// attached images on the message that carries them (vision content parts).
const toCompletionMessages = (messages: ChatMessage[]): CompletionMessage[] =>
messages.map((message) =>
message.images && message.images.length > 0
? {
role: message.role,
content: [
...(message.content ? [{ type: "text" as const, text: message.content }] : []),
...message.images.map((url) => ({ type: "image_url" as const, image_url: { url } })),
],
}
: { role: message.role, content: message.content },
);
type RawMessage = { role?: string | null; content?: string | null };
type HistoryMessage = RawMessage & { parentId?: string | null };
interface RawChat {
@@ -393,7 +727,7 @@ const extractChatMessages = (chat: RawChat | null | undefined): ChatMessage[] =>
.filter((message): message is ChatMessage => message !== null);
};
function MessageBubble({ from, content }: { from: ChatMessage["role"]; content: string }) {
function MessageBubble({ from, content, images }: { from: ChatMessage["role"]; content: string; images?: string[] }) {
const isUser = from === "user";
const avatar = (
<Avatar size={22} radius="xl" variant="filled" color={isUser ? "gray" : "blue"} className={classes.avatar}>
@@ -410,9 +744,18 @@ function MessageBubble({ from, content }: { from: ChatMessage["role"]; content:
p="xs"
radius="md"
>
<Text size="sm" style={{ whiteSpace: "pre-wrap", wordBreak: "break-word" }}>
{content}
</Text>
{images && images.length > 0 ? (
<Group gap={4} mb={content ? 6 : 0}>
{images.map((src, index) => (
<Image key={index} src={src} alt="" w={120} radius="sm" fit="contain" />
))}
</Group>
) : null}
{content ? (
<Text size="sm" style={{ whiteSpace: "pre-wrap", wordBreak: "break-word" }}>
{content}
</Text>
) : null}
</Paper>
{isUser ? avatar : null}
</Group>