feat(open-webui): add image, knowledge and webpage attachments with retry
Extend the AI Chat widget with three ways to ground a conversation:
- Images: attach images to a message, sent inline as OpenAI multimodal
content parts for vision-capable models. A hint warns when the selected
model likely has no vision support.
- Knowledge bases: pick RAG collections exposed by /api/v1/knowledge/.
- Web pages: ingest a URL via /api/v1/retrieval/process/web.
Open WebUI does not run RAG from the chat-completions `files` parameter,
so attached collections (web pages and knowledge bases) are grounded by
retrieving the most relevant chunks at send time
(/api/v1/retrieval/query/collection) and injecting them as system context.
Also add a Retry action to re-run the last turn after a model failure, and
handle the paginated `{ items, total }` shape of the knowledge endpoint on
newer Open WebUI versions.
This commit is contained in:
Lok3sh-Kumar committed
1 parent
005c2e4cd1
commit
f61e7539d7
8 files changed
+624
-30
No files matched your search
@@ -13,7 +13,7 @@ import { openWebUiIntegration } from "@site/docs/integrations/open-webui";
|
||||
|
||||
<WidgetHeader widget={openWebUiWidget} categories={["AI", "Chat"]} />
|
||||
|
||||
The AI Chat widget lets you chat with AI models served by Open WebUI directly from your dashboard. Responses stream in token-by-token, you can pick any model your Open WebUI instance exposes (Ollama, OpenAI, and other configured providers), and you can browse and reopen your Open WebUI chat history.
|
||||
The AI Chat widget lets you chat with AI models served by Open WebUI directly from your dashboard. Responses stream in token-by-token, you can pick any model your Open WebUI instance exposes (Ollama, OpenAI, and other configured providers), and you can browse and reopen your Open WebUI chat history. You can also ground a conversation by attaching images, knowledge bases, or web pages.
|
||||
|
||||
### Supported Integrations
|
||||
|
||||
@@ -40,8 +40,13 @@ The AI Chat widget lets you chat with AI models served by Open WebUI directly fr
|
||||
| **Model picker** | Lists every model exposed by Open WebUI's `/api/models` endpoint, selectable from the chat input. The first model is selected by default. |
|
||||
| **Streaming responses** | Assistant replies stream in live as they are generated. |
|
||||
| **Chat history** | Browse and reopen previous chats. New conversations are saved back to Open WebUI. |
|
||||
| **Image attachments** | Attach images to a message for vision-capable models (sent inline as multimodal content). |
|
||||
| **Knowledge bases** | Ground replies on any RAG knowledge collection (`/api/v1/knowledge/`) your account owns. |
|
||||
| **Web pages** | Attach a URL and Open WebUI fetches and embeds the page as context for the reply. |
|
||||
|
||||
### Notes
|
||||
|
||||
- The widget uses a single Open WebUI integration.
|
||||
- Conversations are stored in Open WebUI itself, under the account that owns the API key, so they also appear in the Open WebUI app.
|
||||
- Knowledge bases and attached web pages stay active for the whole conversation until you remove them; images apply only to the message you attach them to.
|
||||
- Conversation text is saved to Open WebUI history, but attached image data is not persisted.
|
||||
@@ -6,6 +6,7 @@ import { createIntegrationAsync } from "@homarr/integrations";
|
||||
import {
|
||||
openWebUiChatRequestHandler,
|
||||
openWebUiChatsRequestHandler,
|
||||
openWebUiKnowledgeRequestHandler,
|
||||
openWebUiModelsRequestHandler,
|
||||
} from "@homarr/request-handler/open-webui";
|
||||
|
||||
@@ -19,6 +20,18 @@ const messageSchema = z.object({
|
||||
content: z.string(),
|
||||
});
|
||||
|
||||
// Outbound completion messages may carry multimodal parts (text + images),
|
||||
// so users can attach images to a message.
|
||||
const contentPartSchema = z.discriminatedUnion("type", [
|
||||
z.object({ type: z.literal("text"), text: z.string() }),
|
||||
z.object({ type: z.literal("image_url"), image_url: z.object({ url: z.string() }) }),
|
||||
]);
|
||||
|
||||
const completionMessageSchema = z.object({
|
||||
role: z.enum(["system", "user", "assistant"]),
|
||||
content: z.union([z.string(), z.array(contentPartSchema)]),
|
||||
});
|
||||
|
||||
const chatPayloadSchema = z.object({
|
||||
title: z.string(),
|
||||
models: z.array(z.string()),
|
||||
@@ -27,6 +40,47 @@ const chatPayloadSchema = z.object({
|
||||
|
||||
type StreamEvent = { type: "delta"; content: string } | { type: "done" } | { type: "error"; message: string };
|
||||
|
||||
type CompletionMessage = z.infer<typeof completionMessageSchema>;
|
||||
|
||||
// Extract the plain text of a message, flattening multimodal content parts.
|
||||
const messageText = (message: CompletionMessage): string =>
|
||||
typeof message.content === "string"
|
||||
? message.content
|
||||
: message.content
|
||||
.filter((part): part is { type: "text"; text: string } => part.type === "text")
|
||||
.map((part) => part.text)
|
||||
.join(" ");
|
||||
|
||||
// Retrieve relevant chunks from the attached collections and prepend them as a
|
||||
// system context message so the model can ground its reply on them.
|
||||
const groundMessagesAsync = async (
|
||||
client: { queryCollectionsAsync: (names: string[], query: string, k?: number) => Promise<string[]> },
|
||||
messages: CompletionMessage[],
|
||||
collections: string[],
|
||||
): Promise<CompletionMessage[]> => {
|
||||
if (collections.length === 0) return messages;
|
||||
|
||||
const lastUserMessage = messages.findLast((message) => message.role === "user");
|
||||
const query = lastUserMessage ? messageText(lastUserMessage) : "";
|
||||
|
||||
let documents: string[] = [];
|
||||
try {
|
||||
documents = await client.queryCollectionsAsync(collections, query, 4);
|
||||
} catch (error) {
|
||||
logger.warn("Open WebUI context retrieval failed; continuing without it", { error });
|
||||
}
|
||||
if (documents.length === 0) return messages;
|
||||
|
||||
const context = documents.join("\n\n");
|
||||
const contextMessage: CompletionMessage = {
|
||||
role: "system",
|
||||
content:
|
||||
"Use the following context from the user's attached sources (web pages and knowledge bases) " +
|
||||
`to answer. If the answer is not in the context, say so.\n\n${context}`,
|
||||
};
|
||||
return [contextMessage, ...messages];
|
||||
};
|
||||
|
||||
export const openWebUiRouter = createTRPCRouter({
|
||||
getModels: publicProcedure.concat(createOneIntegrationMiddleware("query", "openWebUi")).query(async ({ ctx }) => {
|
||||
const handler = openWebUiModelsRequestHandler.handler(ctx.integration, {});
|
||||
@@ -40,6 +94,22 @@ export const openWebUiRouter = createTRPCRouter({
|
||||
return data;
|
||||
}),
|
||||
|
||||
getKnowledge: publicProcedure.concat(createOneIntegrationMiddleware("query", "openWebUi")).query(async ({ ctx }) => {
|
||||
const handler = openWebUiKnowledgeRequestHandler.handler(ctx.integration, {});
|
||||
const { data } = await handler.getCachedOrUpdatedDataAsync({ forceUpdate: false });
|
||||
return data;
|
||||
}),
|
||||
|
||||
// Ingest a web page so its content can ground later messages. Returns the
|
||||
// collection to retrieve from plus a display title.
|
||||
processWeb: protectedProcedure
|
||||
.input(z.object({ url: z.string().url() }))
|
||||
.concat(createOneIntegrationMiddleware("interact", "openWebUi"))
|
||||
.mutation(async ({ ctx, input }) => {
|
||||
const client = await createIntegrationAsync(ctx.integration);
|
||||
return await client.processWebAsync(input.url);
|
||||
}),
|
||||
|
||||
getChat: publicProcedure
|
||||
.input(z.object({ chatId: z.string() }))
|
||||
.concat(createOneIntegrationMiddleware("query", "openWebUi"))
|
||||
@@ -78,7 +148,15 @@ export const openWebUiRouter = createTRPCRouter({
|
||||
// tokens can be emitted as they arrive. Uses "query" access (same level as
|
||||
// viewing the widget) to match the other integration subscriptions in the app.
|
||||
sendMessage: publicProcedure
|
||||
.input(z.object({ model: z.string(), messages: z.array(messageSchema) }))
|
||||
.input(
|
||||
z.object({
|
||||
model: z.string(),
|
||||
messages: z.array(completionMessageSchema),
|
||||
// Collections to ground the reply on (web pages + knowledge bases). We
|
||||
// retrieve relevant chunks and inject them as context before streaming.
|
||||
collections: z.array(z.string()).optional(),
|
||||
}),
|
||||
)
|
||||
.concat(createOneIntegrationMiddleware("query", "openWebUi"))
|
||||
.subscription(({ ctx, input }) => {
|
||||
return observable<StreamEvent>((emit) => {
|
||||
@@ -87,8 +165,9 @@ export const openWebUiRouter = createTRPCRouter({
|
||||
void (async () => {
|
||||
try {
|
||||
const client = await createIntegrationAsync(ctx.integration);
|
||||
const messages = await groundMessagesAsync(client, input.messages, input.collections ?? []);
|
||||
await client.streamChatCompletionAsync(
|
||||
{ model: input.model, messages: input.messages },
|
||||
{ model: input.model, messages },
|
||||
(delta) => emit.next({ type: "delta", content: delta }),
|
||||
controller.signal,
|
||||
);
|
||||
|
||||
@@ -8,15 +8,20 @@ import type { TestingResult } from "../base/test-connection/test-connection-serv
|
||||
import type {
|
||||
OpenWebUiChat,
|
||||
OpenWebUiChatListItem,
|
||||
OpenWebUiChatMessage,
|
||||
OpenWebUiChatPayload,
|
||||
OpenWebUiCompletionMessage,
|
||||
OpenWebUiKnowledge,
|
||||
OpenWebUiModel,
|
||||
OpenWebUiWebDocument,
|
||||
} from "./open-webui-types";
|
||||
import {
|
||||
openWebUiChatListItemSchema,
|
||||
openWebUiChatSchema,
|
||||
openWebUiCollectionQueryResponseSchema,
|
||||
openWebUiCompletionChunkSchema,
|
||||
openWebUiKnowledgeListSchema,
|
||||
openWebUiModelsResponseSchema,
|
||||
openWebUiProcessWebResponseSchema,
|
||||
} from "./open-webui-types";
|
||||
|
||||
const logger = createLogger({ module: "open-webui-integration" });
|
||||
@@ -49,6 +54,65 @@ export class OpenWebUiIntegration extends Integration {
|
||||
return openWebUiModelsResponseSchema.parse(await response.json()).data;
|
||||
}
|
||||
|
||||
/**
|
||||
* List the knowledge bases (RAG collections) the user can ground chats on.
|
||||
*/
|
||||
public async getKnowledgeAsync(): Promise<OpenWebUiKnowledge[]> {
|
||||
const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/knowledge/"), {
|
||||
headers: this.getAuthHeaders(),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new ResponseError(response);
|
||||
}
|
||||
|
||||
const parsed = openWebUiKnowledgeListSchema.parse(await response.json());
|
||||
return Array.isArray(parsed) ? parsed : parsed.items;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ingest a web page into a temporary vector collection and return a handle to
|
||||
* retrieve from it. Open WebUI does not crawl URLs at completion time, so we
|
||||
* embed the page here and inject retrieved chunks ourselves at send time.
|
||||
*/
|
||||
public async processWebAsync(url: string): Promise<OpenWebUiWebDocument> {
|
||||
const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/retrieval/process/web"), {
|
||||
method: "POST",
|
||||
headers: { ...this.getAuthHeaders(), "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ url, collection_name: "" }),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new ResponseError(response);
|
||||
}
|
||||
|
||||
const parsed = openWebUiProcessWebResponseSchema.parse(await response.json());
|
||||
return { collectionName: parsed.collection_name, title: parsed.file?.meta?.name ?? url };
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve the most relevant chunks for a query across the given collections
|
||||
* (web pages and/or knowledge bases). Returns the chunk texts, best-effort.
|
||||
*/
|
||||
public async queryCollectionsAsync(collectionNames: string[], query: string, k = 4): Promise<string[]> {
|
||||
if (collectionNames.length === 0 || query.trim() === "") {
|
||||
return [];
|
||||
}
|
||||
|
||||
const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/retrieval/query/collection"), {
|
||||
method: "POST",
|
||||
headers: { ...this.getAuthHeaders(), "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ collection_names: collectionNames, query, k }),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new ResponseError(response);
|
||||
}
|
||||
|
||||
const parsed = openWebUiCollectionQueryResponseSchema.parse(await response.json());
|
||||
return (parsed.documents ?? []).flat();
|
||||
}
|
||||
|
||||
/**
|
||||
* List the user's chats (native Open WebUI history).
|
||||
*/
|
||||
@@ -119,7 +183,7 @@ export class OpenWebUiIntegration extends Integration {
|
||||
* content delta. Resolves when the stream completes ([DONE]) or is aborted.
|
||||
*/
|
||||
public async streamChatCompletionAsync(
|
||||
params: { model: string; messages: OpenWebUiChatMessage[] },
|
||||
params: { model: string; messages: OpenWebUiCompletionMessage[] },
|
||||
onDelta: (delta: string) => void,
|
||||
signal: AbortSignal,
|
||||
): Promise<void> {
|
||||
|
||||
@@ -10,6 +10,17 @@ export const openWebUiModelSchema = z.object({
|
||||
name: z.string().nullish(),
|
||||
object: z.string().nullish(),
|
||||
owned_by: z.string().nullish(),
|
||||
// Open WebUI exposes per-model capabilities here when configured; `vision`
|
||||
// tells us whether the model can accept image attachments.
|
||||
info: z
|
||||
.object({
|
||||
meta: z
|
||||
.object({
|
||||
capabilities: z.object({ vision: z.boolean().nullish() }).nullish(),
|
||||
})
|
||||
.nullish(),
|
||||
})
|
||||
.nullish(),
|
||||
});
|
||||
|
||||
export const openWebUiModelsResponseSchema = z.object({
|
||||
@@ -44,6 +55,57 @@ export const openWebUiChatMessageSchema = z.object({
|
||||
|
||||
export type OpenWebUiChatMessage = z.infer<typeof openWebUiChatMessageSchema>;
|
||||
|
||||
// Multimodal content part (OpenAI vision format). Used only for outbound
|
||||
// completion requests so users can attach images to a message.
|
||||
export const openWebUiContentPartSchema = z.discriminatedUnion("type", [
|
||||
z.object({ type: z.literal("text"), text: z.string() }),
|
||||
z.object({ type: z.literal("image_url"), image_url: z.object({ url: z.string() }) }),
|
||||
]);
|
||||
|
||||
// A completion message can carry plain text or multimodal parts (text + images).
|
||||
export const openWebUiCompletionMessageSchema = z.object({
|
||||
role: z.enum(["system", "user", "assistant"]),
|
||||
content: z.union([z.string(), z.array(openWebUiContentPartSchema)]),
|
||||
});
|
||||
|
||||
export type OpenWebUiCompletionMessage = z.infer<typeof openWebUiCompletionMessageSchema>;
|
||||
|
||||
// GET /api/v1/knowledge/ — knowledge bases (RAG collections owned by the user).
|
||||
// Newer Open WebUI versions paginate the list as `{ items, total }`; older ones
|
||||
// return a bare array. Accept both and normalise in the integration.
|
||||
export const openWebUiKnowledgeSchema = z.object({
|
||||
id: z.string(),
|
||||
name: z.string().nullish(),
|
||||
description: z.string().nullish(),
|
||||
});
|
||||
|
||||
export type OpenWebUiKnowledge = z.infer<typeof openWebUiKnowledgeSchema>;
|
||||
|
||||
export const openWebUiKnowledgeListSchema = z.union([
|
||||
z.array(openWebUiKnowledgeSchema),
|
||||
z.object({ items: z.array(openWebUiKnowledgeSchema) }),
|
||||
]);
|
||||
|
||||
// POST /api/v1/retrieval/process/web — ingests a URL into a temporary vector
|
||||
// collection. We retrieve from that collection at send time and inject the
|
||||
// results as context (Open WebUI does not fetch URLs at completion time).
|
||||
export const openWebUiProcessWebResponseSchema = z.object({
|
||||
collection_name: z.string(),
|
||||
filename: z.string().nullish(),
|
||||
file: z.object({ meta: z.object({ name: z.string().nullish() }).nullish() }).nullish(),
|
||||
});
|
||||
|
||||
// POST /api/v1/retrieval/query/collection — top-k chunks per collection.
|
||||
export const openWebUiCollectionQueryResponseSchema = z.object({
|
||||
documents: z.array(z.array(z.string())).nullish(),
|
||||
});
|
||||
|
||||
// A web page ingested for grounding: the collection to retrieve from + a label.
|
||||
export interface OpenWebUiWebDocument {
|
||||
collectionName: string;
|
||||
title: string;
|
||||
}
|
||||
|
||||
// GET /api/v1/chats/ — list of the user's chats (native Open WebUI shape).
|
||||
export const openWebUiChatListItemSchema = z.object({
|
||||
id: z.string(),
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
import dayjs from "dayjs";
|
||||
|
||||
import { createIntegrationAsync } from "@homarr/integrations";
|
||||
import type { OpenWebUiChat, OpenWebUiChatListItem, OpenWebUiModel } from "@homarr/integrations/types";
|
||||
import type {
|
||||
OpenWebUiChat,
|
||||
OpenWebUiChatListItem,
|
||||
OpenWebUiKnowledge,
|
||||
OpenWebUiModel,
|
||||
} from "@homarr/integrations/types";
|
||||
|
||||
import { createCachedIntegrationRequestHandler } from "./lib/cached-integration-request-handler";
|
||||
|
||||
@@ -18,6 +23,19 @@ export const openWebUiModelsRequestHandler = createCachedIntegrationRequestHandl
|
||||
queryKey: "openWebUiModels",
|
||||
});
|
||||
|
||||
export const openWebUiKnowledgeRequestHandler = createCachedIntegrationRequestHandler<
|
||||
OpenWebUiKnowledge[],
|
||||
"openWebUi",
|
||||
Record<string, never>
|
||||
>({
|
||||
async requestAsync(integration) {
|
||||
const integrationInstance = await createIntegrationAsync(integration);
|
||||
return await integrationInstance.getKnowledgeAsync();
|
||||
},
|
||||
cacheDuration: dayjs.duration(5, "minutes"),
|
||||
queryKey: "openWebUiKnowledge",
|
||||
});
|
||||
|
||||
export const openWebUiChatsRequestHandler = createCachedIntegrationRequestHandler<
|
||||
OpenWebUiChatListItem[],
|
||||
"openWebUi",
|
||||
|
||||
@@ -3562,6 +3562,15 @@
|
||||
"back": "Back",
|
||||
"send": "Send",
|
||||
"stop": "Stop",
|
||||
"retry": "Retry",
|
||||
"visionWarning": "This model may not support images",
|
||||
"attach": "Attach",
|
||||
"attachImage": "Attach image",
|
||||
"attachKnowledge": "Knowledge",
|
||||
"attachWebpage": "Webpage",
|
||||
"addUrl": "Add URL",
|
||||
"removeAttachment": "Remove attachment",
|
||||
"webError": "Could not load that web page",
|
||||
"error": {
|
||||
"internalServerError": "Failed to reach Open WebUI"
|
||||
}
|
||||
|
||||
@@ -38,6 +38,20 @@
|
||||
max-width: 85%;
|
||||
}
|
||||
|
||||
.thumb {
|
||||
position: relative;
|
||||
line-height: 0;
|
||||
}
|
||||
|
||||
.thumbRemove {
|
||||
position: absolute;
|
||||
top: -6px;
|
||||
right: -6px;
|
||||
background-color: var(--mantine-color-body);
|
||||
border-radius: var(--mantine-radius-xl);
|
||||
box-shadow: var(--mantine-shadow-xs);
|
||||
}
|
||||
|
||||
.historyOverlay {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
|
||||
@@ -4,26 +4,39 @@ import { useEffect, useMemo, useRef, useState } from "react";
|
||||
import {
|
||||
ActionIcon,
|
||||
Avatar,
|
||||
Badge,
|
||||
Box,
|
||||
Button,
|
||||
Checkbox,
|
||||
CloseButton,
|
||||
Group,
|
||||
Image,
|
||||
Loader,
|
||||
NavLink,
|
||||
Paper,
|
||||
Popover,
|
||||
ScrollArea,
|
||||
Select,
|
||||
Stack,
|
||||
Text,
|
||||
Textarea,
|
||||
TextInput,
|
||||
Tooltip,
|
||||
} from "@mantine/core";
|
||||
import {
|
||||
IconAlertTriangle,
|
||||
IconArrowLeft,
|
||||
IconArrowUp,
|
||||
IconDatabase,
|
||||
IconHistory,
|
||||
IconPaperclip,
|
||||
IconPhoto,
|
||||
IconPlayerStopFilled,
|
||||
IconPlus,
|
||||
IconRefresh,
|
||||
IconRobot,
|
||||
IconUser,
|
||||
IconWorld,
|
||||
} from "@tabler/icons-react";
|
||||
|
||||
import { clientApi } from "@homarr/api/client";
|
||||
@@ -35,8 +48,28 @@ import classes from "./chat.module.css";
|
||||
interface ChatMessage {
|
||||
role: "system" | "user" | "assistant";
|
||||
content: string;
|
||||
images?: string[];
|
||||
}
|
||||
|
||||
type CompletionPart = { type: "text"; text: string } | { type: "image_url"; image_url: { url: string } };
|
||||
type CompletionMessage = { role: ChatMessage["role"]; content: string | CompletionPart[] };
|
||||
|
||||
interface ImageAttachment {
|
||||
name: string;
|
||||
dataUrl: string;
|
||||
}
|
||||
|
||||
interface WebAttachment {
|
||||
url: string;
|
||||
collectionName: string;
|
||||
title: string;
|
||||
}
|
||||
|
||||
// Heuristic fallback for detecting vision-capable models when Open WebUI does
|
||||
// not declare the capability explicitly.
|
||||
const VISION_MODEL_PATTERN =
|
||||
/vision|llava|bakllava|moondream|minicpm-?v|qwen2?-?vl|pixtral|cogvlm|internvl|gemma3|smolvlm|granite[\w-]*vision/i;
|
||||
|
||||
export function Chat({ options, integrationIds, isEditMode }: WidgetComponentProps<"openWebUi">) {
|
||||
const t = useI18n();
|
||||
const integrationId = integrationIds.at(0);
|
||||
@@ -46,16 +79,26 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
|
||||
const [input, setInput] = useState("");
|
||||
const [streamingText, setStreamingText] = useState("");
|
||||
const [isStreaming, setIsStreaming] = useState(false);
|
||||
const [pendingMessages, setPendingMessages] = useState<ChatMessage[]>([]);
|
||||
const [pendingMessages, setPendingMessages] = useState<CompletionMessage[]>([]);
|
||||
const [pendingCollections, setPendingCollections] = useState<string[]>([]);
|
||||
const [activeChatId, setActiveChatId] = useState<string | null>(null);
|
||||
const [selectedChatId, setSelectedChatId] = useState<string | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [historyOpened, setHistoryOpened] = useState(false);
|
||||
|
||||
// Attachments grounding the conversation. Images are per-message; knowledge
|
||||
// collections and web pages are sticky context kept until removed.
|
||||
const [attachments, setAttachments] = useState<ImageAttachment[]>([]);
|
||||
const [knowledgeIds, setKnowledgeIds] = useState<string[]>([]);
|
||||
const [webItems, setWebItems] = useState<WebAttachment[]>([]);
|
||||
const [webInput, setWebInput] = useState("");
|
||||
const [attachOpened, setAttachOpened] = useState(false);
|
||||
|
||||
const messagesRef = useRef<ChatMessage[]>([]);
|
||||
const streamingTextRef = useRef("");
|
||||
const viewportRef = useRef<HTMLDivElement>(null);
|
||||
const composerRef = useRef<HTMLDivElement>(null);
|
||||
const fileInputRef = useRef<HTMLInputElement>(null);
|
||||
const [composerHeight, setComposerHeight] = useState(72);
|
||||
|
||||
const utils = clientApi.useUtils();
|
||||
@@ -65,6 +108,11 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
|
||||
{ enabled: Boolean(integrationId), refetchOnWindowFocus: false, retry: false },
|
||||
);
|
||||
|
||||
const { data: knowledge = [] } = clientApi.widget.openWebUi.getKnowledge.useQuery(
|
||||
{ integrationId: integrationId ?? "" },
|
||||
{ enabled: Boolean(integrationId), refetchOnWindowFocus: false, retry: false },
|
||||
);
|
||||
|
||||
const { data: chats = [] } = clientApi.widget.openWebUi.getChats.useQuery(
|
||||
{ integrationId: integrationId ?? "" },
|
||||
{ enabled: Boolean(integrationId) && options.showHistory, refetchOnWindowFocus: false, retry: false },
|
||||
@@ -77,6 +125,7 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
|
||||
|
||||
const createChat = clientApi.widget.openWebUi.createChat.useMutation();
|
||||
const updateChat = clientApi.widget.openWebUi.updateChat.useMutation();
|
||||
const processWeb = clientApi.widget.openWebUi.processWeb.useMutation();
|
||||
|
||||
// Default to the first available model once the list loads.
|
||||
useEffect(() => {
|
||||
@@ -113,7 +162,12 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
|
||||
}, [messages, streamingText, composerHeight]);
|
||||
|
||||
clientApi.widget.openWebUi.sendMessage.useSubscription(
|
||||
{ integrationId: integrationId ?? "", model: model ?? "", messages: pendingMessages },
|
||||
{
|
||||
integrationId: integrationId ?? "",
|
||||
model: model ?? "",
|
||||
messages: pendingMessages,
|
||||
collections: pendingCollections.length > 0 ? pendingCollections : undefined,
|
||||
},
|
||||
{
|
||||
enabled: isStreaming && pendingMessages.length > 0,
|
||||
onData: (event) => {
|
||||
@@ -146,12 +200,18 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
|
||||
setStreamingText("");
|
||||
setIsStreaming(false);
|
||||
setPendingMessages([]);
|
||||
setPendingCollections([]);
|
||||
};
|
||||
|
||||
const persistChat = async (finalMessages: ChatMessage[]) => {
|
||||
if (!integrationId || !model) return;
|
||||
// Native history stores plain text; image data URLs are dropped on persist.
|
||||
const title = finalMessages.find((message) => message.role === "user")?.content.slice(0, 80) ?? "New chat";
|
||||
const payload = { title, models: [model], messages: finalMessages };
|
||||
const payload = {
|
||||
title,
|
||||
models: [model],
|
||||
messages: finalMessages.map(({ role, content }) => ({ role, content })),
|
||||
};
|
||||
try {
|
||||
if (activeChatId) {
|
||||
await updateChat.mutateAsync({ integrationId, chatId: activeChatId, chat: payload });
|
||||
@@ -167,26 +227,56 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
|
||||
}
|
||||
};
|
||||
|
||||
const handleSend = () => {
|
||||
const trimmed = input.trim();
|
||||
if (!trimmed || !model || !integrationId || isStreaming) return;
|
||||
// Kick off a completion for the given conversation (ending with a user message).
|
||||
// Shared by sending a new message and retrying after a failure.
|
||||
const startCompletion = (baseMessages: ChatMessage[]) => {
|
||||
if (!model || !integrationId) return;
|
||||
setError(null);
|
||||
|
||||
const userMessage: ChatMessage = { role: "user", content: trimmed };
|
||||
const systemPrompt = options.systemPrompt.trim();
|
||||
const payloadMessages: ChatMessage[] = systemPrompt
|
||||
? [{ role: "system", content: systemPrompt }, ...baseMessages]
|
||||
: baseMessages;
|
||||
|
||||
// Knowledge ids double as their collection name; web pages bring their own.
|
||||
const collections = [...knowledgeIds, ...webItems.map((item) => item.collectionName)];
|
||||
|
||||
streamingTextRef.current = "";
|
||||
setStreamingText("");
|
||||
setPendingMessages(toCompletionMessages(payloadMessages));
|
||||
setPendingCollections(collections);
|
||||
setIsStreaming(true);
|
||||
};
|
||||
|
||||
const handleSend = () => {
|
||||
const trimmed = input.trim();
|
||||
if ((!trimmed && attachments.length === 0) || !model || !integrationId || isStreaming) return;
|
||||
|
||||
const userMessage: ChatMessage = {
|
||||
role: "user",
|
||||
content: trimmed,
|
||||
images: attachments.length > 0 ? attachments.map((attachment) => attachment.dataUrl) : undefined,
|
||||
};
|
||||
const nextMessages = [...messagesRef.current, userMessage];
|
||||
setMessages(nextMessages);
|
||||
messagesRef.current = nextMessages;
|
||||
setInput("");
|
||||
setAttachments([]);
|
||||
|
||||
const systemPrompt = options.systemPrompt.trim();
|
||||
const payloadMessages: ChatMessage[] = systemPrompt
|
||||
? [{ role: "system", content: systemPrompt }, ...nextMessages]
|
||||
: nextMessages;
|
||||
startCompletion(nextMessages);
|
||||
};
|
||||
|
||||
streamingTextRef.current = "";
|
||||
setStreamingText("");
|
||||
setPendingMessages(payloadMessages);
|
||||
setIsStreaming(true);
|
||||
// Retry the last turn after a failure: drop any partial assistant reply and
|
||||
// re-run the completion from the most recent user message.
|
||||
const handleRetry = () => {
|
||||
if (isStreaming) return;
|
||||
const lastUserIndex = messagesRef.current.findLastIndex((message) => message.role === "user");
|
||||
if (lastUserIndex === -1) return;
|
||||
|
||||
const truncated = messagesRef.current.slice(0, lastUserIndex + 1);
|
||||
setMessages(truncated);
|
||||
messagesRef.current = truncated;
|
||||
startCompletion(truncated);
|
||||
};
|
||||
|
||||
const handleNewChat = () => {
|
||||
@@ -199,10 +289,56 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
|
||||
setStreamingText("");
|
||||
setIsStreaming(false);
|
||||
setPendingMessages([]);
|
||||
setPendingCollections([]);
|
||||
setAttachments([]);
|
||||
setKnowledgeIds([]);
|
||||
setWebItems([]);
|
||||
setHistoryOpened(false);
|
||||
};
|
||||
|
||||
const handleFilesSelected = async (fileList: FileList | null) => {
|
||||
if (!fileList) return;
|
||||
const images = await Promise.all(
|
||||
Array.from(fileList)
|
||||
.filter((file) => file.type.startsWith("image/"))
|
||||
.map(async (file) => ({ name: file.name, dataUrl: await readFileAsDataUrl(file) })),
|
||||
);
|
||||
setAttachments((previous) => [...previous, ...images]);
|
||||
};
|
||||
|
||||
const addWebUrl = async () => {
|
||||
if (!integrationId) return;
|
||||
const raw = webInput.trim();
|
||||
if (!raw) return;
|
||||
const url = /^https?:\/\//i.test(raw) ? raw : `https://${raw}`;
|
||||
setWebInput("");
|
||||
setError(null);
|
||||
try {
|
||||
const document = await processWeb.mutateAsync({ integrationId, url });
|
||||
setWebItems((previous) =>
|
||||
previous.some((item) => item.collectionName === document.collectionName)
|
||||
? previous
|
||||
: [...previous, { url, collectionName: document.collectionName, title: document.title }],
|
||||
);
|
||||
} catch {
|
||||
setError(t("widget.openWebUi.webError"));
|
||||
}
|
||||
};
|
||||
|
||||
const modelData = useMemo(() => models.map((m) => ({ value: m.id, label: m.name ?? m.id })), [models]);
|
||||
const knowledgeName = (id: string) => knowledge.find((item) => item.id === id)?.name ?? id;
|
||||
const hasAttachments = attachments.length > 0 || knowledgeIds.length > 0 || webItems.length > 0;
|
||||
|
||||
// Whether the selected model can actually see attached images. Prefer Open
|
||||
// WebUI's declared capability; fall back to a name heuristic when it's absent.
|
||||
const selectedModel = useMemo(() => models.find((item) => item.id === model), [models, model]);
|
||||
const modelSupportsVision = useMemo(() => {
|
||||
if (!selectedModel) return true;
|
||||
const capability = selectedModel.info?.meta?.capabilities?.vision;
|
||||
if (typeof capability === "boolean") return capability;
|
||||
return VISION_MODEL_PATTERN.test(selectedModel.id) || VISION_MODEL_PATTERN.test(selectedModel.name ?? "");
|
||||
}, [selectedModel]);
|
||||
const showVisionWarning = attachments.length > 0 && !modelSupportsVision;
|
||||
|
||||
return (
|
||||
<Box className={classes.root} style={isEditMode ? { pointerEvents: "none" } : undefined}>
|
||||
@@ -215,18 +351,92 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
|
||||
</Stack>
|
||||
) : null}
|
||||
{messages.map((message, index) => (
|
||||
<MessageBubble key={index} from={message.role} content={message.content} />
|
||||
<MessageBubble key={index} from={message.role} content={message.content} images={message.images} />
|
||||
))}
|
||||
{isStreaming ? <MessageBubble from="assistant" content={streamingText || "…"} /> : null}
|
||||
{error ? (
|
||||
<Text size="xs" c="red.6">
|
||||
{error}
|
||||
</Text>
|
||||
<Group gap="xs" justify="center" wrap="nowrap">
|
||||
<Text size="xs" c="red.6">
|
||||
{error}
|
||||
</Text>
|
||||
<Button
|
||||
size="compact-xs"
|
||||
variant="light"
|
||||
color="red"
|
||||
leftSection={<IconRefresh size={14} />}
|
||||
onClick={handleRetry}
|
||||
>
|
||||
{t("widget.openWebUi.retry")}
|
||||
</Button>
|
||||
</Group>
|
||||
) : null}
|
||||
</Stack>
|
||||
</ScrollArea>
|
||||
|
||||
<Paper ref={composerRef} className={classes.composer} withBorder radius="md" p="xs">
|
||||
{hasAttachments ? (
|
||||
<Group gap={6} mb={6}>
|
||||
{attachments.map((attachment, index) => (
|
||||
<Box key={`${attachment.name}-${index}`} className={classes.thumb}>
|
||||
<Image src={attachment.dataUrl} alt={attachment.name} w={48} h={48} radius="sm" fit="cover" />
|
||||
<CloseButton
|
||||
size="xs"
|
||||
className={classes.thumbRemove}
|
||||
onClick={() => setAttachments((previous) => previous.filter((_, i) => i !== index))}
|
||||
aria-label={t("widget.openWebUi.removeAttachment")}
|
||||
/>
|
||||
</Box>
|
||||
))}
|
||||
{knowledgeIds.map((id) => (
|
||||
<Badge
|
||||
key={id}
|
||||
variant="light"
|
||||
color="grape"
|
||||
leftSection={<IconDatabase size={12} />}
|
||||
rightSection={
|
||||
<CloseButton
|
||||
size="xs"
|
||||
onClick={() => setKnowledgeIds((previous) => previous.filter((value) => value !== id))}
|
||||
aria-label={t("widget.openWebUi.removeAttachment")}
|
||||
/>
|
||||
}
|
||||
>
|
||||
{knowledgeName(id)}
|
||||
</Badge>
|
||||
))}
|
||||
{webItems.map((item) => (
|
||||
<Badge
|
||||
key={item.collectionName}
|
||||
variant="light"
|
||||
color="blue"
|
||||
leftSection={<IconWorld size={12} />}
|
||||
rightSection={
|
||||
<CloseButton
|
||||
size="xs"
|
||||
onClick={() =>
|
||||
setWebItems((previous) =>
|
||||
previous.filter((value) => value.collectionName !== item.collectionName),
|
||||
)
|
||||
}
|
||||
aria-label={t("widget.openWebUi.removeAttachment")}
|
||||
/>
|
||||
}
|
||||
>
|
||||
{hostnameOf(item.url)}
|
||||
</Badge>
|
||||
))}
|
||||
</Group>
|
||||
) : null}
|
||||
|
||||
{showVisionWarning ? (
|
||||
<Group gap={4} mb={6} wrap="nowrap">
|
||||
<IconAlertTriangle size={14} color="var(--mantine-color-yellow-6)" />
|
||||
<Text size="xs" c="yellow.7">
|
||||
{t("widget.openWebUi.visionWarning")}
|
||||
</Text>
|
||||
</Group>
|
||||
) : null}
|
||||
|
||||
<Textarea
|
||||
variant="unstyled"
|
||||
size="sm"
|
||||
@@ -255,6 +465,86 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
|
||||
<IconPlus size={18} />
|
||||
</ActionIcon>
|
||||
</Tooltip>
|
||||
<Popover
|
||||
opened={attachOpened}
|
||||
onChange={setAttachOpened}
|
||||
position="top-start"
|
||||
withinPortal={false}
|
||||
width={260}
|
||||
shadow="md"
|
||||
>
|
||||
<Popover.Target>
|
||||
<Tooltip label={t("widget.openWebUi.attach")} withinPortal={false}>
|
||||
<ActionIcon
|
||||
variant="subtle"
|
||||
color="gray"
|
||||
onClick={() => setAttachOpened((value) => !value)}
|
||||
aria-label={t("widget.openWebUi.attach")}
|
||||
>
|
||||
<IconPaperclip size={18} />
|
||||
</ActionIcon>
|
||||
</Tooltip>
|
||||
</Popover.Target>
|
||||
<Popover.Dropdown>
|
||||
<Stack gap="sm">
|
||||
<Button
|
||||
variant="light"
|
||||
color="gray"
|
||||
size="xs"
|
||||
leftSection={<IconPhoto size={16} />}
|
||||
justify="flex-start"
|
||||
onClick={() => fileInputRef.current?.click()}
|
||||
>
|
||||
{t("widget.openWebUi.attachImage")}
|
||||
</Button>
|
||||
|
||||
{knowledge.length > 0 ? (
|
||||
<Stack gap={4}>
|
||||
<Text size="xs" fw={600} c="dimmed">
|
||||
{t("widget.openWebUi.attachKnowledge")}
|
||||
</Text>
|
||||
<Checkbox.Group value={knowledgeIds} onChange={setKnowledgeIds}>
|
||||
<Stack gap={4}>
|
||||
{knowledge.map((item) => (
|
||||
<Checkbox key={item.id} value={item.id} label={item.name ?? item.id} size="xs" />
|
||||
))}
|
||||
</Stack>
|
||||
</Checkbox.Group>
|
||||
</Stack>
|
||||
) : null}
|
||||
|
||||
<Stack gap={4}>
|
||||
<Text size="xs" fw={600} c="dimmed">
|
||||
{t("widget.openWebUi.attachWebpage")}
|
||||
</Text>
|
||||
<Group gap={4} wrap="nowrap">
|
||||
<TextInput
|
||||
size="xs"
|
||||
flex={1}
|
||||
placeholder="https://…"
|
||||
value={webInput}
|
||||
onChange={(event) => setWebInput(event.currentTarget.value)}
|
||||
onKeyDown={(event) => {
|
||||
if (event.key === "Enter") {
|
||||
event.preventDefault();
|
||||
void addWebUrl();
|
||||
}
|
||||
}}
|
||||
/>
|
||||
<ActionIcon
|
||||
variant="light"
|
||||
color="gray"
|
||||
loading={processWeb.isPending}
|
||||
onClick={() => void addWebUrl()}
|
||||
aria-label={t("widget.openWebUi.addUrl")}
|
||||
>
|
||||
<IconPlus size={16} />
|
||||
</ActionIcon>
|
||||
</Group>
|
||||
</Stack>
|
||||
</Stack>
|
||||
</Popover.Dropdown>
|
||||
</Popover>
|
||||
{options.showHistory ? (
|
||||
<Tooltip label={t("widget.openWebUi.history")} withinPortal={false}>
|
||||
<ActionIcon
|
||||
@@ -301,7 +591,7 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
|
||||
radius="xl"
|
||||
variant="filled"
|
||||
onClick={handleSend}
|
||||
disabled={!input.trim() || !model}
|
||||
disabled={(!input.trim() && attachments.length === 0) || !model}
|
||||
aria-label={t("widget.openWebUi.send")}
|
||||
>
|
||||
<IconArrowUp size={18} />
|
||||
@@ -310,6 +600,19 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
|
||||
)}
|
||||
</Group>
|
||||
</Group>
|
||||
|
||||
<input
|
||||
ref={fileInputRef}
|
||||
type="file"
|
||||
accept="image/*"
|
||||
multiple
|
||||
hidden
|
||||
aria-label={t("widget.openWebUi.attachImage")}
|
||||
onChange={(event) => {
|
||||
void handleFilesSelected(event.currentTarget.files);
|
||||
event.currentTarget.value = "";
|
||||
}}
|
||||
/>
|
||||
</Paper>
|
||||
|
||||
{options.showHistory && historyOpened ? (
|
||||
@@ -353,6 +656,37 @@ export function Chat({ options, integrationIds, isEditMode }: WidgetComponentPro
|
||||
);
|
||||
}
|
||||
|
||||
const readFileAsDataUrl = (file: File): Promise<string> =>
|
||||
new Promise((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
reader.addEventListener("load", () => resolve(reader.result as string));
|
||||
reader.addEventListener("error", () => reject(reader.error ?? new Error("Failed to read file")));
|
||||
reader.readAsDataURL(file);
|
||||
});
|
||||
|
||||
const hostnameOf = (url: string): string => {
|
||||
try {
|
||||
return new URL(url).hostname || url;
|
||||
} catch {
|
||||
return url;
|
||||
}
|
||||
};
|
||||
|
||||
// Convert displayed messages to the OpenAI completion format, inlining any
|
||||
// attached images on the message that carries them (vision content parts).
|
||||
const toCompletionMessages = (messages: ChatMessage[]): CompletionMessage[] =>
|
||||
messages.map((message) =>
|
||||
message.images && message.images.length > 0
|
||||
? {
|
||||
role: message.role,
|
||||
content: [
|
||||
...(message.content ? [{ type: "text" as const, text: message.content }] : []),
|
||||
...message.images.map((url) => ({ type: "image_url" as const, image_url: { url } })),
|
||||
],
|
||||
}
|
||||
: { role: message.role, content: message.content },
|
||||
);
|
||||
|
||||
type RawMessage = { role?: string | null; content?: string | null };
|
||||
type HistoryMessage = RawMessage & { parentId?: string | null };
|
||||
interface RawChat {
|
||||
@@ -393,7 +727,7 @@ const extractChatMessages = (chat: RawChat | null | undefined): ChatMessage[] =>
|
||||
.filter((message): message is ChatMessage => message !== null);
|
||||
};
|
||||
|
||||
function MessageBubble({ from, content }: { from: ChatMessage["role"]; content: string }) {
|
||||
function MessageBubble({ from, content, images }: { from: ChatMessage["role"]; content: string; images?: string[] }) {
|
||||
const isUser = from === "user";
|
||||
const avatar = (
|
||||
<Avatar size={22} radius="xl" variant="filled" color={isUser ? "gray" : "blue"} className={classes.avatar}>
|
||||
@@ -410,9 +744,18 @@ function MessageBubble({ from, content }: { from: ChatMessage["role"]; content:
|
||||
p="xs"
|
||||
radius="md"
|
||||
>
|
||||
<Text size="sm" style={{ whiteSpace: "pre-wrap", wordBreak: "break-word" }}>
|
||||
{content}
|
||||
</Text>
|
||||
{images && images.length > 0 ? (
|
||||
<Group gap={4} mb={content ? 6 : 0}>
|
||||
{images.map((src, index) => (
|
||||
<Image key={index} src={src} alt="" w={120} radius="sm" fit="contain" />
|
||||
))}
|
||||
</Group>
|
||||
) : null}
|
||||
{content ? (
|
||||
<Text size="sm" style={{ whiteSpace: "pre-wrap", wordBreak: "break-word" }}>
|
||||
{content}
|
||||
</Text>
|
||||
) : null}
|
||||
</Paper>
|
||||
{isUser ? avatar : null}
|
||||
</Group>
|
||||
|
||||
Reference in new issue
Block a user