refactor(open-webui): split chat widget into hooks and components
Break the monolithic chat.tsx into focused hooks (use-attachments, use-chat-stream, use-voice-recording, use-camera-capture) and presentational components (chat-components), with shared types and utils extracted to chat-types/chat-utils. Review fixes applied on top: - require auth for the streaming completion (protectedProcedure) so anonymous board viewers cannot spend the owner's LLM credits - rename image-only state to images and the picker query options object to pickerQueryOptions for clarity - extract AttachPopover, clearPending(), composerEmpty and releaseAudioResources() to remove duplication
This commit is contained in:
Lok3sh-Kumar committed
1 parent
84275bd19a
commit
8e0f8cd46a
11 files changed
+1563
-1189
No files matched your search
@@ -55,6 +55,15 @@ const messageText = (message: CompletionMessage): string =>
|
||||
.map((part) => part.text)
|
||||
.join(" ");
|
||||
|
||||
// Number of chunks to retrieve per collection when grounding a reply.
|
||||
const RETRIEVAL_K = 4;
|
||||
|
||||
// Instruction prepended to the retrieved context so the model knows how to use
|
||||
// it. Model-facing, so intentionally not localized.
|
||||
const RETRIEVAL_INSTRUCTION =
|
||||
"Use the following context from the user's attached sources (web pages, knowledge bases, " +
|
||||
"files, notes and chats) to answer. If the answer is not in the context, say so.";
|
||||
|
||||
// Retrieve relevant chunks from the attached collections and prepend them as a
|
||||
// system context message so the model can ground its reply on them.
|
||||
const groundMessagesAsync = async (
|
||||
@@ -65,13 +74,19 @@ const groundMessagesAsync = async (
|
||||
): Promise<CompletionMessage[]> => {
|
||||
if (collections.length === 0 && contextTexts.length === 0) return messages;
|
||||
|
||||
const lastUserMessage = messages.findLast((message) => message.role === "user");
|
||||
const query = lastUserMessage ? messageText(lastUserMessage) : "";
|
||||
// Use the most recent user message that has text. The latest turn can be
|
||||
// image-only (no text), in which case we fall back to the prior question so
|
||||
// attached collections still get a meaningful retrieval query.
|
||||
const query =
|
||||
messages
|
||||
.filter((message) => message.role === "user")
|
||||
.map(messageText)
|
||||
.findLast((text) => text.trim() !== "") ?? "";
|
||||
|
||||
let documents: string[] = [];
|
||||
if (collections.length > 0) {
|
||||
try {
|
||||
documents = await client.queryCollectionsAsync(collections, query, 4);
|
||||
documents = await client.queryCollectionsAsync(collections, query, RETRIEVAL_K);
|
||||
} catch (error) {
|
||||
logger.warn("Open WebUI context retrieval failed; continuing without it", { error });
|
||||
}
|
||||
@@ -82,9 +97,7 @@ const groundMessagesAsync = async (
|
||||
|
||||
const contextMessage: CompletionMessage = {
|
||||
role: "system",
|
||||
content:
|
||||
"Use the following context from the user's attached sources (web pages, knowledge bases, " +
|
||||
`files, notes and chats) to answer. If the answer is not in the context, say so.\n\n${blocks.join("\n\n")}`,
|
||||
content: `${RETRIEVAL_INSTRUCTION}\n\n${blocks.join("\n\n")}`,
|
||||
};
|
||||
return [contextMessage, ...messages];
|
||||
};
|
||||
@@ -205,9 +218,12 @@ export const openWebUiRouter = createTRPCRouter({
|
||||
}),
|
||||
|
||||
// Streaming chat completion. Delivered over the WebSocket subscription link so
|
||||
// tokens can be emitted as they arrive. Uses "query" access (same level as
|
||||
// viewing the widget) to match the other integration subscriptions in the app.
|
||||
sendMessage: publicProcedure
|
||||
// tokens can be emitted as they arrive. A completion spends the owner's upstream
|
||||
// LLM credits, so it must not be reachable by anonymous viewers of a public
|
||||
// board: protectedProcedure requires a session (the WS context resolves it from
|
||||
// the session-token cookie) and the "interact" middleware checks integration
|
||||
// access, matching processWeb/uploadFile/transcribe above.
|
||||
sendMessage: protectedProcedure
|
||||
.input(
|
||||
z.object({
|
||||
model: z.string(),
|
||||
@@ -219,7 +235,7 @@ export const openWebUiRouter = createTRPCRouter({
|
||||
contextTexts: z.array(z.string()).optional(),
|
||||
}),
|
||||
)
|
||||
.concat(createOneIntegrationMiddleware("query", "openWebUi"))
|
||||
.concat(createOneIntegrationMiddleware("interact", "openWebUi"))
|
||||
.subscription(({ ctx, input }) => {
|
||||
return observable<StreamEvent>((emit) => {
|
||||
const controller = new AbortController();
|
||||
|
||||
@@ -35,6 +35,10 @@ import {
|
||||
|
||||
const logger = createLogger({ module: "open-webui-integration" });
|
||||
|
||||
// The (undici) request options accepted by the shared fetch helper. Derived so
|
||||
// the request helpers stay in sync with the underlying fetch types.
|
||||
type RequestOptions = NonNullable<Parameters<typeof fetchWithTrustedCertificatesAsync>[1]>;
|
||||
|
||||
export class OpenWebUiIntegration extends Integration {
|
||||
protected async testingAsync(input: IntegrationTestingInput): Promise<TestingResult> {
|
||||
const response = await input.fetchAsync(this.url("/api/models"), {
|
||||
@@ -52,30 +56,15 @@ export class OpenWebUiIntegration extends Integration {
|
||||
* List the models served by Open WebUI (aggregates all configured providers).
|
||||
*/
|
||||
public async getModelsAsync(): Promise<OpenWebUiModel[]> {
|
||||
const response = await fetchWithTrustedCertificatesAsync(this.url("/api/models"), {
|
||||
headers: this.getAuthHeaders(),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new ResponseError(response);
|
||||
}
|
||||
|
||||
return openWebUiModelsResponseSchema.parse(await response.json()).data;
|
||||
const parsed = await this.getJsonAsync("/api/models", openWebUiModelsResponseSchema);
|
||||
return parsed.data;
|
||||
}
|
||||
|
||||
/**
|
||||
* List the knowledge bases (RAG collections) the user can ground chats on.
|
||||
*/
|
||||
public async getKnowledgeAsync(): Promise<OpenWebUiKnowledge[]> {
|
||||
const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/knowledge/"), {
|
||||
headers: this.getAuthHeaders(),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new ResponseError(response);
|
||||
}
|
||||
|
||||
const parsed = openWebUiKnowledgeListSchema.parse(await response.json());
|
||||
const parsed = await this.getJsonAsync("/api/v1/knowledge/", openWebUiKnowledgeListSchema);
|
||||
return Array.isArray(parsed) ? parsed : parsed.items;
|
||||
}
|
||||
|
||||
@@ -83,16 +72,7 @@ export class OpenWebUiIntegration extends Integration {
|
||||
* List the user's uploaded files (for the "Attach Files" picker).
|
||||
*/
|
||||
public async listFilesAsync(): Promise<OpenWebUiFileSummary[]> {
|
||||
const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/files/"), {
|
||||
headers: this.getAuthHeaders(),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new ResponseError(response);
|
||||
}
|
||||
|
||||
const parsed = openWebUiFileListSchema.parse(await response.json());
|
||||
const files = Array.isArray(parsed) ? parsed : parsed.items;
|
||||
const files = await this.fetchFilesAsync();
|
||||
return files.map((file) => ({ id: file.id, name: file.meta?.name ?? file.filename ?? file.id }));
|
||||
}
|
||||
|
||||
@@ -102,34 +82,23 @@ export class OpenWebUiIntegration extends Integration {
|
||||
* we read the files list and filter by the base's collection id instead.
|
||||
*/
|
||||
public async getKnowledgeFilesAsync(knowledgeId: string): Promise<OpenWebUiFileSummary[]> {
|
||||
const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/files/"), {
|
||||
headers: this.getAuthHeaders(),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new ResponseError(response);
|
||||
}
|
||||
|
||||
const parsed = openWebUiFileListSchema.parse(await response.json());
|
||||
const files = Array.isArray(parsed) ? parsed : parsed.items;
|
||||
const files = await this.fetchFilesAsync();
|
||||
return files
|
||||
.filter((file) => file.meta?.collection_name === knowledgeId)
|
||||
.map((file) => ({ id: file.id, name: file.meta?.name ?? file.filename ?? file.id }));
|
||||
}
|
||||
|
||||
/** Fetch the raw file list, normalizing the paginated and bare-array shapes. */
|
||||
private async fetchFilesAsync() {
|
||||
const parsed = await this.getJsonAsync("/api/v1/files/", openWebUiFileListSchema);
|
||||
return Array.isArray(parsed) ? parsed : parsed.items;
|
||||
}
|
||||
|
||||
/**
|
||||
* List the user's notes (for the "Attach Notes" picker).
|
||||
*/
|
||||
public async listNotesAsync(): Promise<OpenWebUiNote[]> {
|
||||
const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/notes/"), {
|
||||
headers: this.getAuthHeaders(),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new ResponseError(response);
|
||||
}
|
||||
|
||||
const parsed = openWebUiNoteListSchema.parse(await response.json());
|
||||
const parsed = await this.getJsonAsync("/api/v1/notes/", openWebUiNoteListSchema);
|
||||
const notes = Array.isArray(parsed) ? parsed : parsed.items;
|
||||
return notes.map((note) => ({
|
||||
id: note.id,
|
||||
@@ -143,15 +112,10 @@ export class OpenWebUiIntegration extends Integration {
|
||||
* truncates the body, so we fetch the note directly when attaching it.
|
||||
*/
|
||||
public async getNoteAsync(noteId: string): Promise<OpenWebUiNote> {
|
||||
const response = await fetchWithTrustedCertificatesAsync(this.url(`/api/v1/notes/${noteId}`), {
|
||||
headers: this.getAuthHeaders(),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new ResponseError(response);
|
||||
}
|
||||
|
||||
const parsed = openWebUiNoteSchema.parse(await response.json());
|
||||
const parsed = await this.getJsonAsync(
|
||||
`/api/v1/notes/${encodeURIComponent(noteId)}`,
|
||||
openWebUiNoteSchema,
|
||||
);
|
||||
return { id: parsed.id, title: parsed.title ?? "Untitled note", content: parsed.data?.content?.md ?? "" };
|
||||
}
|
||||
|
||||
@@ -162,17 +126,8 @@ export class OpenWebUiIntegration extends Integration {
|
||||
const form = new FormData();
|
||||
form.append("file", new Blob([Buffer.from(base64Content, "base64")], { type: contentType }), filename);
|
||||
|
||||
const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/audio/transcriptions"), {
|
||||
method: "POST",
|
||||
headers: this.getAuthHeaders(),
|
||||
body: form,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new ResponseError(response);
|
||||
}
|
||||
|
||||
return openWebUiTranscriptionSchema.parse(await response.json()).text;
|
||||
const parsed = await this.postFormAsync("/api/v1/audio/transcriptions", form, openWebUiTranscriptionSchema);
|
||||
return parsed.text;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -187,17 +142,7 @@ export class OpenWebUiIntegration extends Integration {
|
||||
const form = new FormData();
|
||||
form.append("file", new Blob([Buffer.from(base64Content, "base64")], { type: contentType }), filename);
|
||||
|
||||
const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/files/"), {
|
||||
method: "POST",
|
||||
headers: this.getAuthHeaders(),
|
||||
body: form,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new ResponseError(response);
|
||||
}
|
||||
|
||||
const parsed = openWebUiUploadedFileSchema.parse(await response.json());
|
||||
const parsed = await this.postFormAsync("/api/v1/files/", form, openWebUiUploadedFileSchema);
|
||||
return { id: parsed.id, name: parsed.meta?.name ?? parsed.filename ?? filename };
|
||||
}
|
||||
|
||||
@@ -207,17 +152,11 @@ export class OpenWebUiIntegration extends Integration {
|
||||
* embed the page here and inject retrieved chunks ourselves at send time.
|
||||
*/
|
||||
public async processWebAsync(url: string): Promise<OpenWebUiWebDocument> {
|
||||
const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/retrieval/process/web"), {
|
||||
method: "POST",
|
||||
headers: { ...this.getAuthHeaders(), "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ url, collection_name: "" }),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new ResponseError(response);
|
||||
}
|
||||
|
||||
const parsed = openWebUiProcessWebResponseSchema.parse(await response.json());
|
||||
const parsed = await this.postJsonAsync(
|
||||
"/api/v1/retrieval/process/web",
|
||||
{ url, collection_name: "" },
|
||||
openWebUiProcessWebResponseSchema,
|
||||
);
|
||||
return { collectionName: parsed.collection_name, title: parsed.file?.meta?.name ?? url };
|
||||
}
|
||||
|
||||
@@ -230,17 +169,11 @@ export class OpenWebUiIntegration extends Integration {
|
||||
return [];
|
||||
}
|
||||
|
||||
const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/retrieval/query/collection"), {
|
||||
method: "POST",
|
||||
headers: { ...this.getAuthHeaders(), "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ collection_names: collectionNames, query, k }),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new ResponseError(response);
|
||||
}
|
||||
|
||||
const parsed = openWebUiCollectionQueryResponseSchema.parse(await response.json());
|
||||
const parsed = await this.postJsonAsync(
|
||||
"/api/v1/retrieval/query/collection",
|
||||
{ collection_names: collectionNames, query, k },
|
||||
openWebUiCollectionQueryResponseSchema,
|
||||
);
|
||||
return (parsed.documents ?? []).flat();
|
||||
}
|
||||
|
||||
@@ -248,64 +181,32 @@ export class OpenWebUiIntegration extends Integration {
|
||||
* List the user's chats (native Open WebUI history).
|
||||
*/
|
||||
public async listChatsAsync(): Promise<OpenWebUiChatListItem[]> {
|
||||
const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/chats/"), {
|
||||
headers: this.getAuthHeaders(),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new ResponseError(response);
|
||||
}
|
||||
|
||||
return openWebUiChatListItemSchema.array().parse(await response.json());
|
||||
return await this.getJsonAsync("/api/v1/chats/", openWebUiChatListItemSchema.array());
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a single chat including its messages.
|
||||
*/
|
||||
public async getChatAsync(chatId: string): Promise<OpenWebUiChat> {
|
||||
const response = await fetchWithTrustedCertificatesAsync(this.url(`/api/v1/chats/${chatId}`), {
|
||||
headers: this.getAuthHeaders(),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new ResponseError(response);
|
||||
}
|
||||
|
||||
return openWebUiChatSchema.parse(await response.json());
|
||||
return await this.getJsonAsync(`/api/v1/chats/${encodeURIComponent(chatId)}`, openWebUiChatSchema);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new chat in the native history.
|
||||
*/
|
||||
public async createChatAsync(payload: OpenWebUiChatPayload): Promise<OpenWebUiChat> {
|
||||
const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/chats/new"), {
|
||||
method: "POST",
|
||||
headers: { ...this.getAuthHeaders(), "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ chat: payload }),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new ResponseError(response);
|
||||
}
|
||||
|
||||
return openWebUiChatSchema.parse(await response.json());
|
||||
return await this.postJsonAsync("/api/v1/chats/new", { chat: payload }, openWebUiChatSchema);
|
||||
}
|
||||
|
||||
/**
|
||||
* Update an existing chat in the native history.
|
||||
*/
|
||||
public async updateChatAsync(chatId: string, payload: OpenWebUiChatPayload): Promise<OpenWebUiChat> {
|
||||
const response = await fetchWithTrustedCertificatesAsync(this.url(`/api/v1/chats/${chatId}`), {
|
||||
method: "POST",
|
||||
headers: { ...this.getAuthHeaders(), "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ chat: payload }),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new ResponseError(response);
|
||||
}
|
||||
|
||||
return openWebUiChatSchema.parse(await response.json());
|
||||
return await this.postJsonAsync(
|
||||
`/api/v1/chats/${encodeURIComponent(chatId)}`,
|
||||
{ chat: payload },
|
||||
openWebUiChatSchema,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -318,14 +219,14 @@ export class OpenWebUiIntegration extends Integration {
|
||||
onDelta: (delta: string) => void,
|
||||
signal: AbortSignal,
|
||||
): Promise<void> {
|
||||
const response = await fetchWithTrustedCertificatesAsync(this.url("/api/chat/completions"), {
|
||||
const response = await this.requestAsync("/api/chat/completions", {
|
||||
method: "POST",
|
||||
headers: { ...this.getAuthHeaders(), "Content-Type": "application/json" },
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ model: params.model, messages: params.messages, stream: true }),
|
||||
signal,
|
||||
});
|
||||
|
||||
if (!response.ok || !response.body) {
|
||||
if (!response.body) {
|
||||
throw new ResponseError(response);
|
||||
}
|
||||
|
||||
@@ -343,16 +244,20 @@ export class OpenWebUiIntegration extends Integration {
|
||||
|
||||
try {
|
||||
const parsed = openWebUiCompletionChunkSchema.parse(JSON.parse(payload));
|
||||
const delta = parsed.choices?.[0]?.delta?.content;
|
||||
if (delta) {
|
||||
onDelta(delta);
|
||||
const choice = parsed.choices?.[0];
|
||||
if (choice?.delta?.content) {
|
||||
onDelta(choice.delta.content);
|
||||
}
|
||||
// Some providers end the stream with a finish_reason chunk and no
|
||||
// explicit [DONE] sentinel; treat that as a clean completion too.
|
||||
if (choice?.finish_reason) return true;
|
||||
} catch {
|
||||
logger.debug("Failed to parse Open WebUI completion chunk", { line: trimmed });
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
let completed = false;
|
||||
try {
|
||||
while (!signal.aborted) {
|
||||
const { done, value } = await reader.read();
|
||||
@@ -362,14 +267,25 @@ export class OpenWebUiIntegration extends Integration {
|
||||
const lines = buffer.split("\n");
|
||||
buffer = lines.pop() ?? "";
|
||||
for (const line of lines) {
|
||||
if (processLine(line)) return;
|
||||
if (processLine(line)) {
|
||||
completed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (completed) break;
|
||||
}
|
||||
} finally {
|
||||
reader.cancel().catch(() => {
|
||||
// ignore cancellation errors during teardown
|
||||
});
|
||||
}
|
||||
|
||||
// The reader ended without a [DONE]/finish_reason marker: the upstream
|
||||
// connection was cut mid-stream. Surface it instead of presenting the
|
||||
// partial reply as a complete answer.
|
||||
if (!signal.aborted && !completed) {
|
||||
throw new Error("Open WebUI stream ended before completion");
|
||||
}
|
||||
}
|
||||
|
||||
private getAuthHeaders(): Record<string, string> {
|
||||
@@ -377,4 +293,58 @@ export class OpenWebUiIntegration extends Integration {
|
||||
Authorization: `Bearer ${this.getSecretValue("apiKey")}`,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Issue an authenticated request against the Open WebUI API and throw a
|
||||
* {@link ResponseError} on a non-2xx status. Centralizes the auth header,
|
||||
* trusted-certificate fetch and error handling shared by every endpoint.
|
||||
*/
|
||||
private async requestAsync(
|
||||
path: `/${string}`,
|
||||
init?: Omit<RequestOptions, "headers"> & { headers?: Record<string, string> },
|
||||
) {
|
||||
const response = await fetchWithTrustedCertificatesAsync(this.url(path), {
|
||||
...init,
|
||||
headers: { ...this.getAuthHeaders(), ...init?.headers },
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new ResponseError(response);
|
||||
}
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
/** GET an endpoint and validate the JSON body against `schema`. */
|
||||
private async getJsonAsync<TOutput>(
|
||||
path: `/${string}`,
|
||||
schema: { parse: (data: unknown) => TOutput },
|
||||
): Promise<TOutput> {
|
||||
const response = await this.requestAsync(path);
|
||||
return schema.parse(await response.json());
|
||||
}
|
||||
|
||||
/** POST a JSON body and validate the JSON response against `schema`. */
|
||||
private async postJsonAsync<TOutput>(
|
||||
path: `/${string}`,
|
||||
body: unknown,
|
||||
schema: { parse: (data: unknown) => TOutput },
|
||||
): Promise<TOutput> {
|
||||
const response = await this.requestAsync(path, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
return schema.parse(await response.json());
|
||||
}
|
||||
|
||||
/** POST a multipart form (file upload) and validate the JSON response. */
|
||||
private async postFormAsync<TOutput>(
|
||||
path: `/${string}`,
|
||||
form: FormData,
|
||||
schema: { parse: (data: unknown) => TOutput },
|
||||
): Promise<TOutput> {
|
||||
const response = await this.requestAsync(path, { method: "POST", body: form });
|
||||
return schema.parse(await response.json());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,484 @@
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import {
|
||||
ActionIcon,
|
||||
Avatar,
|
||||
Badge,
|
||||
Box,
|
||||
Button,
|
||||
Checkbox,
|
||||
CloseButton,
|
||||
Group,
|
||||
Image,
|
||||
Loader,
|
||||
Paper,
|
||||
Popover,
|
||||
ScrollArea,
|
||||
Stack,
|
||||
Text,
|
||||
TextInput,
|
||||
Tooltip,
|
||||
} from "@mantine/core";
|
||||
import {
|
||||
IconArrowLeft,
|
||||
IconCamera,
|
||||
IconChevronDown,
|
||||
IconChevronRight,
|
||||
IconDatabase,
|
||||
IconFile,
|
||||
IconFileText,
|
||||
IconMessage,
|
||||
IconPaperclip,
|
||||
IconPlus,
|
||||
IconRobot,
|
||||
IconUser,
|
||||
IconWorld,
|
||||
} from "@tabler/icons-react";
|
||||
import ReactMarkdown from "react-markdown";
|
||||
|
||||
import { clientApi } from "@homarr/api/client";
|
||||
import { useI18n } from "@homarr/translation/client";
|
||||
|
||||
import type { AttachView, ChatMessage, FileAttachment } from "./chat-types";
|
||||
import type { useAttachments } from "./use-attachments";
|
||||
import classes from "./chat.module.css";
|
||||
|
||||
export function MenuRow({
|
||||
icon,
|
||||
label,
|
||||
hasSub,
|
||||
onClick,
|
||||
}: {
|
||||
icon: React.ReactNode;
|
||||
label: string;
|
||||
hasSub?: boolean;
|
||||
onClick: () => void;
|
||||
}) {
|
||||
return (
|
||||
<Button
|
||||
variant="subtle"
|
||||
color="gray"
|
||||
size="sm"
|
||||
fullWidth
|
||||
justify="space-between"
|
||||
leftSection={icon}
|
||||
rightSection={hasSub ? <IconChevronRight size={14} /> : <Box w={14} />}
|
||||
onClick={onClick}
|
||||
classNames={{ label: classes.menuRowLabel }}
|
||||
>
|
||||
{label}
|
||||
</Button>
|
||||
);
|
||||
}
|
||||
|
||||
export function PickerList({
|
||||
count,
|
||||
emptyLabel,
|
||||
children,
|
||||
}: {
|
||||
count: number;
|
||||
emptyLabel: string;
|
||||
children: React.ReactNode;
|
||||
}) {
|
||||
if (count === 0) {
|
||||
return (
|
||||
<Text size="xs" c="dimmed" p="xs">
|
||||
{emptyLabel}
|
||||
</Text>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<ScrollArea.Autosize mah={220} type="auto">
|
||||
<Stack gap={6} p={4}>
|
||||
{children}
|
||||
</Stack>
|
||||
</ScrollArea.Autosize>
|
||||
);
|
||||
}
|
||||
|
||||
export function KnowledgeRow({
|
||||
integrationId,
|
||||
knowledgeId,
|
||||
name,
|
||||
selected,
|
||||
onToggle,
|
||||
isFileSelected,
|
||||
onToggleFile,
|
||||
emptyFilesLabel,
|
||||
}: {
|
||||
integrationId: string;
|
||||
knowledgeId: string;
|
||||
name: string;
|
||||
selected: boolean;
|
||||
onToggle: () => void;
|
||||
isFileSelected: (id: string) => boolean;
|
||||
onToggleFile: (file: FileAttachment) => void;
|
||||
emptyFilesLabel: string;
|
||||
}) {
|
||||
// Default to expanded so a base's documents are visible without an extra click.
|
||||
const [expanded, setExpanded] = useState(true);
|
||||
const { data: files = [], isLoading } = clientApi.widget.openWebUi.getKnowledgeFiles.useQuery(
|
||||
{ integrationId, knowledgeId },
|
||||
{ enabled: expanded && Boolean(integrationId), refetchOnWindowFocus: false, retry: false },
|
||||
);
|
||||
|
||||
return (
|
||||
<Stack gap={4}>
|
||||
<Group gap={6} wrap="nowrap" justify="space-between">
|
||||
<Checkbox size="xs" label={name} checked={selected} onChange={onToggle} />
|
||||
<ActionIcon
|
||||
variant="subtle"
|
||||
color="gray"
|
||||
size="sm"
|
||||
onClick={() => setExpanded((value) => !value)}
|
||||
aria-label={name}
|
||||
>
|
||||
{expanded ? <IconChevronDown size={14} /> : <IconChevronRight size={14} />}
|
||||
</ActionIcon>
|
||||
</Group>
|
||||
{expanded ? (
|
||||
<Stack gap={4} pl="md">
|
||||
{isLoading ? (
|
||||
<Loader size="xs" />
|
||||
) : files.length === 0 ? (
|
||||
<Text size="xs" c="dimmed">
|
||||
{emptyFilesLabel}
|
||||
</Text>
|
||||
) : (
|
||||
files.map((file) => (
|
||||
<Checkbox
|
||||
key={file.id}
|
||||
size="xs"
|
||||
label={file.name}
|
||||
checked={isFileSelected(file.id)}
|
||||
onChange={() => onToggleFile(file)}
|
||||
/>
|
||||
))
|
||||
)}
|
||||
</Stack>
|
||||
) : null}
|
||||
</Stack>
|
||||
);
|
||||
}
|
||||
|
||||
export function AttachmentBadge({
|
||||
color,
|
||||
icon,
|
||||
label,
|
||||
onRemove,
|
||||
removeLabel,
|
||||
}: {
|
||||
color: string;
|
||||
icon: React.ReactNode;
|
||||
label: string;
|
||||
onRemove: () => void;
|
||||
removeLabel: string;
|
||||
}) {
|
||||
return (
|
||||
<Badge
|
||||
variant="light"
|
||||
color={color}
|
||||
leftSection={icon}
|
||||
rightSection={<CloseButton size="xs" onClick={onRemove} aria-label={removeLabel} />}
|
||||
className={classes.attachmentBadge}
|
||||
>
|
||||
{label}
|
||||
</Badge>
|
||||
);
|
||||
}
|
||||
|
||||
// The "attach" popover: a root menu of sources and the per-source sub-views
|
||||
// (web URL input, file / note / knowledge / chat pickers) used to ground a reply.
|
||||
export function AttachPopover({
|
||||
attachments,
|
||||
chats,
|
||||
integrationId,
|
||||
onUploadClick,
|
||||
onCapture,
|
||||
}: {
|
||||
attachments: ReturnType<typeof useAttachments>;
|
||||
chats: { id: string; title?: string | null }[];
|
||||
integrationId: string | undefined;
|
||||
onUploadClick: () => void;
|
||||
onCapture: () => void;
|
||||
}) {
|
||||
const t = useI18n();
|
||||
|
||||
const attachViewTitles: Record<Exclude<AttachView, "root">, string> = {
|
||||
webpage: t("widget.openWebUi.attachWebpage"),
|
||||
files: t("widget.openWebUi.attachFiles"),
|
||||
notes: t("widget.openWebUi.attachNotes"),
|
||||
knowledge: t("widget.openWebUi.attachKnowledge"),
|
||||
chats: t("widget.openWebUi.referenceChats"),
|
||||
};
|
||||
const attachViewTitle = attachments.attachView === "root" ? "" : attachViewTitles[attachments.attachView];
|
||||
|
||||
return (
|
||||
<Popover
|
||||
opened={attachments.attachOpened}
|
||||
onChange={attachments.openAttach}
|
||||
position="top-start"
|
||||
withinPortal={false}
|
||||
width={280}
|
||||
shadow="md"
|
||||
>
|
||||
<Popover.Target>
|
||||
<Tooltip label={t("widget.openWebUi.attach")} withinPortal={false}>
|
||||
<ActionIcon
|
||||
variant="subtle"
|
||||
color="gray"
|
||||
loading={attachments.isUploading}
|
||||
onClick={() => attachments.openAttach(!attachments.attachOpened)}
|
||||
aria-label={t("widget.openWebUi.attach")}
|
||||
>
|
||||
<IconPaperclip size={18} />
|
||||
</ActionIcon>
|
||||
</Tooltip>
|
||||
</Popover.Target>
|
||||
<Popover.Dropdown p={4}>
|
||||
{attachments.attachView === "root" ? (
|
||||
<Stack gap={2}>
|
||||
<MenuRow
|
||||
icon={<IconPaperclip size={16} />}
|
||||
label={t("widget.openWebUi.uploadFiles")}
|
||||
onClick={onUploadClick}
|
||||
/>
|
||||
<MenuRow icon={<IconCamera size={16} />} label={t("widget.openWebUi.capture")} onClick={onCapture} />
|
||||
<MenuRow
|
||||
icon={<IconWorld size={16} />}
|
||||
label={t("widget.openWebUi.attachWebpage")}
|
||||
hasSub
|
||||
onClick={() => attachments.setAttachView("webpage")}
|
||||
/>
|
||||
<MenuRow
|
||||
icon={<IconFile size={16} />}
|
||||
label={t("widget.openWebUi.attachFiles")}
|
||||
hasSub
|
||||
onClick={() => attachments.setAttachView("files")}
|
||||
/>
|
||||
<MenuRow
|
||||
icon={<IconFileText size={16} />}
|
||||
label={t("widget.openWebUi.attachNotes")}
|
||||
hasSub
|
||||
onClick={() => attachments.setAttachView("notes")}
|
||||
/>
|
||||
<MenuRow
|
||||
icon={<IconDatabase size={16} />}
|
||||
label={t("widget.openWebUi.attachKnowledge")}
|
||||
hasSub
|
||||
onClick={() => attachments.setAttachView("knowledge")}
|
||||
/>
|
||||
<MenuRow
|
||||
icon={<IconMessage size={16} />}
|
||||
label={t("widget.openWebUi.referenceChats")}
|
||||
hasSub
|
||||
onClick={() => attachments.setAttachView("chats")}
|
||||
/>
|
||||
</Stack>
|
||||
) : (
|
||||
<Stack gap={4}>
|
||||
<Group gap={6} wrap="nowrap" px={4} pt={2}>
|
||||
<ActionIcon
|
||||
variant="subtle"
|
||||
color="gray"
|
||||
size="sm"
|
||||
onClick={() => attachments.setAttachView("root")}
|
||||
aria-label={t("widget.openWebUi.back")}
|
||||
>
|
||||
<IconArrowLeft size={16} />
|
||||
</ActionIcon>
|
||||
<Text fw={600} size="sm">
|
||||
{attachViewTitle}
|
||||
</Text>
|
||||
</Group>
|
||||
|
||||
{attachments.attachView === "webpage" ? (
|
||||
<Group gap={4} wrap="nowrap" p={4}>
|
||||
<TextInput
|
||||
size="xs"
|
||||
flex={1}
|
||||
placeholder="https://…"
|
||||
value={attachments.webInput}
|
||||
onChange={(event) => attachments.setWebInput(event.currentTarget.value)}
|
||||
onKeyDown={(event) => {
|
||||
if (event.key === "Enter") {
|
||||
event.preventDefault();
|
||||
void attachments.addWebUrl();
|
||||
}
|
||||
}}
|
||||
/>
|
||||
<ActionIcon
|
||||
variant="light"
|
||||
color="gray"
|
||||
loading={attachments.isAddingWeb}
|
||||
onClick={() => void attachments.addWebUrl()}
|
||||
aria-label={t("widget.openWebUi.addUrl")}
|
||||
>
|
||||
<IconPlus size={16} />
|
||||
</ActionIcon>
|
||||
</Group>
|
||||
) : null}
|
||||
|
||||
{attachments.attachView === "files" ? (
|
||||
<PickerList emptyLabel={t("widget.openWebUi.noFiles")} count={attachments.files.length}>
|
||||
{attachments.files.map((file) => (
|
||||
<Checkbox
|
||||
key={file.id}
|
||||
size="xs"
|
||||
label={file.name}
|
||||
checked={attachments.fileItems.some((item) => item.id === file.id)}
|
||||
onChange={() => attachments.toggleFile(file)}
|
||||
/>
|
||||
))}
|
||||
</PickerList>
|
||||
) : null}
|
||||
|
||||
{attachments.attachView === "notes" ? (
|
||||
<PickerList emptyLabel={t("widget.openWebUi.noNotes")} count={attachments.notes.length}>
|
||||
{attachments.notes.map((note) => (
|
||||
<Checkbox
|
||||
key={note.id}
|
||||
size="xs"
|
||||
label={note.title}
|
||||
checked={attachments.noteItems.some((item) => item.id === note.id)}
|
||||
onChange={() => void attachments.toggleNote({ id: note.id, title: note.title })}
|
||||
/>
|
||||
))}
|
||||
</PickerList>
|
||||
) : null}
|
||||
|
||||
{attachments.attachView === "knowledge" ? (
|
||||
<PickerList emptyLabel={t("widget.openWebUi.noKnowledge")} count={attachments.knowledge.length}>
|
||||
{attachments.knowledge.map((item) => (
|
||||
<KnowledgeRow
|
||||
key={item.id}
|
||||
integrationId={integrationId ?? ""}
|
||||
knowledgeId={item.id}
|
||||
name={item.name ?? item.id}
|
||||
selected={attachments.knowledgeIds.includes(item.id)}
|
||||
onToggle={() => attachments.toggleKnowledge(item.id)}
|
||||
isFileSelected={(id) => attachments.fileItems.some((file) => file.id === id)}
|
||||
onToggleFile={attachments.toggleFile}
|
||||
emptyFilesLabel={t("widget.openWebUi.noFiles")}
|
||||
/>
|
||||
))}
|
||||
</PickerList>
|
||||
) : null}
|
||||
|
||||
{attachments.attachView === "chats" ? (
|
||||
<PickerList emptyLabel={t("widget.openWebUi.noChats")} count={chats.length}>
|
||||
{chats.map((chat) => (
|
||||
<Checkbox
|
||||
key={chat.id}
|
||||
size="xs"
|
||||
label={chat.title ?? chat.id}
|
||||
checked={attachments.chatItems.some((item) => item.id === chat.id)}
|
||||
onChange={() => void attachments.toggleChatReference(chat)}
|
||||
/>
|
||||
))}
|
||||
</PickerList>
|
||||
) : null}
|
||||
</Stack>
|
||||
)}
|
||||
</Popover.Dropdown>
|
||||
</Popover>
|
||||
);
|
||||
}
|
||||
|
||||
// Live microphone waveform driven by a Web Audio analyser node.
|
||||
export function RecordingWaveform({ analyser }: { analyser: AnalyserNode | null }) {
|
||||
const canvasRef = useRef<HTMLCanvasElement>(null);
|
||||
|
||||
useEffect(() => {
|
||||
const canvas = canvasRef.current;
|
||||
if (!analyser || !canvas) return;
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) return;
|
||||
|
||||
const data = new Uint8Array(analyser.frequencyBinCount);
|
||||
const color = getComputedStyle(canvas).color;
|
||||
let raf = 0;
|
||||
|
||||
const render = () => {
|
||||
raf = requestAnimationFrame(render);
|
||||
analyser.getByteFrequencyData(data);
|
||||
const { width, height } = canvas;
|
||||
ctx.clearRect(0, 0, width, height);
|
||||
ctx.fillStyle = color;
|
||||
const barCount = 56;
|
||||
const step = Math.max(1, Math.floor(data.length / barCount));
|
||||
const slot = width / barCount;
|
||||
for (let index = 0; index < barCount; index++) {
|
||||
const value = (data[index * step] ?? 0) / 255;
|
||||
const barHeight = Math.max(2, value * height);
|
||||
ctx.fillRect(index * slot + slot * 0.3, (height - barHeight) / 2, slot * 0.4, barHeight);
|
||||
}
|
||||
};
|
||||
|
||||
render();
|
||||
return () => cancelAnimationFrame(raf);
|
||||
}, [analyser]);
|
||||
|
||||
return <canvas ref={canvasRef} width={640} height={32} aria-hidden className={classes.waveformCanvas} />;
|
||||
}
|
||||
|
||||
function TypingDots() {
|
||||
return (
|
||||
<span className={classes.typing} aria-label="…">
|
||||
<span className={classes.typingDot} />
|
||||
<span className={classes.typingDot} />
|
||||
<span className={classes.typingDot} />
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
export function MessageBubble({
|
||||
from,
|
||||
content,
|
||||
images,
|
||||
loading,
|
||||
}: {
|
||||
from: ChatMessage["role"];
|
||||
content: string;
|
||||
images?: string[];
|
||||
loading?: boolean;
|
||||
}) {
|
||||
const isUser = from === "user";
|
||||
const avatar = (
|
||||
<Avatar size={22} radius="xl" variant="filled" color={isUser ? "gray" : "blue"} className={classes.avatar}>
|
||||
{isUser ? <IconUser size={14} /> : <IconRobot size={14} />}
|
||||
</Avatar>
|
||||
);
|
||||
return (
|
||||
<Group gap={6} align="flex-start" wrap="nowrap" justify={isUser ? "flex-end" : "flex-start"}>
|
||||
{isUser ? null : avatar}
|
||||
<Paper
|
||||
className={classes.bubble}
|
||||
withBorder={false}
|
||||
bg={isUser ? "var(--mantine-primary-color-light)" : "var(--mantine-color-default-hover)"}
|
||||
p="xs"
|
||||
radius="md"
|
||||
>
|
||||
{images && images.length > 0 ? (
|
||||
<Group gap={4} mb={content ? 6 : 0}>
|
||||
{images.map((src, index) => (
|
||||
<Image key={index} src={src} alt="" w={120} radius="sm" fit="contain" />
|
||||
))}
|
||||
</Group>
|
||||
) : null}
|
||||
{loading && !content ? <TypingDots /> : null}
|
||||
{content ? (
|
||||
isUser ? (
|
||||
<Text size="sm" style={{ whiteSpace: "pre-wrap", wordBreak: "break-word" }}>
|
||||
{content}
|
||||
</Text>
|
||||
) : (
|
||||
<div className={classes.markdown}>
|
||||
<ReactMarkdown skipHtml>{content}</ReactMarkdown>
|
||||
</div>
|
||||
)
|
||||
) : null}
|
||||
</Paper>
|
||||
{isUser ? avatar : null}
|
||||
</Group>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
export interface ChatMessage {
|
||||
role: "system" | "user" | "assistant";
|
||||
content: string;
|
||||
images?: string[];
|
||||
}
|
||||
|
||||
export type CompletionPart = { type: "text"; text: string } | { type: "image_url"; image_url: { url: string } };
|
||||
export type CompletionMessage = { role: ChatMessage["role"]; content: string | CompletionPart[] };
|
||||
|
||||
export interface ImageAttachment {
|
||||
name: string;
|
||||
dataUrl: string;
|
||||
}
|
||||
export interface WebAttachment {
|
||||
url: string;
|
||||
collectionName: string;
|
||||
title: string;
|
||||
}
|
||||
export interface FileAttachment {
|
||||
id: string;
|
||||
name: string;
|
||||
}
|
||||
export interface NoteAttachment {
|
||||
id: string;
|
||||
title: string;
|
||||
content: string;
|
||||
}
|
||||
export interface ChatAttachment {
|
||||
id: string;
|
||||
title: string;
|
||||
transcript: string;
|
||||
}
|
||||
|
||||
// Which sub-view the attach popover is showing.
|
||||
export type AttachView = "root" | "webpage" | "files" | "notes" | "knowledge" | "chats";
|
||||
|
||||
// Retrieval collections + verbatim context blocks injected before a completion.
|
||||
export interface Grounding {
|
||||
collections: string[];
|
||||
contextTexts: string[];
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
import type { ChatMessage, CompletionMessage } from "./chat-types";
|
||||
|
||||
// Heuristic fallback for detecting vision-capable models when Open WebUI does
|
||||
// not declare the capability explicitly.
|
||||
export const VISION_MODEL_PATTERN =
|
||||
/vision|llava|bakllava|moondream|minicpm-?v|qwen2?-?vl|pixtral|cogvlm|internvl|gemma3|smolvlm|granite[\w-]*vision/i;
|
||||
|
||||
export const readBlobAsDataUrl = (blob: Blob): Promise<string> =>
|
||||
new Promise((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
reader.addEventListener("load", () => resolve(reader.result as string));
|
||||
reader.addEventListener("error", () => reject(reader.error ?? new Error("Failed to read file")));
|
||||
reader.readAsDataURL(blob);
|
||||
});
|
||||
|
||||
export const hostnameOf = (url: string): string => {
|
||||
try {
|
||||
return new URL(url).hostname || url;
|
||||
} catch {
|
||||
return url;
|
||||
}
|
||||
};
|
||||
|
||||
export const formatSeconds = (total: number): string => {
|
||||
const minutes = Math.floor(total / 60);
|
||||
const seconds = total % 60;
|
||||
return `${minutes}:${String(seconds).padStart(2, "0")}`;
|
||||
};
|
||||
|
||||
// Convert displayed messages to the OpenAI completion format, inlining any
|
||||
// attached images on the message that carries them (vision content parts).
|
||||
export const toCompletionMessages = (messages: ChatMessage[]): CompletionMessage[] =>
|
||||
messages.map((message) =>
|
||||
message.images && message.images.length > 0
|
||||
? {
|
||||
role: message.role,
|
||||
content: [
|
||||
...(message.content ? [{ type: "text" as const, text: message.content }] : []),
|
||||
...message.images.map((url) => ({ type: "image_url" as const, image_url: { url } })),
|
||||
],
|
||||
}
|
||||
: { role: message.role, content: message.content },
|
||||
);
|
||||
|
||||
type RawMessage = { role?: string | null; content?: string | null };
|
||||
type HistoryMessage = RawMessage & { parentId?: string | null };
|
||||
interface RawChat {
|
||||
messages?: RawMessage[] | null;
|
||||
history?: { currentId?: string | null; messages?: Record<string, HistoryMessage> | null } | null;
|
||||
}
|
||||
|
||||
const toChatMessage = (role: string | null | undefined, content: string | null | undefined): ChatMessage | null =>
|
||||
(role === "system" || role === "user" || role === "assistant") && typeof content === "string"
|
||||
? { role, content }
|
||||
: null;
|
||||
|
||||
/**
|
||||
* Reconstructs the ordered conversation from an Open WebUI chat. Chats created in
|
||||
* the Open WebUI UI store the active branch as a tree under `chat.history`
|
||||
* (walk from `currentId` up through `parentId`); `chat.messages` can be stale, so
|
||||
* the history tree takes precedence and we fall back to the flat list.
|
||||
*/
|
||||
export const extractChatMessages = (chat: RawChat | null | undefined): ChatMessage[] => {
|
||||
const history = chat?.history;
|
||||
if (history?.messages && history.currentId) {
|
||||
const map = history.messages;
|
||||
const ordered: ChatMessage[] = [];
|
||||
const seen = new Set<string>();
|
||||
let cursor: string | null | undefined = history.currentId;
|
||||
while (cursor && !seen.has(cursor)) {
|
||||
seen.add(cursor);
|
||||
const node: HistoryMessage | undefined = map[cursor];
|
||||
if (!node) break;
|
||||
const message = toChatMessage(node.role, node.content);
|
||||
if (message) ordered.unshift(message);
|
||||
cursor = node.parentId;
|
||||
}
|
||||
if (ordered.length > 0) return ordered;
|
||||
}
|
||||
return (chat?.messages ?? [])
|
||||
.map((message) => toChatMessage(message.role, message.content))
|
||||
.filter((message): message is ChatMessage => message !== null);
|
||||
};
|
||||
@@ -29,6 +29,45 @@
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
|
||||
/* Markdown-rendered assistant replies: keep block spacing tight inside the bubble. */
|
||||
.markdown {
|
||||
font-size: var(--mantine-font-size-sm);
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
.markdown > :first-child {
|
||||
margin-top: 0;
|
||||
}
|
||||
|
||||
.markdown > :last-child {
|
||||
margin-bottom: 0;
|
||||
}
|
||||
|
||||
.markdown p,
|
||||
.markdown ul,
|
||||
.markdown ol,
|
||||
.markdown pre {
|
||||
margin-block: var(--mantine-spacing-xs);
|
||||
}
|
||||
|
||||
.markdown pre {
|
||||
overflow-x: auto;
|
||||
padding: var(--mantine-spacing-xs);
|
||||
border-radius: var(--mantine-radius-sm);
|
||||
background-color: var(--mantine-color-default);
|
||||
}
|
||||
|
||||
.markdown code {
|
||||
font-family: var(--mantine-font-family-monospace);
|
||||
font-size: 0.85em;
|
||||
}
|
||||
|
||||
.markdown :not(pre) > code {
|
||||
padding: 0.1em 0.3em;
|
||||
border-radius: var(--mantine-radius-sm);
|
||||
background-color: var(--mantine-color-default);
|
||||
}
|
||||
|
||||
.avatar {
|
||||
flex-shrink: 0;
|
||||
margin-top: 2px;
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,225 @@
|
||||
import { useState } from "react";
|
||||
|
||||
import { clientApi } from "@homarr/api/client";
|
||||
|
||||
import type {
|
||||
AttachView,
|
||||
ChatAttachment,
|
||||
FileAttachment,
|
||||
Grounding,
|
||||
ImageAttachment,
|
||||
NoteAttachment,
|
||||
WebAttachment,
|
||||
} from "./chat-types";
|
||||
import { extractChatMessages, readBlobAsDataUrl } from "./chat-utils";
|
||||
|
||||
interface UseAttachmentsOptions {
|
||||
integrationId: string | undefined;
|
||||
onError: (message: string) => void;
|
||||
errorMessages: { upload: string; web: string; note: string; chat: string };
|
||||
}
|
||||
|
||||
/**
|
||||
* Owns every "ground the conversation" attachment source: inline vision images,
|
||||
* uploaded files, ingested web pages, knowledge bases, notes and referenced
|
||||
* chats — including the pickers' data, the toggle handlers, and assembling the
|
||||
* retrieval collections / verbatim context blocks sent with a completion.
|
||||
*/
|
||||
export function useAttachments({ integrationId, onError, errorMessages }: UseAttachmentsOptions) {
|
||||
// Images are inline per-message (vision); the rest are sticky context kept for
|
||||
// the conversation until removed.
|
||||
const [images, setImages] = useState<ImageAttachment[]>([]);
|
||||
const [knowledgeIds, setKnowledgeIds] = useState<string[]>([]);
|
||||
const [webItems, setWebItems] = useState<WebAttachment[]>([]);
|
||||
const [fileItems, setFileItems] = useState<FileAttachment[]>([]);
|
||||
const [noteItems, setNoteItems] = useState<NoteAttachment[]>([]);
|
||||
const [chatItems, setChatItems] = useState<ChatAttachment[]>([]);
|
||||
const [webInput, setWebInput] = useState("");
|
||||
const [attachOpened, setAttachOpened] = useState(false);
|
||||
const [attachView, setAttachView] = useState<AttachView>("root");
|
||||
|
||||
const utils = clientApi.useUtils();
|
||||
const processWeb = clientApi.widget.openWebUi.processWeb.useMutation();
|
||||
const uploadFile = clientApi.widget.openWebUi.uploadFile.useMutation();
|
||||
|
||||
const pickerQueryOptions = {
|
||||
enabled: Boolean(integrationId) && attachOpened,
|
||||
refetchOnWindowFocus: false,
|
||||
retry: false,
|
||||
} as const;
|
||||
const { data: knowledge = [] } = clientApi.widget.openWebUi.getKnowledge.useQuery(
|
||||
{ integrationId: integrationId ?? "" },
|
||||
pickerQueryOptions,
|
||||
);
|
||||
const { data: files = [] } = clientApi.widget.openWebUi.getFiles.useQuery(
|
||||
{ integrationId: integrationId ?? "" },
|
||||
pickerQueryOptions,
|
||||
);
|
||||
const { data: notes = [] } = clientApi.widget.openWebUi.getNotes.useQuery(
|
||||
{ integrationId: integrationId ?? "" },
|
||||
pickerQueryOptions,
|
||||
);
|
||||
|
||||
// Reset the popover to the root view whenever it closes.
|
||||
const openAttach = (open: boolean) => {
|
||||
setAttachOpened(open);
|
||||
if (!open) setAttachView("root");
|
||||
};
|
||||
|
||||
// Upload Files: images become inline vision attachments; everything else is
|
||||
// uploaded to Open WebUI and grounded via retrieval.
|
||||
const handleFilesSelected = async (fileList: FileList | null) => {
|
||||
if (!fileList || !integrationId) return;
|
||||
for (const file of Array.from(fileList)) {
|
||||
try {
|
||||
const dataUrl = await readBlobAsDataUrl(file);
|
||||
if (file.type.startsWith("image/")) {
|
||||
setImages((previous) => [...previous, { name: file.name, dataUrl }]);
|
||||
} else {
|
||||
const contentBase64 = dataUrl.split(",")[1] ?? "";
|
||||
const uploaded = await uploadFile.mutateAsync({
|
||||
integrationId,
|
||||
filename: file.name,
|
||||
contentBase64,
|
||||
contentType: file.type || "application/octet-stream",
|
||||
});
|
||||
setFileItems((previous) =>
|
||||
previous.some((item) => item.id === uploaded.id) ? previous : [...previous, uploaded],
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
onError(errorMessages.upload);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const addWebUrl = async () => {
|
||||
if (!integrationId) return;
|
||||
const raw = webInput.trim();
|
||||
if (!raw) return;
|
||||
const url = /^https?:\/\//i.test(raw) ? raw : `https://${raw}`;
|
||||
setWebInput("");
|
||||
try {
|
||||
const document = await processWeb.mutateAsync({ integrationId, url });
|
||||
setWebItems((previous) =>
|
||||
previous.some((item) => item.collectionName === document.collectionName)
|
||||
? previous
|
||||
: [...previous, { url, collectionName: document.collectionName, title: document.title }],
|
||||
);
|
||||
openAttach(false);
|
||||
} catch {
|
||||
onError(errorMessages.web);
|
||||
}
|
||||
};
|
||||
|
||||
const toggleFile = (file: FileAttachment) =>
|
||||
setFileItems((previous) =>
|
||||
previous.some((item) => item.id === file.id)
|
||||
? previous.filter((item) => item.id !== file.id)
|
||||
: [...previous, file],
|
||||
);
|
||||
|
||||
// The list endpoint truncates note bodies, so fetch the full note on attach.
|
||||
const toggleNote = async (note: { id: string; title: string }) => {
|
||||
if (!integrationId) return;
|
||||
if (noteItems.some((item) => item.id === note.id)) {
|
||||
setNoteItems((previous) => previous.filter((item) => item.id !== note.id));
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const full = await utils.widget.openWebUi.getNote.fetch({ integrationId, noteId: note.id });
|
||||
setNoteItems((previous) => [...previous, { id: full.id, title: full.title, content: full.content }]);
|
||||
} catch {
|
||||
onError(errorMessages.note);
|
||||
}
|
||||
};
|
||||
|
||||
const toggleKnowledge = (id: string) =>
|
||||
setKnowledgeIds((previous) =>
|
||||
previous.includes(id) ? previous.filter((value) => value !== id) : [...previous, id],
|
||||
);
|
||||
|
||||
const toggleChatReference = async (chat: { id: string; title?: string | null }) => {
|
||||
if (!integrationId) return;
|
||||
if (chatItems.some((item) => item.id === chat.id)) {
|
||||
setChatItems((previous) => previous.filter((item) => item.id !== chat.id));
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const full = await utils.widget.openWebUi.getChat.fetch({ integrationId, chatId: chat.id });
|
||||
const transcript = extractChatMessages(full.chat)
|
||||
.map((message) => `${message.role}: ${message.content}`)
|
||||
.join("\n");
|
||||
setChatItems((previous) => [...previous, { id: chat.id, title: chat.title ?? chat.id, transcript }]);
|
||||
} catch {
|
||||
onError(errorMessages.chat);
|
||||
}
|
||||
};
|
||||
|
||||
const buildGrounding = (): Grounding => ({
|
||||
// Retrieval collections: knowledge bases (id), web pages and files.
|
||||
collections: [
|
||||
...knowledgeIds,
|
||||
...webItems.map((item) => item.collectionName),
|
||||
...fileItems.map((item) => `file-${item.id}`),
|
||||
],
|
||||
// Verbatim context: notes and referenced chats.
|
||||
contextTexts: [
|
||||
...noteItems.map((note) => `Note "${note.title}":\n${note.content}`),
|
||||
...chatItems.map((chat) => `Previous chat "${chat.title}":\n${chat.transcript}`),
|
||||
],
|
||||
});
|
||||
|
||||
const reset = () => {
|
||||
setImages([]);
|
||||
setKnowledgeIds([]);
|
||||
setWebItems([]);
|
||||
setFileItems([]);
|
||||
setNoteItems([]);
|
||||
setChatItems([]);
|
||||
};
|
||||
|
||||
const hasAttachments =
|
||||
images.length > 0 ||
|
||||
knowledgeIds.length > 0 ||
|
||||
webItems.length > 0 ||
|
||||
fileItems.length > 0 ||
|
||||
noteItems.length > 0 ||
|
||||
chatItems.length > 0;
|
||||
|
||||
return {
|
||||
// state
|
||||
images,
|
||||
setImages,
|
||||
knowledgeIds,
|
||||
webItems,
|
||||
setWebItems,
|
||||
fileItems,
|
||||
noteItems,
|
||||
chatItems,
|
||||
setChatItems,
|
||||
webInput,
|
||||
setWebInput,
|
||||
attachOpened,
|
||||
attachView,
|
||||
setAttachView,
|
||||
hasAttachments,
|
||||
// picker data
|
||||
knowledge,
|
||||
files,
|
||||
notes,
|
||||
// status
|
||||
isUploading: processWeb.isPending || uploadFile.isPending,
|
||||
isAddingWeb: processWeb.isPending,
|
||||
// handlers
|
||||
openAttach,
|
||||
handleFilesSelected,
|
||||
addWebUrl,
|
||||
toggleFile,
|
||||
toggleNote,
|
||||
toggleKnowledge,
|
||||
toggleChatReference,
|
||||
buildGrounding,
|
||||
reset,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
|
||||
import type { ImageAttachment } from "./chat-types";
|
||||
|
||||
interface UseCameraCaptureOptions {
|
||||
onError: (message: string) => void;
|
||||
onCapture: (attachment: ImageAttachment) => void;
|
||||
captureErrorMessage: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Owns the webcam capture lifecycle: acquiring the stream, attaching it to the
|
||||
* <video> the moment it mounts, taking a still frame, and tearing the stream
|
||||
* down (including on unmount).
|
||||
*/
|
||||
export function useCameraCapture({ onError, onCapture, captureErrorMessage }: UseCameraCaptureOptions) {
|
||||
const [captureOpened, setCaptureOpened] = useState(false);
|
||||
const captureStreamRef = useRef<MediaStream | null>(null);
|
||||
const videoRef = useRef<HTMLVideoElement | null>(null);
|
||||
|
||||
const stopCaptureStream = () => {
|
||||
captureStreamRef.current?.getTracks().forEach((track) => track.stop());
|
||||
captureStreamRef.current = null;
|
||||
};
|
||||
|
||||
const openCapture = async () => {
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ video: true });
|
||||
captureStreamRef.current = stream;
|
||||
setCaptureOpened(true);
|
||||
} catch {
|
||||
onError(captureErrorMessage);
|
||||
}
|
||||
};
|
||||
|
||||
const closeCapture = () => {
|
||||
stopCaptureStream();
|
||||
setCaptureOpened(false);
|
||||
};
|
||||
|
||||
// Attach the live stream the moment the <video> mounts (avoids a ref race
|
||||
// where an effect runs before the modal's video element exists).
|
||||
const setVideoNode = useCallback((node: HTMLVideoElement | null) => {
|
||||
videoRef.current = node;
|
||||
if (node && captureStreamRef.current) {
|
||||
node.srcObject = captureStreamRef.current;
|
||||
void node.play().catch(() => undefined);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const takePhoto = () => {
|
||||
const video = videoRef.current;
|
||||
if (!video) return;
|
||||
// Bail if the stream hasn't delivered a frame yet, otherwise we'd capture a
|
||||
// 0x0 (blank) canvas and attach an empty image.
|
||||
if (video.readyState < 2 || video.videoWidth === 0 || video.videoHeight === 0) return;
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = video.videoWidth;
|
||||
canvas.height = video.videoHeight;
|
||||
canvas.getContext("2d")?.drawImage(video, 0, 0);
|
||||
onCapture({ name: `capture-${Date.now()}.png`, dataUrl: canvas.toDataURL("image/png") });
|
||||
closeCapture();
|
||||
};
|
||||
|
||||
// Stop the camera stream on unmount.
|
||||
useEffect(() => () => stopCaptureStream(), []);
|
||||
|
||||
return { captureOpened, openCapture, closeCapture, setVideoNode, takePhoto };
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
|
||||
import { clientApi } from "@homarr/api/client";
|
||||
|
||||
import type { ChatMessage, CompletionMessage, Grounding } from "./chat-types";
|
||||
import { extractChatMessages, toCompletionMessages } from "./chat-utils";
|
||||
|
||||
interface UseChatStreamOptions {
|
||||
integrationId: string | undefined;
|
||||
model: string | null;
|
||||
setModel: (model: string) => void;
|
||||
systemPrompt: string;
|
||||
buildGrounding: () => Grounding;
|
||||
setError: (message: string | null) => void;
|
||||
onChatLoaded: () => void;
|
||||
newChatTitle: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Owns the conversation itself: the streamed completion subscription, the
|
||||
* in-memory message list, persistence to Open WebUI's native history, and the
|
||||
* send / retry / stop / new-chat transitions.
|
||||
*/
|
||||
export function useChatStream({
|
||||
integrationId,
|
||||
model,
|
||||
setModel,
|
||||
systemPrompt,
|
||||
buildGrounding,
|
||||
setError,
|
||||
onChatLoaded,
|
||||
newChatTitle,
|
||||
}: UseChatStreamOptions) {
|
||||
const [messages, setMessages] = useState<ChatMessage[]>([]);
|
||||
const [streamingText, setStreamingText] = useState("");
|
||||
const [isStreaming, setIsStreaming] = useState(false);
|
||||
const [pendingMessages, setPendingMessages] = useState<CompletionMessage[]>([]);
|
||||
const [pendingCollections, setPendingCollections] = useState<string[]>([]);
|
||||
const [pendingContextTexts, setPendingContextTexts] = useState<string[]>([]);
|
||||
const [activeChatId, setActiveChatId] = useState<string | null>(null);
|
||||
const [selectedChatId, setSelectedChatId] = useState<string | null>(null);
|
||||
|
||||
const messagesRef = useRef<ChatMessage[]>([]);
|
||||
const streamingTextRef = useRef("");
|
||||
// Read inside the serialized persist loop so a burst of turns reuses the chat
|
||||
// created by the first write instead of racing on the not-yet-committed state.
|
||||
const activeChatIdRef = useRef<string | null>(null);
|
||||
const persistInFlightRef = useRef(false);
|
||||
const pendingPersistRef = useRef<ChatMessage[] | null>(null);
|
||||
|
||||
const utils = clientApi.useUtils();
|
||||
const createChat = clientApi.widget.openWebUi.createChat.useMutation();
|
||||
const updateChat = clientApi.widget.openWebUi.updateChat.useMutation();
|
||||
|
||||
const { data: loadedChat } = clientApi.widget.openWebUi.getChat.useQuery(
|
||||
{ integrationId: integrationId ?? "", chatId: selectedChatId ?? "" },
|
||||
{ enabled: Boolean(integrationId) && Boolean(selectedChatId) },
|
||||
);
|
||||
|
||||
// Hydrate the conversation when a history chat finishes loading.
|
||||
useEffect(() => {
|
||||
if (!loadedChat) return;
|
||||
const loadedMessages = extractChatMessages(loadedChat.chat);
|
||||
setMessages(loadedMessages);
|
||||
messagesRef.current = loadedMessages;
|
||||
setActiveChatId(loadedChat.id);
|
||||
activeChatIdRef.current = loadedChat.id;
|
||||
if (loadedChat.chat?.models?.[0]) setModel(loadedChat.chat.models[0]);
|
||||
onChatLoaded();
|
||||
}, [loadedChat, setModel, onChatLoaded]);
|
||||
|
||||
const runPersist = async (finalMessages: ChatMessage[]) => {
|
||||
if (!integrationId || !model) return;
|
||||
// Native history stores plain text; image data URLs are dropped on persist.
|
||||
const title = finalMessages.find((message) => message.role === "user")?.content.slice(0, 80) ?? newChatTitle;
|
||||
const payload = {
|
||||
title,
|
||||
models: [model],
|
||||
messages: finalMessages.map(({ role, content }) => ({ role, content })),
|
||||
};
|
||||
try {
|
||||
const chatId = activeChatIdRef.current;
|
||||
if (chatId) {
|
||||
await updateChat.mutateAsync({ integrationId, chatId, chat: payload });
|
||||
} else {
|
||||
const chat = await createChat.mutateAsync({ integrationId, ...payload });
|
||||
activeChatIdRef.current = chat.id;
|
||||
setActiveChatId(chat.id);
|
||||
}
|
||||
await utils.widget.openWebUi.getChats.invalidate({ integrationId });
|
||||
} catch {
|
||||
// Persistence is best-effort; the conversation remains usable in-memory.
|
||||
}
|
||||
};
|
||||
|
||||
// Open WebUI's chat endpoint replaces the whole chat object, so each turn
|
||||
// re-sends the full transcript. Serialize the writes and collapse a burst into
|
||||
// a single trailing save of the latest messages so turns don't stack
|
||||
// concurrent POSTs (or create duplicate chats on the first save).
|
||||
const persistChat = async (finalMessages: ChatMessage[]) => {
|
||||
if (persistInFlightRef.current) {
|
||||
pendingPersistRef.current = finalMessages;
|
||||
return;
|
||||
}
|
||||
persistInFlightRef.current = true;
|
||||
try {
|
||||
let next: ChatMessage[] | null = finalMessages;
|
||||
while (next) {
|
||||
pendingPersistRef.current = null;
|
||||
await runPersist(next);
|
||||
next = pendingPersistRef.current;
|
||||
}
|
||||
} finally {
|
||||
persistInFlightRef.current = false;
|
||||
}
|
||||
};
|
||||
|
||||
const clearPending = () => {
|
||||
setPendingMessages([]);
|
||||
setPendingCollections([]);
|
||||
setPendingContextTexts([]);
|
||||
};
|
||||
|
||||
const finalizeAssistantMessage = (stopped = false) => {
|
||||
const finalText = streamingTextRef.current;
|
||||
if (finalText.length > 0) {
|
||||
const finalMessages = [...messagesRef.current, { role: "assistant" as const, content: finalText }];
|
||||
setMessages(finalMessages);
|
||||
messagesRef.current = finalMessages;
|
||||
// A user-stopped reply is kept on screen but not written to history as a
|
||||
// finished turn; a later real turn will persist the whole conversation.
|
||||
if (!stopped) void persistChat(finalMessages);
|
||||
}
|
||||
streamingTextRef.current = "";
|
||||
setStreamingText("");
|
||||
setIsStreaming(false);
|
||||
clearPending();
|
||||
};
|
||||
|
||||
clientApi.widget.openWebUi.sendMessage.useSubscription(
|
||||
{
|
||||
integrationId: integrationId ?? "",
|
||||
model: model ?? "",
|
||||
messages: pendingMessages,
|
||||
collections: pendingCollections.length > 0 ? pendingCollections : undefined,
|
||||
contextTexts: pendingContextTexts.length > 0 ? pendingContextTexts : undefined,
|
||||
},
|
||||
{
|
||||
enabled: isStreaming && pendingMessages.length > 0,
|
||||
onData: (event) => {
|
||||
if (event.type === "delta") {
|
||||
streamingTextRef.current += event.content;
|
||||
setStreamingText(streamingTextRef.current);
|
||||
} else if (event.type === "done") {
|
||||
finalizeAssistantMessage();
|
||||
} else if (event.type === "error") {
|
||||
setError(event.message);
|
||||
finalizeAssistantMessage();
|
||||
}
|
||||
},
|
||||
onError: (err) => {
|
||||
setError(err.message);
|
||||
finalizeAssistantMessage();
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
// Kick off a completion for the given conversation (ending with a user message).
|
||||
// Shared by sending a new message and retrying after a failure.
|
||||
const startCompletion = (baseMessages: ChatMessage[]) => {
|
||||
if (!model || !integrationId) return;
|
||||
setError(null);
|
||||
|
||||
const trimmedSystemPrompt = systemPrompt.trim();
|
||||
const payloadMessages: ChatMessage[] = trimmedSystemPrompt
|
||||
? [{ role: "system", content: trimmedSystemPrompt }, ...baseMessages]
|
||||
: baseMessages;
|
||||
|
||||
const grounding = buildGrounding();
|
||||
|
||||
streamingTextRef.current = "";
|
||||
setStreamingText("");
|
||||
setPendingMessages(toCompletionMessages(payloadMessages));
|
||||
setPendingCollections(grounding.collections);
|
||||
setPendingContextTexts(grounding.contextTexts);
|
||||
setIsStreaming(true);
|
||||
};
|
||||
|
||||
const send = (userMessage: ChatMessage) => {
|
||||
const nextMessages = [...messagesRef.current, userMessage];
|
||||
setMessages(nextMessages);
|
||||
messagesRef.current = nextMessages;
|
||||
startCompletion(nextMessages);
|
||||
};
|
||||
|
||||
// Retry the last turn after a failure: drop any partial assistant reply and
|
||||
// re-run the completion from the most recent user message.
|
||||
const retry = () => {
|
||||
if (isStreaming) return;
|
||||
const lastUserIndex = messagesRef.current.findLastIndex((message) => message.role === "user");
|
||||
if (lastUserIndex === -1) return;
|
||||
|
||||
const truncated = messagesRef.current.slice(0, lastUserIndex + 1);
|
||||
setMessages(truncated);
|
||||
messagesRef.current = truncated;
|
||||
startCompletion(truncated);
|
||||
};
|
||||
|
||||
const reset = () => {
|
||||
setMessages([]);
|
||||
messagesRef.current = [];
|
||||
setActiveChatId(null);
|
||||
activeChatIdRef.current = null;
|
||||
setSelectedChatId(null);
|
||||
streamingTextRef.current = "";
|
||||
setStreamingText("");
|
||||
setIsStreaming(false);
|
||||
clearPending();
|
||||
};
|
||||
|
||||
return {
|
||||
messages,
|
||||
streamingText,
|
||||
isStreaming,
|
||||
activeChatId,
|
||||
selectedChatId,
|
||||
setSelectedChatId,
|
||||
send,
|
||||
retry,
|
||||
stop: () => finalizeAssistantMessage(true),
|
||||
reset,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
|
||||
import { clientApi } from "@homarr/api/client";
|
||||
|
||||
import { readBlobAsDataUrl } from "./chat-utils";
|
||||
|
||||
interface UseVoiceRecordingOptions {
|
||||
integrationId: string | undefined;
|
||||
onError: (message: string) => void;
|
||||
onTranscribed: (text: string) => void;
|
||||
micErrorMessage: string;
|
||||
transcribeErrorMessage: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Owns the microphone recording lifecycle: capturing audio, driving the live
|
||||
* waveform analyser, and transcribing the result to text. Cancel discards the
|
||||
* recording; confirm transcribes it. Streams/timers are torn down on unmount.
|
||||
*/
|
||||
export function useVoiceRecording({
|
||||
integrationId,
|
||||
onError,
|
||||
onTranscribed,
|
||||
micErrorMessage,
|
||||
transcribeErrorMessage,
|
||||
}: UseVoiceRecordingOptions) {
|
||||
const [isRecording, setIsRecording] = useState(false);
|
||||
const [recordingSeconds, setRecordingSeconds] = useState(0);
|
||||
const [analyser, setAnalyser] = useState<AnalyserNode | null>(null);
|
||||
|
||||
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
|
||||
const audioChunksRef = useRef<Blob[]>([]);
|
||||
const audioStreamRef = useRef<MediaStream | null>(null);
|
||||
const audioContextRef = useRef<AudioContext | null>(null);
|
||||
const recordingTimerRef = useRef<number | null>(null);
|
||||
const recordingCancelledRef = useRef(false);
|
||||
|
||||
const transcribe = clientApi.widget.openWebUi.transcribe.useMutation();
|
||||
|
||||
// Stop the mic stream, audio context and timer. No state updates, so it is
|
||||
// safe to call during unmount as well as from teardownRecording.
|
||||
const releaseAudioResources = () => {
|
||||
if (recordingTimerRef.current !== null) {
|
||||
window.clearInterval(recordingTimerRef.current);
|
||||
recordingTimerRef.current = null;
|
||||
}
|
||||
audioStreamRef.current?.getTracks().forEach((track) => track.stop());
|
||||
audioStreamRef.current = null;
|
||||
void audioContextRef.current?.close().catch(() => undefined);
|
||||
audioContextRef.current = null;
|
||||
};
|
||||
|
||||
const teardownRecording = () => {
|
||||
releaseAudioResources();
|
||||
setAnalyser(null);
|
||||
setIsRecording(false);
|
||||
};
|
||||
|
||||
const finishRecording = async () => {
|
||||
const cancelled = recordingCancelledRef.current;
|
||||
const chunks = audioChunksRef.current;
|
||||
audioChunksRef.current = [];
|
||||
teardownRecording();
|
||||
if (cancelled || chunks.length === 0 || !integrationId) return;
|
||||
|
||||
const blob = new Blob(chunks, { type: chunks[0]?.type ?? "audio/webm" });
|
||||
try {
|
||||
const dataUrl = await readBlobAsDataUrl(blob);
|
||||
const result = await transcribe.mutateAsync({
|
||||
integrationId,
|
||||
filename: "recording.webm",
|
||||
contentBase64: dataUrl.split(",")[1] ?? "",
|
||||
contentType: blob.type || "audio/webm",
|
||||
});
|
||||
const text = result.text.trim();
|
||||
if (text) onTranscribed(text);
|
||||
} catch {
|
||||
onError(transcribeErrorMessage);
|
||||
}
|
||||
};
|
||||
|
||||
const startRecording = async () => {
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
audioStreamRef.current = stream;
|
||||
|
||||
// Live audio analyser drives the waveform animation.
|
||||
const audioContext = new AudioContext();
|
||||
const sourceNode = audioContext.createMediaStreamSource(stream);
|
||||
const analyserNode = audioContext.createAnalyser();
|
||||
analyserNode.fftSize = 256;
|
||||
sourceNode.connect(analyserNode);
|
||||
audioContextRef.current = audioContext;
|
||||
setAnalyser(analyserNode);
|
||||
|
||||
const recorder = new MediaRecorder(stream);
|
||||
audioChunksRef.current = [];
|
||||
recordingCancelledRef.current = false;
|
||||
recorder.addEventListener("dataavailable", (event) => {
|
||||
if (event.data.size > 0) audioChunksRef.current.push(event.data);
|
||||
});
|
||||
recorder.addEventListener("stop", () => void finishRecording());
|
||||
mediaRecorderRef.current = recorder;
|
||||
recorder.start();
|
||||
|
||||
setRecordingSeconds(0);
|
||||
recordingTimerRef.current = window.setInterval(() => setRecordingSeconds((value) => value + 1), 1000);
|
||||
setIsRecording(true);
|
||||
} catch {
|
||||
onError(micErrorMessage);
|
||||
}
|
||||
};
|
||||
|
||||
const cancelRecording = () => {
|
||||
recordingCancelledRef.current = true;
|
||||
mediaRecorderRef.current?.stop();
|
||||
};
|
||||
|
||||
const confirmRecording = () => {
|
||||
recordingCancelledRef.current = false;
|
||||
mediaRecorderRef.current?.stop();
|
||||
};
|
||||
|
||||
// Tear down the microphone stream and timer on unmount.
|
||||
useEffect(() => () => releaseAudioResources(), []);
|
||||
|
||||
return {
|
||||
isRecording,
|
||||
recordingSeconds,
|
||||
analyser,
|
||||
isTranscribing: transcribe.isPending,
|
||||
startRecording,
|
||||
cancelRecording,
|
||||
confirmRecording,
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user