fix: harden assistant authoring and UI

This commit is contained in:
ajnart authored and Thomas Camlong committed 2026-08-09 18:09:11 +02:00
1 parent fa0e55cc93
commit edc7b4d55b
51 files changed
+4118 -916

No files matched your search

@@ -0,0 +1,102 @@
import type { UIMessage } from "ai";
import { describe, expect, test } from "vitest";
import { prepareAssistantMessagesForTransport, prepareAssistantRequestBody } from "./assistant-attachment-payload";
const routeRequestBudget = 12_000_000;
// A valid padded base64 payload that decodes to exactly the composer's 1,000,000-byte image limit.
const oneMegabyteBase64 = `${"A".repeat(1_333_334)}==`;
const createAttachmentTurn = (id: string, payloadMarker: string): UIMessage => ({
id,
role: "user",
parts: [
{ type: "text", text: `Review attachments from ${id}` },
...Array.from({ length: 5 }, (_, index) => ({
type: "file" as const,
filename: `${id}-${index}.png`,
mediaType: "image/png",
url: `data:image/png;base64,${payloadMarker}${oneMegabyteBase64.slice(payloadMarker.length)}`,
})),
],
});
describe("assistant attachment payload", () => {
test("keeps five current attachments but removes historical bytes before a second heavy turn", () => {
const firstMarker = "FIRSTPAYLOAD";
const secondMarker = "SECONDPAYLOAD";
const messages: UIMessage[] = [
createAttachmentTurn("user-1", firstMarker),
{ id: "assistant-1", role: "assistant", parts: [{ type: "text", text: "I reviewed the first set." }] },
createAttachmentTurn("user-2", secondMarker),
];
const preparedMessages = prepareAssistantMessagesForTransport(messages);
const body = JSON.stringify({ id: "thread-1", messages: preparedMessages });
expect(new TextEncoder().encode(body).byteLength).toBeLessThan(routeRequestBudget);
expect(body).not.toContain(firstMarker);
expect(body.match(/data:image\/png;base64,/gu)).toHaveLength(5);
expect(body).toContain(secondMarker);
expect(preparedMessages[0]?.parts).toContainEqual(
expect.objectContaining({ type: "text", text: expect.stringContaining('"filename":"user-1-0.png"') }),
);
expect(messages[0]?.parts.filter((part) => part.type === "file")).toHaveLength(5);
expect(JSON.stringify(messages[0])).toContain(firstMarker);
});
test("prunes the serialized transport envelope without changing its initialized thread id", () => {
const originalBody = JSON.stringify({
id: "remote-thread-id",
reasoning: "auto",
messages: [
createAttachmentTurn("user-1", "OLDINLINEBYTES"),
{ id: "assistant-1", role: "assistant", parts: [{ type: "text", text: "Done" }] },
{ id: "user-2", role: "user", parts: [{ type: "text", text: "Follow up" }] },
],
});
const prepared = prepareAssistantRequestBody(originalBody);
expect(typeof prepared).toBe("string");
expect(prepared).not.toContain("OLDINLINEBYTES");
expect(JSON.parse(prepared as string)).toMatchObject({ id: "remote-thread-id", reasoning: "auto" });
});
test("keeps the active user turn available during an automatic assistant continuation", () => {
const messages: UIMessage[] = [
createAttachmentTurn("user-1", "ORIGINALTURNBYTES"),
{ id: "assistant-1", role: "assistant", parts: [{ type: "text", text: "Calling a tool" }] },
];
const prepared = prepareAssistantMessagesForTransport(messages);
expect(JSON.stringify(prepared)).toContain("ORIGINALTURNBYTES");
expect(prepared[0]?.parts.filter((part) => part.type === "file")).toHaveLength(5);
});
test("preserves hosted and provider-referenced files in historical messages", () => {
const hostedPart = {
type: "file" as const,
filename: "report.pdf",
mediaType: "application/pdf",
url: "https://files.example.test/report.pdf",
};
const providerPart = {
type: "file" as const,
filename: "provider-report.pdf",
mediaType: "application/pdf",
url: "",
providerReference: { openai: "file-1" },
};
const messages: UIMessage[] = [
{ id: "user-1", role: "user", parts: [hostedPart, providerPart] },
{ id: "assistant-1", role: "assistant", parts: [{ type: "text", text: "Done" }] },
{ id: "user-2", role: "user", parts: [{ type: "text", text: "Follow up" }] },
];
const prepared = prepareAssistantMessagesForTransport(messages);
expect(prepared[0]?.parts).toEqual([hostedPart, providerPart]);
});
});
@@ -0,0 +1,86 @@
import type { UIMessage } from "ai";
const inlineAttachmentUrlPattern = /^data:[^,]+;base64,/u;
const isRecord = (value: unknown): value is Record<string, unknown> =>
typeof value === "object" && value !== null && !Array.isArray(value);
const isFilePart = (value: unknown): value is Record<string, unknown> & { type: "file" } =>
isRecord(value) && value.type === "file";
const isUIMessage = (value: unknown): value is UIMessage =>
isRecord(value) &&
typeof value.id === "string" &&
(value.role === "user" || value.role === "assistant" || value.role === "system") &&
Array.isArray(value.parts);
const replaceControlCharacters = (value: string) =>
[...value]
.map((character) => {
const codePoint = character.codePointAt(0);
return codePoint !== undefined && (codePoint <= 0x1f || codePoint === 0x7f) ? " " : character;
})
.join("");
const getAttachmentMetadataPart = (part: Record<string, unknown>) => {
const filename =
typeof part.filename === "string" && part.filename.trim().length > 0
? replaceControlCharacters(part.filename).trim().slice(0, 160)
: "attachment";
const mediaType = typeof part.mediaType === "string" ? part.mediaType.slice(0, 100) : "application/octet-stream";
return {
type: "text" as const,
text: `[Historical attachment; binary omitted after its original turn: ${JSON.stringify({ filename, mediaType })}]`,
};
};
/**
* Keep inline bytes only for the most recent user message. The provider receives those bytes on
* the attachment's original turn; replaying every prior data URL on later turns wastes the route's
* bounded request budget and can make two otherwise valid five-image turns fail with HTTP 413.
*/
export const prepareAssistantMessagesForTransport = <TMessage extends UIMessage>(
messages: readonly TMessage[],
): TMessage[] => {
const currentAttachmentMessageIndex = messages.findLastIndex((message) => message.role === "user");
return messages.map((message, messageIndex) => {
let changed = false;
const parts: UIMessage["parts"] = [];
for (const part of message.parts) {
if (!isFilePart(part)) {
parts.push(part);
continue;
}
const url = typeof part.url === "string" ? part.url : "";
if (!inlineAttachmentUrlPattern.test(url) || messageIndex === currentAttachmentMessageIndex) {
parts.push(part);
continue;
}
changed = true;
parts.push(getAttachmentMetadataPart(part));
}
return changed ? ({ ...message, parts } as TMessage) : message;
});
};
/**
* AssistantChatTransport has already initialized the remote thread id when its fetch middleware is
* called. Prune the serialized envelope here so changing the message body cannot accidentally
* replace that remote id with assistant-ui's pre-initialization local id.
*/
export const prepareAssistantRequestBody = (body: BodyInit | null | undefined): BodyInit | null | undefined => {
if (typeof body !== "string") return body;
try {
const parsed = JSON.parse(body) as unknown;
if (!isRecord(parsed) || !Array.isArray(parsed.messages)) return body;
if (!parsed.messages.every(isUIMessage)) return body;
return JSON.stringify({ ...parsed, messages: prepareAssistantMessagesForTransport(parsed.messages) });
} catch {
return body;
}
};
@@ -1,6 +1,6 @@
import { describe, expect, test } from "vitest";
import { getSafeAssistantMarkdownImageSource } from "./assistant-markdown-image";
import { getSafeAssistantAttachmentImageSource, getSafeAssistantMarkdownImageSource } from "./assistant-markdown-image";
describe("getSafeAssistantMarkdownImageSource", () => {
test("allows Homarr-relative and ordinary remote images", () => {
@@ -17,3 +17,25 @@ describe("getSafeAssistantMarkdownImageSource", () => {
expect(getSafeAssistantMarkdownImageSource("https://user:secret@example.com/icon.svg")).toBeNull();
});
});
describe("getSafeAssistantAttachmentImageSource", () => {
test("allows persisted raster image data and same-origin image URLs", () => {
expect(getSafeAssistantAttachmentImageSource("data:image/png;base64,iVBORw0KGgo=", "https://homarr.local")).toBe(
"data:image/png;base64,iVBORw0KGgo=",
);
expect(getSafeAssistantAttachmentImageSource("/api/assistant/file/one", "https://homarr.local")).toBe(
"/api/assistant/file/one",
);
expect(
getSafeAssistantAttachmentImageSource("https://homarr.local/api/assistant/file/one", "https://homarr.local"),
).toBe("https://homarr.local/api/assistant/file/one");
});
test("rejects active image formats, unsafe schemes, and cross-origin URLs", () => {
expect(
getSafeAssistantAttachmentImageSource("data:image/svg+xml;base64,PHN2Zy8+", "https://homarr.local"),
).toBeNull();
expect(getSafeAssistantAttachmentImageSource("javascript:alert(1)", "https://homarr.local")).toBeNull();
expect(getSafeAssistantAttachmentImageSource("https://example.com/private.png", "https://homarr.local")).toBeNull();
});
});
@@ -1,11 +1,33 @@
import { getSafeAssistantHttpUrl } from "./assistant-url";
export const getSafeAssistantMarkdownImageSource = (source: string | Blob | undefined) => {
if (typeof source !== "string" || !source) return null;
if (source.startsWith("/") && !source.startsWith("//")) return source;
return getSafeAssistantHttpUrl(source);
};
const safeAttachmentDataUrl =
/^data:image\/(?:gif|jpeg|png|webp);base64,(?:[a-z\d+/]{4})*(?:[a-z\d+/]{2}==|[a-z\d+/]{3}=)?$/iu;
export const getSafeAssistantAttachmentImageSource = (source: unknown, currentOrigin?: string) => {
if (typeof source !== "string" || !source) return null;
if (safeAttachmentDataUrl.test(source)) return source;
if (source.startsWith("/") && !source.startsWith("//")) return source;
if (!currentOrigin) return null;
try {
const url = new URL(source);
if ((url.protocol !== "http:" && url.protocol !== "https:") || url.username || url.password) return null;
return url.toString();
const sourceUrl = new URL(source);
const originUrl = new URL(currentOrigin);
if (
sourceUrl.origin !== originUrl.origin ||
(sourceUrl.protocol !== "http:" && sourceUrl.protocol !== "https:") ||
sourceUrl.username ||
sourceUrl.password
) {
return null;
}
return sourceUrl.toString();
} catch {
return null;
}
@@ -1,11 +1,42 @@
import { describe, expect, test } from "vitest";
import {
getAssistantConversationUsage,
getAssistantTelemetry,
getAssistantUsage,
resolveAssistantContextWindowTelemetry,
} from "./assistant-message-metadata";
const createUsageMetadata = ({
requestId,
inputTokens,
outputTokens,
cost,
contextUsed,
}: {
requestId: string;
inputTokens: number;
outputTokens: number;
cost: number;
contextUsed?: number;
}) => ({
usage: { inputTokens, outputTokens, totalTokens: inputTokens + outputTokens },
custom: {
telemetry: {
requestId,
provider: "openrouter",
modelId: "example/model",
startedAt: "2026-08-04T10:00:00.000Z",
completedAt: "2026-08-04T10:00:01.000Z",
durationMs: 1000,
cost,
contextLength: 128_000,
...(contextUsed === undefined ? {} : { contextUsed }),
steps: [],
},
},
});
describe("getAssistantTelemetry", () => {
test("extracts persisted request telemetry", () => {
expect(
@@ -171,3 +202,63 @@ describe("resolveAssistantContextWindowTelemetry", () => {
).toBeNull();
});
});
describe("getAssistantConversationUsage", () => {
test("aggregates unique request usage and keeps the latest reported context reading", () => {
const first = createUsageMetadata({
requestId: "first",
inputTokens: 100,
outputTokens: 25,
cost: 0.001,
contextUsed: 125,
});
const second = createUsageMetadata({ requestId: "second", inputTokens: 200, outputTokens: 50, cost: 0.002 });
expect(getAssistantConversationUsage([first, second, second, undefined])).toMatchObject({
inputTokens: 300,
outputTokens: 75,
totalTokens: 375,
cost: 0.003,
contextUsed: 125,
contextLength: 128_000,
turns: [{ requestId: "first" }, { requestId: "second" }],
});
});
test("falls back to step metrics when top-level usage is unavailable", () => {
const result = getAssistantConversationUsage([
{
custom: {
telemetry: {
requestId: "step-only",
provider: "openrouter",
modelId: "example/model",
startedAt: "2026-08-04T10:00:00.000Z",
steps: [
{
index: 1,
durationMs: 100,
modelDurationMs: 100,
toolDurationMs: 0,
inputTokens: 80,
outputTokens: 20,
cachedInputTokens: 10,
reasoningTokens: 5,
cost: 0.0004,
},
],
},
},
},
]);
expect(result).toMatchObject({
inputTokens: 80,
outputTokens: 20,
totalTokens: 100,
cachedInputTokens: 10,
reasoningTokens: 5,
cost: 0.0004,
});
});
});
@@ -92,6 +92,36 @@ export type AssistantContextWindowTelemetry = {
source: "current" | "previous";
};
export type AssistantConversationTurnUsage = {
requestId: string;
provider: string;
modelId: string;
startedAt: string;
inputTokens?: number;
outputTokens?: number;
totalTokens?: number;
cachedInputTokens?: number;
reasoningTokens?: number;
cost?: number;
durationMs?: number;
contextUsed?: number;
contextLength?: number;
usage: AssistantUsage;
telemetry: AssistantRequestTelemetry;
};
export type AssistantConversationUsage = {
turns: AssistantConversationTurnUsage[];
inputTokens: number;
outputTokens: number;
totalTokens: number;
cachedInputTokens: number;
reasoningTokens: number;
cost: number;
contextUsed?: number;
contextLength?: number;
};
const getFiniteNonNegativeNumber = (value: unknown) =>
typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined;
@@ -306,6 +336,82 @@ export const getAssistantTelemetry = (metadata: unknown): AssistantRequestTeleme
};
};
const sumReportedStepMetric = (
steps: AssistantRequestStep[],
getValue: (step: AssistantRequestStep) => number | undefined,
) => {
const values = steps.flatMap((step) => {
const value = getValue(step);
return value === undefined ? [] : [value];
});
return values.length > 0 ? values.reduce((sum, value) => sum + value, 0) : undefined;
};
/**
* Produces a stable conversation-level usage snapshot from persisted message metadata.
*
* A newly-created or streaming message can temporarily have no usage metadata. Keeping
* the latest complete context reading from earlier turns avoids flashing the composer
* meter back to zero while the next request starts or when its popover is reopened.
*/
export const getAssistantConversationUsage = (messageMetadata: unknown[]): AssistantConversationUsage => {
const turnsByRequestId = new Map<string, AssistantConversationTurnUsage>();
for (const metadata of messageMetadata) {
const telemetry = getAssistantTelemetry(metadata);
if (!telemetry) continue;
const usage = getAssistantUsage(metadata);
const inputTokens = usage?.inputTokens ?? sumReportedStepMetric(telemetry.steps, (step) => step.inputTokens);
const outputTokens = usage?.outputTokens ?? sumReportedStepMetric(telemetry.steps, (step) => step.outputTokens);
const totalTokens =
usage?.totalTokens ??
(inputTokens !== undefined || outputTokens !== undefined ? (inputTokens ?? 0) + (outputTokens ?? 0) : undefined);
const cachedInputTokens =
usage?.cachedInputTokens ?? sumReportedStepMetric(telemetry.steps, (step) => step.cachedInputTokens);
const reasoningTokens =
usage?.reasoningTokens ?? sumReportedStepMetric(telemetry.steps, (step) => step.reasoningTokens);
const cost = telemetry.cost ?? sumReportedStepMetric(telemetry.steps, (step) => step.cost);
turnsByRequestId.set(telemetry.requestId, {
requestId: telemetry.requestId,
provider: telemetry.provider,
modelId: telemetry.modelId,
startedAt: telemetry.startedAt,
...(inputTokens !== undefined ? { inputTokens } : {}),
...(outputTokens !== undefined ? { outputTokens } : {}),
...(totalTokens !== undefined ? { totalTokens } : {}),
...(cachedInputTokens !== undefined ? { cachedInputTokens } : {}),
...(reasoningTokens !== undefined ? { reasoningTokens } : {}),
...(cost !== undefined ? { cost } : {}),
...(telemetry.durationMs !== undefined ? { durationMs: telemetry.durationMs } : {}),
...(telemetry.contextUsed !== undefined ? { contextUsed: telemetry.contextUsed } : {}),
...(telemetry.contextLength !== undefined ? { contextLength: telemetry.contextLength } : {}),
usage: usage ?? {},
telemetry,
});
}
const turns = [...turnsByRequestId.values()];
const latestContextTurn = turns.findLast(
(turn) => turn.contextUsed !== undefined && turn.contextLength !== undefined,
);
const sum = (getValue: (turn: AssistantConversationTurnUsage) => number | undefined) =>
turns.reduce((total, turn) => total + (getValue(turn) ?? 0), 0);
return {
turns,
inputTokens: sum((turn) => turn.inputTokens),
outputTokens: sum((turn) => turn.outputTokens),
totalTokens: sum((turn) => turn.totalTokens),
cachedInputTokens: sum((turn) => turn.cachedInputTokens),
reasoningTokens: sum((turn) => turn.reasoningTokens),
cost: sum((turn) => turn.cost),
...(latestContextTurn?.contextUsed !== undefined ? { contextUsed: latestContextTurn.contextUsed } : {}),
...(latestContextTurn?.contextLength !== undefined ? { contextLength: latestContextTurn.contextLength } : {}),
};
};
export const resolveAssistantContextWindowTelemetry = (
current: AssistantRequestTelemetry,
previousMetadata: unknown[],
@@ -205,7 +205,7 @@
}
.composerWrap {
padding: 0.45rem;
padding: 0;
}
.composer {
@@ -225,6 +225,15 @@
.suggestions {
grid-template-columns: 1fr;
}
.humanToolOption {
grid-template-columns: max-content minmax(0, 1fr);
}
.humanToolOptionBadge {
grid-column: 2;
justify-self: start;
}
}
@container assistant-widget (max-width: 22rem) {
@@ -251,6 +260,15 @@
.runtimeSelectorEffort {
display: none;
}
.composerContext {
width: 1.9rem;
padding-inline: 0.15rem;
}
.composerContext > :last-child {
display: none;
}
}
@container assistant-widget (max-width: 18rem) {
@@ -493,6 +511,33 @@
font-size: var(--mantine-font-size-xs);
}
.syntaxHighlighter {
max-width: 100%;
margin-block: 0.75rem;
overflow: hidden;
border: 1px solid var(--mantine-color-default-border);
border-radius: var(--mantine-radius-md);
background: var(--mantine-color-default);
}
.syntaxHighlighter :global(pre) {
max-width: 100%;
margin: 0;
overflow: auto;
padding: 0.75rem;
border: 0;
border-radius: 0;
background: transparent !important;
line-height: 1.5;
}
.syntaxHighlighter :global(code) {
padding: 0;
background: transparent;
font-family: var(--mantine-font-family-monospace);
font-size: var(--mantine-font-size-xs);
}
.messageMarkdown hr {
margin-block: 1rem;
border: 0;
@@ -638,19 +683,25 @@
}
.pendingQuestionDock {
flex: 0 1 auto;
max-height: min(22rem, 50cqh);
min-height: 0;
flex: 1 1 auto;
overflow-y: auto;
overscroll-behavior: contain;
padding: 0.65rem 0.75rem 0.35rem;
border-top: 1px solid var(--mantine-color-default-border);
padding: 0.75rem;
background: var(--mantine-color-body);
}
.thread[data-pending-question] .viewport {
display: none;
}
.pendingQuestionTarget {
display: flex;
width: 100%;
min-height: 100%;
max-width: 44rem;
margin-inline: auto;
align-items: safe center;
}
.pendingQuestionTarget .humanTool {
@@ -723,6 +774,11 @@
white-space: nowrap;
}
.humanToolOptionBadge {
min-width: max-content;
max-width: none;
}
.humanToolOptionText {
min-width: 0;
}
@@ -770,6 +826,10 @@
white-space: pre-wrap;
}
.toolResultNavigation {
padding-top: 0.125rem;
}
.toolResultProperties {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(7.5rem, 1fr));
@@ -849,9 +909,8 @@
.reasoning {
margin-block: 0.45rem;
padding: 0.45rem 0.6rem;
border-radius: var(--mantine-radius-md);
background: var(--mantine-color-default);
padding-inline-start: 0.6rem;
border-inline-start: 2px solid var(--mantine-color-default-border);
}
.reasoningToggle {
@@ -859,6 +918,7 @@
width: 100%;
align-items: center;
justify-content: space-between;
padding-block: 0.25rem;
border-radius: var(--mantine-radius-sm);
cursor: pointer;
}
@@ -877,11 +937,16 @@
}
.reasoningContent {
padding-top: 0.45rem;
padding: 0.3rem 0 0.2rem;
}
.reasoningText {
white-space: pre-wrap;
color: var(--mantine-color-dimmed);
font-size: var(--mantine-font-size-sm);
}
.reasoningParts {
min-width: 0;
}
.source {
@@ -941,30 +1006,57 @@
.attachment {
display: inline-flex;
min-width: min(11rem, 100%);
align-items: center;
gap: 0.35rem;
gap: 0.5rem;
margin: 0 0.35rem 0.35rem 0;
padding: 0.3rem;
}
.attachmentThumb {
.attachmentImage,
.attachmentFileIcon {
display: grid;
width: 1.7rem;
height: 1.7rem;
width: 2.5rem;
height: 2.5rem;
flex: 0 0 auto;
place-items: center;
border-radius: var(--mantine-radius-sm);
color: var(--mantine-color-dimmed);
background: var(--mantine-color-body);
font-size: 0.62rem;
font-weight: 700;
}
.attachmentImage {
object-fit: cover;
}
.attachmentCopy {
min-width: 0;
flex: 1;
}
.attachmentName {
max-width: 12rem;
max-width: 13rem;
}
.attachmentType {
max-width: 13rem;
color: light-dark(var(--mantine-color-gray-7), var(--mantine-color-gray-4));
}
.messageActions {
display: flex;
min-width: 0;
min-height: 1.75rem;
align-items: center;
gap: 0.2rem;
margin-top: 0.3rem;
flex-wrap: wrap;
}
.messageActionBar {
min-width: 0;
max-width: 100%;
flex: 0 1 auto;
}
.userActions {
@@ -973,11 +1065,41 @@
.branchPicker {
display: flex;
width: max-content;
min-height: 1.75rem;
flex: 0 0 auto;
align-items: center;
gap: 0.1rem;
}
.providerAction,
.conversationTurnProvider {
display: grid;
flex: 0 0 auto;
place-items: center;
color: var(--mantine-color-dimmed);
}
.providerAction {
width: 1.75rem;
height: 1.75rem;
border-radius: var(--mantine-radius-sm);
}
.providerAction:hover {
background: var(--mantine-color-default-hover);
}
.providerAction:focus-visible {
outline: 2px solid var(--mantine-primary-color-filled);
outline-offset: 1px;
}
.providerIcon {
display: block;
object-fit: contain;
}
.editComposer {
margin-block: 0.9rem;
padding: 0.65rem;
@@ -997,75 +1119,6 @@
font: inherit;
}
.telemetry {
margin-top: 0.45rem;
padding-inline: 0.15rem;
}
.telemetryFacts {
flex: 1 1 auto;
min-width: 0;
}
.modelBadge {
max-width: min(100%, 14rem);
}
.contextMeter {
display: grid;
width: 2.75rem;
height: 2.75rem;
flex: 0 0 auto;
place-items: center;
border-radius: 50%;
color: var(--mantine-color-text);
transition:
background-color 160ms ease,
transform 160ms ease;
}
.contextMeter:hover {
background: var(--mantine-color-default-hover);
}
.contextMeter:active {
transform: scale(0.96);
}
.contextMeter:focus-visible {
outline: 2px solid var(--mantine-primary-color-filled);
outline-offset: 2px;
}
.contextMeterLabel {
font-size: 0.55rem;
line-height: 1;
}
.telemetryPopover {
max-height: min(32rem, calc(100dvh - 2rem));
overflow-y: auto;
}
.contextStats {
display: grid;
grid-template-columns: repeat(3, minmax(0, 1fr));
gap: 0.5rem;
}
.contextStats > div {
min-width: 0;
padding: 0.45rem 0.5rem;
border-radius: var(--mantine-radius-sm);
background: var(--mantine-color-body);
}
.contextStats :global(.mantine-Text-root) {
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.usageGrid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(7rem, 1fr));
@@ -1086,9 +1139,72 @@
white-space: nowrap;
}
.composerContext {
display: flex;
min-width: 0;
max-width: 14rem;
height: 1.75rem;
align-items: center;
gap: 0.25rem;
padding-inline: 0.25rem 0.45rem;
border: 1px solid var(--mantine-color-default-border);
border-radius: var(--mantine-radius-sm);
color: var(--mantine-color-text);
background: var(--mantine-color-body);
}
.composerContext:hover {
background: var(--mantine-color-default-hover);
}
.composerContext:focus-visible {
outline: 2px solid var(--mantine-primary-color-filled);
outline-offset: 2px;
}
.conversationContextPopover {
max-height: min(34rem, calc(100dvh - 1rem));
overflow: hidden;
}
.conversationUsageSummary {
display: grid;
grid-template-columns: repeat(3, minmax(0, 1fr));
gap: 0.4rem;
}
.conversationUsageSummary > div {
min-width: 0;
padding: 0.45rem 0.5rem;
border-radius: var(--mantine-radius-sm);
background: var(--mantine-color-default);
}
.conversationTurn {
min-width: 0;
padding: 0.5rem 0.55rem;
border: 1px solid var(--mantine-color-default-border);
border-radius: var(--mantine-radius-sm);
}
.conversationTurnTrigger {
width: 100%;
text-align: start;
}
.conversationTurnTrigger:focus-visible {
outline: 2px solid var(--mantine-primary-color-filled);
outline-offset: 2px;
}
.conversationTurnProvider {
width: 1.5rem;
height: 1.5rem;
}
.composerWrap {
flex: 0 0 auto;
padding: 0.65rem 0.75rem 0.75rem;
padding: 0;
border-top: 1px solid var(--mantine-color-default-border);
background: var(--mantine-color-body);
}
@@ -1097,20 +1213,18 @@
position: relative;
display: flex;
width: 100%;
max-width: 44rem;
margin-inline: auto;
max-width: none;
flex-direction: column;
align-items: stretch;
gap: 0.5rem;
padding: 0.4rem;
border: 1px solid var(--mantine-color-default-border);
border-radius: var(--mantine-radius-lg);
background: var(--mantine-color-default);
padding: 0.45rem 0.75rem 0.55rem;
border: 0;
border-radius: 0;
background: var(--mantine-color-body);
}
.composerDropzone[data-dragging] .composer {
border-color: var(--mantine-primary-color-filled);
box-shadow: 0 0 0 2px var(--mantine-primary-color-light);
box-shadow: inset 0 0 0 2px var(--mantine-primary-color-filled);
}
.composerQuote {
@@ -1147,10 +1261,6 @@
width: 100%;
}
.composer:focus-within {
border-color: var(--mantine-primary-color-filled);
}
.composerInput {
position: relative;
min-height: 2.25rem;
@@ -1180,7 +1290,7 @@
.composerInput :global(.aui-lexical-placeholder) {
position: absolute;
inset: 0.35rem 0.45rem auto;
color: var(--mantine-color-dimmed);
color: light-dark(var(--mantine-color-gray-7), var(--mantine-color-gray-5));
line-height: 1.5;
pointer-events: none;
}
@@ -1194,6 +1304,11 @@
padding-inline: 0.2rem;
}
.composerControls {
min-width: 0;
overflow: hidden;
}
.runtimeSelectorTrigger {
width: fit-content;
min-width: 0;
@@ -1246,7 +1361,7 @@
.modelOption {
margin-inline: 0.25rem;
padding: 0.32rem 0.45rem;
padding: 0.32rem 0.45rem 0.32rem 0.3rem;
border-radius: var(--mantine-radius-sm);
}
@@ -1254,7 +1369,16 @@
.modelOption[data-combobox-active] {
color: var(--mantine-color-text);
background: var(--mantine-color-default-hover);
box-shadow: inset 2px 0 0 var(--mantine-primary-color-filled);
}
.modelOptionDivider {
height: 2rem;
align-self: center;
border-color: transparent;
}
.modelOptionDivider[data-active] {
border-color: var(--mantine-primary-color-filled);
}
.modelOptionText {
@@ -1337,6 +1461,10 @@
text-align: center;
}
.emptyDescription {
color: light-dark(var(--mantine-color-gray-7), var(--mantine-color-gray-5));
}
.suggestions {
display: grid;
width: 100%;
@@ -1411,7 +1539,7 @@
}
.composerWrap {
padding: 0.5rem;
padding: 0;
}
.composer {
@@ -1482,6 +1610,19 @@
grid-template-columns: repeat(2, minmax(0, 1fr));
}
.conversationUsageSummary {
grid-template-columns: repeat(2, minmax(0, 1fr));
}
.humanToolOption {
grid-template-columns: max-content minmax(0, 1fr);
}
.humanToolOptionBadge {
grid-column: 2;
justify-self: start;
}
.approvalActions {
flex-direction: column;
}
@@ -1492,6 +1633,10 @@
padding: 0.8rem;
}
.pendingQuestionDock {
padding: 0.5rem;
}
.boardSettingsActions,
.boardSettingsSubmitGroup {
width: 100%;
@@ -1506,7 +1651,6 @@
@media (prefers-reduced-motion: reduce) {
.activityBar,
.floatingPanel,
.contextMeter,
.disclosureIcon,
.reasoningChevron {
animation: none;
File diff suppressed because it is too large. Load diff
@@ -28,13 +28,18 @@ import { AssistantWidgetRendererProvider } from "@homarr/widgets";
import { AssistantContext, AssistantPreferencesContext, useAssistantPreferences } from "./assistant-context";
import { shouldAutomaticallyContinueAssistant } from "./assistant-auto-submit";
import { AssistantAutoApprovalProvider } from "./assistant-auto-approval";
import { prepareAssistantRequestBody } from "./assistant-attachment-payload";
import { createAssistantBrowserToolExecutors } from "./assistant-browser-tool-executors";
import { AssistantAskUserTool, AssistantConfigureAppTool } from "./assistant-human-tools";
import { AssistantPanel } from "./assistant-panel";
import { getPendingAssistantAction } from "./assistant-pending-action";
import type { AssistantReasoningMode, AssistantRuntimeModelOption } from "./assistant-preferences";
import { resolveAssistantPreferenceModelId, resolveAssistantThreadPreferenceModelId } from "./assistant-preferences";
import { AssistantRuntimeProviderWithTools } from "./assistant-runtime-provider";
import {
AssistantComposerSurfaceProvider,
AssistantRunFocusPreserver,
AssistantRuntimeProviderWithTools,
} from "./assistant-runtime-provider";
import { sendAssistantPrompt as sendPromptThroughRuntime } from "./assistant-send";
import { createAssistantPromptInteraction } from "./assistant-spotlight";
import { browserToolContracts } from "./assistant-tool-contracts";
@@ -69,14 +74,9 @@ const fileToDataUrlAsync = (file: File) =>
});
const createAssistantAttachmentAdapter = (allowImages: boolean): AttachmentAdapter => {
const pendingAttachmentIds = new Set<string>();
return {
accept: [...(allowImages ? assistantImageAttachmentTypes : []), ...assistantDocumentAttachmentTypes].join(","),
async add({ file }) {
if (pendingAttachmentIds.size >= 5) {
throw new Error("A message can include up to 5 attachments.");
}
const isImage = file.type.startsWith("image/");
if (isImage && !allowImages) {
throw new Error("The selected model does not support image input.");
@@ -93,7 +93,6 @@ const createAssistantAttachmentAdapter = (allowImages: boolean): AttachmentAdapt
throw new Error(isImage ? "Images must be smaller than 1 MB." : "Documents must be smaller than 350 KB.");
}
const id = createId();
pendingAttachmentIds.add(id);
return {
id,
type: isImage ? "image" : "document",
@@ -104,26 +103,20 @@ const createAssistantAttachmentAdapter = (allowImages: boolean): AttachmentAdapt
};
},
async send(attachment) {
try {
return {
...attachment,
status: { type: "complete" },
content: [
{
type: "file",
filename: attachment.name,
mimeType: attachment.contentType ?? "application/octet-stream",
data: await fileToDataUrlAsync(attachment.file),
},
],
};
} finally {
pendingAttachmentIds.delete(attachment.id);
}
},
async remove(attachment) {
pendingAttachmentIds.delete(attachment.id);
return {
...attachment,
status: { type: "complete" },
content: [
{
type: "file",
filename: attachment.name,
mimeType: attachment.contentType ?? "application/octet-stream",
data: await fileToDataUrlAsync(attachment.file),
},
],
};
},
async remove() {},
};
};
@@ -334,6 +327,7 @@ const AssistantThreadRuntime = () => {
new AssistantChatTransport({
api: "/api/assistant/chat",
body: preferences.getRequestBody,
fetch: (input, init) => globalThis.fetch(input, { ...init, body: prepareAssistantRequestBody(init?.body) }),
}),
[preferences.getRequestBody],
);
@@ -460,6 +454,7 @@ const AssistantRuntime = ({ children }: PropsWithChildren) => {
return (
<AssistantRuntimeProviderWithTools runtime={runtime} toolkit={toolkit}>
<AssistantRuntimeEvents />
<AssistantRunFocusPreserver />
<AssistantPreferenceSync />
{children}
</AssistantRuntimeProviderWithTools>
@@ -690,30 +685,32 @@ const EnabledAssistantProvider = ({ children }: PropsWithChildren) => {
<AssistantAutoApprovalProvider conversationId={conversationId}>
<AssistantContext.Provider value={value}>
<AssistantWidgetRendererProvider renderer={AssistantBoardWidget}>{children}</AssistantWidgetRendererProvider>
<AssistantPanel
opened={opened}
onOpen={open}
onClose={close}
onDismissActivity={() => {
markRead();
setActivityDismissed(true);
}}
activityDismissed={activityDismissed}
isRunning={assistantIsRunning}
unreadCount={unreadCount}
latestAssistantText={latestAssistantText}
latestUserText={queuedPrompt ?? latestUserText}
latestStatus={latestStatus}
pendingAction={pendingAction}
modelId={preferences.modelId}
models={preferences.models}
modelOptionsLoading={preferences.isLoading}
reasoning={preferences.reasoning}
isRefreshing={isRefreshing}
onRefresh={refreshCurrentView}
onModelChange={selectModel}
onReasoningChange={preferences.setReasoning}
/>
<AssistantComposerSurfaceProvider surfaceId="floating-panel">
<AssistantPanel
opened={opened}
onOpen={open}
onClose={close}
onDismissActivity={() => {
markRead();
setActivityDismissed(true);
}}
activityDismissed={activityDismissed}
isRunning={assistantIsRunning}
unreadCount={unreadCount}
latestAssistantText={latestAssistantText}
latestUserText={queuedPrompt ?? latestUserText}
latestStatus={latestStatus}
pendingAction={pendingAction}
modelId={preferences.modelId}
models={preferences.models}
modelOptionsLoading={preferences.isLoading}
reasoning={preferences.reasoning}
isRefreshing={isRefreshing}
onRefresh={refreshCurrentView}
onModelChange={selectModel}
onReasoningChange={preferences.setReasoning}
/>
</AssistantComposerSurfaceProvider>
</AssistantContext.Provider>
</AssistantAutoApprovalProvider>
);
@@ -1,17 +1,63 @@
// @vitest-environment jsdom
import { act, createElement } from "react";
import type { ComponentProps } from "react";
import { act, createElement, useEffect } from "react";
import { createRoot } from "react-dom/client";
import type { ChatModelAdapter, ThreadMessageLike } from "@assistant-ui/react";
import { defineToolkit, MessagePrimitive, ThreadPrimitive, useLocalRuntime } from "@assistant-ui/react";
import { afterEach, describe, expect, test } from "vitest";
import type {
AssistantClient,
AssistantRuntime,
AttachmentAdapter,
ChatModelAdapter,
ThreadMessageLike,
} from "@assistant-ui/react";
import {
defineToolkit,
INTERNAL,
MessagePrimitive,
ThreadPrimitive,
useAui,
useLocalRuntime,
} from "@assistant-ui/react";
import { LexicalComposerInput } from "@assistant-ui/react-lexical";
import { afterEach, describe, expect, test, vi } from "vitest";
import { z } from "zod/v4";
import { AssistantRuntimeProviderWithTools } from "./assistant-runtime-provider";
import { showErrorNotification } from "@homarr/notifications";
import {
AssistantComposerSurfaceProvider,
AssistantRunFocusPreserver,
AssistantRuntimeProviderWithTools,
assistantSurfaceComposerCacheLimit,
} from "./assistant-runtime-provider";
vi.mock("@homarr/notifications", () => ({ showErrorNotification: vi.fn() }));
vi.mock("@homarr/translation/client", () => ({ useScopedI18n: () => (key: string) => key }));
(globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = true;
const noOpAdapter: ChatModelAdapter = { async *run() {} };
const createAttachmentAdapter = (accept: string): AttachmentAdapter => ({
accept,
async add({ file }) {
return {
id: `${file.name}-${file.size}`,
type: file.type.startsWith("image/") ? "image" : "document",
name: file.name,
contentType: file.type,
file,
status: { type: "requires-action", reason: "composer-send" },
};
},
async send(attachment) {
return {
...attachment,
status: { type: "complete" },
content: [{ type: "text", text: attachment.name }],
};
},
async remove() {},
});
const toolkit = defineToolkit({
ask_user: {
type: "human",
@@ -54,6 +100,104 @@ const TestAssistant = () => {
);
};
interface SurfaceProbeProps {
id: string;
onReady: (id: string, aui: AssistantClient) => void;
}
const SurfaceProbe = ({ id, onReady }: SurfaceProbeProps) => {
const aui = useAui();
useEffect(() => {
onReady(id, aui);
}, [aui, id, onReady]);
return createElement(LexicalComposerInput, {
"data-assistant-composer-input": true,
"data-testid": `input-${id}`,
} as ComponentProps<typeof LexicalComposerInput>);
};
const SurfaceTestAssistant = ({
onReady,
onRuntimeReady,
allowImages = true,
}: {
onReady: SurfaceProbeProps["onReady"];
onRuntimeReady: (runtime: AssistantRuntime) => void;
allowImages?: boolean;
}) => {
const runtime = useLocalRuntime(noOpAdapter, {
adapters: { attachments: createAttachmentAdapter(allowImages ? "image/png,text/plain" : "text/plain") },
});
useEffect(() => onRuntimeReady(runtime), [onRuntimeReady, runtime]);
return createElement(
AssistantRuntimeProviderWithTools,
{ runtime, toolkit },
createElement(AssistantRunFocusPreserver),
createElement(
AssistantComposerSurfaceProvider,
{ surfaceId: "surface-a" },
createElement(SurfaceProbe, { id: "a", onReady }),
),
createElement(
AssistantComposerSurfaceProvider,
{ surfaceId: "surface-b" },
createElement(SurfaceProbe, { id: "b", onReady }),
),
);
};
const renderSurfaceTestAssistant = async ({ allowImages = true }: { allowImages?: boolean } = {}) => {
const container = document.createElement("div");
containers.push(container);
document.body.append(container);
const root = createRoot(container);
const clients = new Map<string, AssistantClient>();
let sharedRuntime: AssistantRuntime | undefined;
await act(async () => {
root.render(
createElement(SurfaceTestAssistant, {
allowImages,
onReady: (id, aui) => clients.set(id, aui),
onRuntimeReady: (runtime) => {
sharedRuntime = runtime;
},
}),
);
await new Promise((resolve) => setTimeout(resolve, 0));
});
const a = clients.get("a");
const b = clients.get("b");
if (!a || !b || !sharedRuntime) throw new Error("The assistant runtime and both surfaces must be registered");
return {
container,
root,
a,
b,
sharedRuntime,
};
};
const getRequiredComposerInput = (container: HTMLElement, testId: string) => {
const wrapper = container.querySelector<HTMLElement>(`[data-testid="${testId}"]`);
const target = wrapper?.querySelector<HTMLElement>('[contenteditable="true"]');
if (!target) throw new Error(`Missing composer input ${testId}`);
return target;
};
const getUserMessageText = (aui: AssistantClient) =>
aui
.thread()
.getState()
.messages.filter((message) => message.role === "user")
.flatMap((message) => message.content)
.filter((part) => part.type === "text")
.map((part) => part.text);
const containers: HTMLDivElement[] = [];
afterEach(() => {
@@ -77,4 +221,293 @@ describe("AssistantRuntimeProviderWithTools", () => {
await act(async () => root.unmount());
});
test("keeps drafts and attachments independent while both surfaces share one thread", async () => {
const { container, root, a, b } = await renderSurfaceTestAssistant();
act(() => {
a.composer().setText("Draft in the board widget");
b.composer().setText("Draft in the floating panel");
});
await act(async () => new Promise((resolve) => setTimeout(resolve, 0)));
await act(async () => {
await a.composer().addAttachment({
id: "board-file",
name: "board.txt",
contentType: "text/plain",
content: [{ type: "text", text: "board attachment" }],
});
});
expect(a.composer().getState().text).toBe("Draft in the board widget");
expect(b.composer().getState().text).toBe("Draft in the floating panel");
expect(
a
.composer()
.getState()
.attachments.map((attachment) => attachment.name),
).toEqual(["board.txt"]);
expect(b.composer().getState().attachments).toHaveLength(0);
expect(a.thread().getState().messages).toEqual(b.thread().getState().messages);
expect(getRequiredComposerInput(container, "input-a").textContent).toBe("Draft in the board widget");
expect(getRequiredComposerInput(container, "input-b").textContent).toBe("Draft in the floating panel");
await act(async () => root.unmount());
});
test("sends either local composer into the same shared conversation without clearing the other draft", async () => {
const { root, a, b } = await renderSurfaceTestAssistant();
act(() => {
a.composer().setText("Message from the widget");
b.composer().setText("Message from the panel");
a.composer().send();
});
await act(async () => new Promise((resolve) => setTimeout(resolve, 0)));
expect(getUserMessageText(a)).toContain("Message from the widget");
expect(getUserMessageText(b)).toContain("Message from the widget");
expect(a.composer().getState().text).toBe("");
expect(b.composer().getState().text).toBe("Message from the panel");
act(() => b.composer().send());
await act(async () => new Promise((resolve) => setTimeout(resolve, 0)));
expect(getUserMessageText(a)).toContain("Message from the panel");
expect(getUserMessageText(b)).toContain("Message from the panel");
await act(async () => root.unmount());
});
test("keeps each surface draft scoped to the selected conversation", async () => {
const { root, a, b } = await renderSurfaceTestAssistant();
act(() => {
a.composer().setText("Initialize the first conversation");
a.composer().send();
});
await act(async () => new Promise((resolve) => setTimeout(resolve, 0)));
const firstThreadId = a.threads().getState().mainThreadId;
act(() => {
a.composer().setText("First thread / widget");
b.composer().setText("First thread / panel");
});
await act(async () => a.threads().switchToNewThread());
expect(a.threads().getState().mainThreadId).not.toBe(firstThreadId);
expect(a.composer().getState().text).toBe("");
expect(b.composer().getState().text).toBe("");
act(() => a.composer().setText("Second thread / widget"));
await act(async () => a.threads().switchToThread(firstThreadId));
expect(a.composer().getState().text).toBe("First thread / widget");
expect(b.composer().getState().text).toBe("First thread / panel");
await act(async () => root.unmount());
});
test("restores focus after the Lexical run-start behavior tries to focus every surface", async () => {
const outsideInput = document.createElement("input");
document.body.append(outsideInput);
const { root, a } = await renderSurfaceTestAssistant();
outsideInput.focus();
act(() => {
a.composer().setText("Run without stealing focus");
a.composer().send();
});
await act(async () => {
await new Promise((resolve) => setTimeout(resolve, 0));
});
expect(document.activeElement).toBe(outsideInput);
outsideInput.remove();
await act(async () => root.unmount());
});
test("adds pasted images only to the active surface composer", async () => {
const { container, root, a, b } = await renderSurfaceTestAssistant();
const target = getRequiredComposerInput(container, "input-a");
const paste = new Event("paste", { bubbles: true, cancelable: true });
Object.defineProperty(paste, "clipboardData", {
value: { files: [new File(["image"], "dashboard.png", { type: "image/png" })] },
});
await act(async () => {
target.dispatchEvent(paste);
await new Promise((resolve) => setTimeout(resolve, 0));
});
expect(paste.defaultPrevented).toBe(true);
expect(
a
.composer()
.getState()
.attachments.map((attachment) => attachment.name),
).toEqual(["dashboard.png"]);
expect(b.composer().getState().attachments).toHaveLength(0);
await act(async () => root.unmount());
});
test("rejects a pasted image with an actionable message when the selected model is text-only", async () => {
vi.mocked(showErrorNotification).mockClear();
const { container, root, a } = await renderSurfaceTestAssistant({ allowImages: false });
const target = getRequiredComposerInput(container, "input-a");
const paste = new Event("paste", { bubbles: true, cancelable: true });
Object.defineProperty(paste, "clipboardData", {
value: { files: [new File(["image"], "dashboard.png", { type: "image/png" })] },
});
await act(async () => target.dispatchEvent(paste));
expect(a.composer().getState().attachments).toHaveLength(0);
expect(showErrorNotification).toHaveBeenCalledWith({
title: "attachments.errorTitle",
message: "attachments.imageUnsupported",
});
await act(async () => root.unmount());
});
test("does not capture file paste from a non-composer textarea", async () => {
const { container, root, a } = await renderSurfaceTestAssistant();
const textarea = document.createElement("textarea");
container.querySelector('[data-testid="input-a"]')?.parentElement?.append(textarea);
const paste = new Event("paste", { bubbles: true, cancelable: true });
Object.defineProperty(paste, "clipboardData", {
value: { files: [new File(["image"], "dashboard.png", { type: "image/png" })] },
});
await act(async () => textarea.dispatchEvent(paste));
expect(paste.defaultPrevented).toBe(false);
expect(a.composer().getState().attachments).toHaveLength(0);
await act(async () => root.unmount());
});
test("enforces the attachment limit independently for each surface", async () => {
vi.mocked(showErrorNotification).mockClear();
const { root, a, b } = await renderSurfaceTestAssistant();
await act(async () => {
await Promise.all(
Array.from({ length: 5 }, (_, index) =>
a.composer().addAttachment(new File([`a-${index}`], `a-${index}.txt`, { type: "text/plain" })),
),
);
await Promise.all(
Array.from({ length: 5 }, (_, index) =>
b.composer().addAttachment(new File([`b-${index}`], `b-${index}.txt`, { type: "text/plain" })),
),
);
});
expect(a.composer().getState().attachments).toHaveLength(5);
expect(b.composer().getState().attachments).toHaveLength(5);
await expect(a.composer().addAttachment(new File(["six"], "a-six.txt", { type: "text/plain" }))).rejects.toThrow(
"A message can include up to 5 attachments.",
);
expect(b.composer().getState().attachments).toHaveLength(5);
await act(async () => root.unmount());
});
test("disposes scoped composer subscriptions when its surfaces unmount", async () => {
const originalConnect = INTERNAL.DefaultThreadComposerRuntimeCore.prototype.connect;
let activeConnections = 0;
const connectSpy = vi
.spyOn(INTERNAL.DefaultThreadComposerRuntimeCore.prototype, "connect")
.mockImplementation(function (this: InstanceType<typeof INTERNAL.DefaultThreadComposerRuntimeCore>) {
const unsubscribe = originalConnect.call(this);
activeConnections += 1;
let connected = true;
return () => {
if (!connected) return;
connected = false;
activeConnections -= 1;
unsubscribe();
};
});
const { root } = await renderSurfaceTestAssistant();
const mountedConnections = activeConnections;
expect(mountedConnections).toBeGreaterThanOrEqual(3);
await act(async () => root.unmount());
expect(activeConnections).toBeLessThan(mountedConnections);
expect(mountedConnections - activeConnections).toBe(2);
connectSpy.mockRestore();
});
test("recreates scoped composers when the shared conversation core is replaced", async () => {
const originalConnect = INTERNAL.DefaultThreadComposerRuntimeCore.prototype.connect;
let connectCalls = 0;
let disconnectCalls = 0;
const connectSpy = vi
.spyOn(INTERNAL.DefaultThreadComposerRuntimeCore.prototype, "connect")
.mockImplementation(function (this: InstanceType<typeof INTERNAL.DefaultThreadComposerRuntimeCore>) {
connectCalls += 1;
const unsubscribe = originalConnect.call(this);
let connected = true;
return () => {
if (!connected) return;
connected = false;
disconnectCalls += 1;
unsubscribe();
};
});
const { root, a, b, sharedRuntime } = await renderSurfaceTestAssistant();
const mountedConnectCalls = connectCalls;
act(() => {
a.composer().setText("Draft on the first board core");
b.composer().setText("Draft on the first panel core");
});
const sharedThread = sharedRuntime.thread as InstanceType<typeof INTERNAL.ThreadRuntimeImpl>;
// oxlint-disable-next-line no-underscore-dangle -- replacing the bound core reproduces assistant-ui's outer runtime swap.
const binding = sharedThread.__internal_threadBinding;
const originalGetState = binding.getState;
const originalCore = originalGetState();
const replacementCore = new Proxy(originalCore, {
get(target, property) {
const value = Reflect.get(target, property, target) as unknown;
return typeof value === "function" ? value.bind(target) : value;
},
});
binding.getState = () => replacementCore;
act(() => originalCore.reset());
await act(async () => new Promise((resolve) => setTimeout(resolve, 0)));
expect(connectCalls - mountedConnectCalls).toBeGreaterThanOrEqual(2);
expect(disconnectCalls).toBeGreaterThanOrEqual(2);
expect(a.composer().getState().text).toBe("Draft on the first board core");
expect(b.composer().getState().text).toBe("Draft on the first panel core");
await act(async () => root.unmount());
binding.getState = originalGetState;
connectSpy.mockRestore();
});
test("evicts the least recently used inactive conversation draft", async () => {
const { root, a } = await renderSurfaceTestAssistant();
act(() => {
a.composer().setText("Initialize the oldest conversation");
a.composer().send();
});
await act(async () => new Promise((resolve) => setTimeout(resolve, 0)));
const oldestThreadId = a.threads().getState().mainThreadId;
act(() => a.composer().setText("Draft that should eventually be evicted"));
for (let index = 0; index < assistantSurfaceComposerCacheLimit; index += 1) {
await act(async () => a.threads().switchToNewThread());
act(() => {
a.composer().setText(`Initialize conversation ${index}`);
a.composer().send();
});
await act(async () => new Promise((resolve) => setTimeout(resolve, 0)));
}
await act(async () => a.threads().switchToThread(oldestThreadId));
expect(a.composer().getState().text).toBe("");
await act(async () => root.unmount());
});
});
@@ -1,8 +1,12 @@
"use client";
import type { PropsWithChildren } from "react";
import type { AssistantRuntime, Toolkit } from "@assistant-ui/react";
import { AssistantRuntimeProvider, Tools, useAui } from "@assistant-ui/react";
import type { ClipboardEvent, PropsWithChildren } from "react";
import { createContext, useContext, useEffect, useId, useLayoutEffect, useMemo, useRef } from "react";
import type { AssistantRuntime, AttachmentAdapter, ThreadListRuntime, Toolkit } from "@assistant-ui/react";
import { AssistantRuntimeProvider, INTERNAL, Tools, useAui, useAuiEvent } from "@assistant-ui/react";
import { showErrorNotification } from "@homarr/notifications";
import { useScopedI18n } from "@homarr/translation/client";
interface AssistantRuntimeProviderWithToolsProps extends PropsWithChildren {
runtime: AssistantRuntime;
@@ -21,3 +25,501 @@ export const AssistantRuntimeProviderWithTools = ({
</AssistantRuntimeProvider>
);
};
type ThreadCore = INTERNAL.ThreadRuntimeCore;
type ThreadCoreWithAdapters = ConstructorParameters<typeof INTERNAL.DefaultThreadComposerRuntimeCore>[0];
type ComposerConnectionDisposer = () => void;
export const assistantSurfaceComposerCacheLimit = 12;
const assistantSurfaceAttachmentLimit = 5;
const composerConnectionDisposers = new WeakMap<object, ComposerConnectionDisposer>();
class AssistantSurfaceComposerRuntimeCore extends INTERNAL.DefaultThreadComposerRuntimeCore {
public override connect() {
const dispose = super.connect();
composerConnectionDisposers.set(this, dispose);
return dispose;
}
public dispose() {
composerConnectionDisposers.get(this)?.();
composerConnectionDisposers.delete(this);
}
}
interface AssistantSurfaceRuntime extends AssistantRuntime {
dispose(): void;
}
interface AssistantSurfaceComposerEntry {
composer: AssistantSurfaceComposerRuntimeCore;
source: ThreadCore;
}
const getThreadAdapters = (threadCore: ThreadCore) => (threadCore as ThreadCoreWithAdapters).adapters;
const bindRuntimeValue = (target: object, property: PropertyKey, cache: WeakMap<object, Map<PropertyKey, unknown>>) => {
const value = Reflect.get(target, property, target) as unknown;
if (typeof value !== "function") return value;
let targetCache = cache.get(target);
if (!targetCache) {
targetCache = new Map();
cache.set(target, targetCache);
}
const cached = targetCache.get(property);
if (cached) return cached;
const bound = value.bind(target) as unknown;
targetCache.set(property, bound);
return bound;
};
const createSurfaceAttachmentAdapter = (threadCore: ThreadCore): AttachmentAdapter => {
const pendingAttachmentIds = new Set<string>();
let pendingAdds = 0;
const getAdapter = () => {
const adapter = getThreadAdapters(threadCore)?.attachments;
if (!adapter) throw new Error("Attachments are not supported");
return adapter;
};
return {
get accept() {
return getThreadAdapters(threadCore)?.attachments?.accept ?? "";
},
add({ file }) {
if (pendingAttachmentIds.size + pendingAdds >= assistantSurfaceAttachmentLimit) {
throw new Error(`A message can include up to ${assistantSurfaceAttachmentLimit} attachments.`);
}
pendingAdds += 1;
let result: ReturnType<AttachmentAdapter["add"]>;
try {
result = getAdapter().add({ file });
} catch (error) {
pendingAdds -= 1;
throw error;
}
if (Symbol.asyncIterator in result) {
return (async function* () {
let reservationPending = true;
try {
for await (const attachment of result) {
if (reservationPending) {
reservationPending = false;
pendingAdds -= 1;
}
pendingAttachmentIds.add(attachment.id);
yield attachment;
}
} finally {
if (reservationPending) pendingAdds -= 1;
}
})();
}
return result.then(
(attachment) => {
pendingAdds -= 1;
pendingAttachmentIds.add(attachment.id);
return attachment;
},
(error: unknown) => {
pendingAdds -= 1;
throw error;
},
);
},
async remove(attachment) {
try {
await getAdapter().remove(attachment);
} finally {
pendingAttachmentIds.delete(attachment.id);
}
},
async send(attachment) {
try {
return await getAdapter().send(attachment);
} finally {
pendingAttachmentIds.delete(attachment.id);
}
},
};
};
const createSurfaceComposerHost = (
threadCore: ThreadCore,
boundRuntimeValues: WeakMap<object, Map<PropertyKey, unknown>>,
) => {
const attachments = createSurfaceAttachmentAdapter(threadCore);
const adapters = {
get attachments() {
return getThreadAdapters(threadCore)?.attachments ? attachments : undefined;
},
get dictation() {
return getThreadAdapters(threadCore)?.dictation;
},
};
return new Proxy(threadCore as ThreadCoreWithAdapters, {
get(target, property) {
if (property === "adapters") return adapters;
return bindRuntimeValue(target, property, boundRuntimeValues);
},
});
};
/**
* assistant-ui intentionally stores the thread composer on the thread runtime. That is the right
* default for a single chat surface, but Homarr can render the same conversation in the floating
* panel and in one or more board widgets at the same time. A second runtime-bound Lexical input
* would otherwise mirror the first input's draft, attachments and selection.
*
* This adapter keeps the thread runtime itself shared and replaces only its composer core. The
* local composer still delegates send/cancel to the shared thread, so messages, runs, tools,
* branches and history remain one conversation. Composers are cached per conversation id because
* some assistant-ui runtimes reuse the same underlying core while switching conversations.
*/
export const createAssistantSurfaceRuntime = (runtime: AssistantRuntime): AssistantSurfaceRuntime => {
const sharedThread = runtime.thread as InstanceType<typeof INTERNAL.ThreadRuntimeImpl>;
// oxlint-disable-next-line no-underscore-dangle -- assistant-ui exposes the binding only through its internal adapter API.
const sharedBinding = sharedThread.__internal_threadBinding;
const composerByConversation = new Map<string, AssistantSurfaceComposerEntry>();
const surfaceCoreByConversation = new Map<string, { source: ThreadCore; surface: ThreadCore }>();
const activeConversationCounts = new Map<string, number>();
const boundRuntimeValues = new WeakMap<object, Map<PropertyKey, unknown>>();
const disposeComposer = (conversationId: string, entry: AssistantSurfaceComposerEntry) => {
const reset = entry.composer.reset();
entry.composer.dispose();
composerByConversation.delete(conversationId);
surfaceCoreByConversation.delete(conversationId);
void reset.catch(() => {});
};
const evictInactiveComposers = () => {
while (composerByConversation.size > assistantSurfaceComposerCacheLimit) {
let candidate: [string, AssistantSurfaceComposerEntry] | undefined;
for (const entry of composerByConversation) {
if (!activeConversationCounts.has(entry[0])) {
candidate = entry;
break;
}
}
if (!candidate) return;
disposeComposer(candidate[0], candidate[1]);
}
};
const retainConversation = (conversationId: string) => {
activeConversationCounts.set(conversationId, (activeConversationCounts.get(conversationId) ?? 0) + 1);
};
const releaseConversation = (conversationId: string) => {
const count = activeConversationCounts.get(conversationId);
if (count === undefined) return;
if (count <= 1) activeConversationCounts.delete(conversationId);
else activeConversationCounts.set(conversationId, count - 1);
evictInactiveComposers();
};
const getComposer = (conversationId: string, threadCore: ThreadCore) => {
const existing = composerByConversation.get(conversationId);
if (existing?.source === threadCore) {
composerByConversation.delete(conversationId);
composerByConversation.set(conversationId, existing);
return existing.composer;
}
const draft = existing
? {
attachments: existing.composer.attachments,
quote: existing.composer.quote,
role: existing.composer.role,
runConfig: existing.composer.runConfig,
text: existing.composer.text,
}
: undefined;
if (existing) disposeComposer(conversationId, existing);
const host = createSurfaceComposerHost(threadCore, boundRuntimeValues);
const composer = new AssistantSurfaceComposerRuntimeCore(host);
composerByConversation.set(conversationId, { composer, source: threadCore });
if (draft) {
composer.setText(draft.text);
composer.setRole(draft.role);
composer.setRunConfig(draft.runConfig);
composer.setQuote(draft.quote);
for (const attachment of draft.attachments) {
if (attachment.status.type === "complete" && attachment.content) {
void composer
.addAttachment({
id: attachment.id,
type: attachment.type,
name: attachment.name,
contentType: attachment.contentType,
content: attachment.content,
})
.catch(() => {});
} else if (attachment.file) {
void composer.addAttachment(attachment.file).catch(() => {});
}
}
}
evictInactiveComposers();
return composer;
};
const getSurfaceCore = (conversationId: string, threadCore: ThreadCore) => {
const existing = surfaceCoreByConversation.get(conversationId);
if (existing?.source === threadCore) return existing.surface;
const composer = getComposer(conversationId, threadCore);
const surfaceCore = new Proxy(threadCore, {
get(target, property) {
if (property === "composer") return composer;
return bindRuntimeValue(target, property, boundRuntimeValues);
},
});
surfaceCoreByConversation.set(conversationId, { source: threadCore, surface: surfaceCore });
return surfaceCore;
};
const getConversationId = () => runtime.threads.mainItem.getState().id;
const surfaceBinding: INTERNAL.ThreadRuntimeCoreBinding = {
path: sharedBinding.path,
getState: () => getSurfaceCore(getConversationId(), sharedBinding.getState()),
outerSubscribe: (callback) => sharedBinding.outerSubscribe(callback),
subscribe(callback) {
let activeCore = sharedBinding.getState();
let activeConversationId = getConversationId();
retainConversation(activeConversationId);
let unsubscribeComposer = getComposer(activeConversationId, activeCore).subscribe(callback);
const handleSharedUpdate = () => {
const nextCore = sharedBinding.getState();
const nextConversationId = getConversationId();
if (nextCore !== activeCore || nextConversationId !== activeConversationId) {
unsubscribeComposer();
releaseConversation(activeConversationId);
activeCore = nextCore;
activeConversationId = nextConversationId;
retainConversation(activeConversationId);
unsubscribeComposer = getComposer(activeConversationId, activeCore).subscribe(callback);
}
callback();
};
const unsubscribeThread = sharedBinding.subscribe(handleSharedUpdate);
const unsubscribeThreadItem = runtime.threads.mainItem.subscribe(handleSharedUpdate);
return () => {
unsubscribeComposer();
releaseConversation(activeConversationId);
unsubscribeThread();
unsubscribeThreadItem();
};
},
};
const sharedMainItem = runtime.threads.mainItem;
const mainItemBinding: INTERNAL.ThreadListItemRuntimeBinding = {
path: sharedMainItem.path,
getState: () => sharedMainItem.getState(),
subscribe: (callback) => sharedMainItem.subscribe(callback),
};
const surfaceThread = new INTERNAL.ThreadRuntimeImpl(surfaceBinding, mainItemBinding);
const sharedThreads = runtime.threads;
const getSurfaceThreadById = (threadId: string) => {
const requestedItem = sharedThreads.getItemById(threadId).getState();
if (requestedItem.id === sharedMainItem.getState().id) return surfaceThread;
// Homarr renders only the selected conversation. Preserve the upstream behavior for inactive
// thread inspection without manufacturing another composer outside this surface's LRU cache.
return sharedThreads.getById(threadId);
};
const surfaceThreads: ThreadListRuntime = {
main: surfaceThread,
mainItem: sharedMainItem,
getState: () => sharedThreads.getState(),
subscribe: (callback) => sharedThreads.subscribe(callback),
getById: getSurfaceThreadById,
getItemById: (threadId) => sharedThreads.getItemById(threadId),
getItemByIndex: (index) => sharedThreads.getItemByIndex(index),
getArchivedItemByIndex: (index) => sharedThreads.getArchivedItemByIndex(index),
switchToThread: (threadId, options) => sharedThreads.switchToThread(threadId, options),
switchToNewThread: () => sharedThreads.switchToNewThread(),
getLoadThreadsPromise: () => sharedThreads.getLoadThreadsPromise(),
reload: () => sharedThreads.reload(),
loadMore: () => sharedThreads.loadMore(),
};
return {
threads: surfaceThreads,
thread: surfaceThread,
// The root runtime already owns the inherited model-context client. Re-registering that same
// provider for every surface would make one unmount remove it from the shared runtime's Set.
registerModelContextProvider: () => () => {},
dispose() {
activeConversationCounts.clear();
for (const [conversationId, entry] of composerByConversation) disposeComposer(conversationId, entry);
},
};
};
interface AssistantComposerSurfaceValue {
id: string;
}
const AssistantComposerSurfaceContext = createContext<AssistantComposerSurfaceValue | null>(null);
export const useAssistantComposerSurface = () => {
const value = useContext(AssistantComposerSurfaceContext);
if (!value) throw new Error("useAssistantComposerSurface must be used within AssistantComposerSurfaceProvider");
return value;
};
const surfaceElements = new Map<string, HTMLElement>();
const getSurfaceIdForElement = (element: Element | null) => {
if (!element) return null;
for (const [surfaceId, surfaceElement] of surfaceElements) {
if (surfaceElement.contains(element)) return surfaceId;
}
return null;
};
const getClipboardAttachmentName = (file: File, index: number) => {
if (file.name.trim().length > 0) return file.name;
const extension = file.type.split("/")[1]?.replace("jpeg", "jpg") ?? "bin";
return `pasted-file-${index + 1}.${extension}`;
};
const AssistantComposerSurfaceEvents = () => {
const t = useScopedI18n("common.assistant");
useAuiEvent("composer.attachmentAddError", ({ message }) => {
showErrorNotification({ title: t("attachments.errorTitle"), message });
});
return null;
};
const AssistantComposerSurfaceBoundary = ({ children, surfaceId }: PropsWithChildren<{ surfaceId: string }>) => {
const t = useScopedI18n("common.assistant");
const aui = useAui();
const elementRef = useRef<HTMLDivElement>(null);
const contextValue = useMemo(() => ({ id: surfaceId }), [surfaceId]);
useLayoutEffect(() => {
const element = elementRef.current;
if (!element) return undefined;
surfaceElements.set(surfaceId, element);
return () => {
if (surfaceElements.get(surfaceId) === element) surfaceElements.delete(surfaceId);
};
}, [surfaceId]);
const handlePasteCapture = (event: ClipboardEvent<HTMLDivElement>) => {
const target = event.target;
if (!(target instanceof HTMLElement) || !target.closest("[data-assistant-composer-input]")) return;
const files = Array.from(event.clipboardData.files);
if (files.length === 0) return;
event.preventDefault();
const composer = aui.composer();
const acceptedTypes = new Set(
composer
.getState()
.attachmentAccept.split(",")
.map((type) => type.trim())
.filter(Boolean),
);
files.forEach((file, index) => {
if (
file.type.startsWith("image/") &&
!acceptedTypes.has("*") &&
!acceptedTypes.has(file.type) &&
!acceptedTypes.has("image/*")
) {
showErrorNotification({
title: t("attachments.errorTitle"),
message: t("attachments.imageUnsupported"),
});
return;
}
const attachment =
file.name.trim().length > 0
? file
: new File([file], getClipboardAttachmentName(file, index), {
type: file.type,
lastModified: file.lastModified,
});
void composer.addAttachment(attachment).catch(() => {
// The scoped composer event above presents the adapter's actionable error message.
});
});
};
return (
<AssistantComposerSurfaceContext.Provider value={contextValue}>
<div ref={elementRef} style={{ display: "contents" }} onPasteCapture={handlePasteCapture}>
<AssistantComposerSurfaceEvents />
{children}
</div>
</AssistantComposerSurfaceContext.Provider>
);
};
/**
* The current react-lexical FocusPlugin focuses every mounted composer when a run starts. Mount
* this once above all surfaces so programmatic focus returns to the element/surface the user was
* actually interacting with after those listeners have run.
*/
export const AssistantRunFocusPreserver = () => {
useAuiEvent("thread.runStart", () => {
const focusedBeforeRun = document.activeElement instanceof HTMLElement ? document.activeElement : null;
const surfaceIdBeforeRun = getSurfaceIdForElement(focusedBeforeRun);
const restoreFocus = () => {
const surfaceIdAfterRun = getSurfaceIdForElement(document.activeElement);
if (!surfaceIdAfterRun || surfaceIdAfterRun === surfaceIdBeforeRun) return;
if (focusedBeforeRun?.isConnected) focusedBeforeRun.focus({ preventScroll: true });
if (
getSurfaceIdForElement(document.activeElement) !== surfaceIdBeforeRun &&
document.activeElement instanceof HTMLElement
) {
document.activeElement.blur();
}
};
queueMicrotask(() => queueMicrotask(restoreFocus));
requestAnimationFrame(restoreFocus);
});
return null;
};
interface AssistantComposerSurfaceProviderProps extends PropsWithChildren {
surfaceId: string;
}
/**
* Gives one rendered assistant surface a local composer while preserving the shared conversation.
* Every component below this boundary can keep using the normal assistant-ui primitives and hooks.
*/
export const AssistantComposerSurfaceProvider = ({ children, surfaceId }: AssistantComposerSurfaceProviderProps) => {
const parentAui = useAui();
const instanceId = useId();
const resolvedSurfaceId = `${surfaceId}:${instanceId}`;
// oxlint-disable-next-line no-underscore-dangle -- required to derive a scoped runtime without cloning the shared thread.
const sharedRuntime = parentAui.threads().__internal_getAssistantRuntime?.();
const surfaceRuntime = useMemo(() => {
if (!sharedRuntime) throw new Error("Assistant runtime is not available for this composer surface");
return createAssistantSurfaceRuntime(sharedRuntime);
}, [sharedRuntime]);
useEffect(() => () => surfaceRuntime.dispose(), [surfaceRuntime]);
return (
<AssistantRuntimeProvider aui={parentAui} runtime={surfaceRuntime}>
<AssistantComposerSurfaceBoundary surfaceId={resolvedSurfaceId}>{children}</AssistantComposerSurfaceBoundary>
</AssistantRuntimeProvider>
);
};
@@ -0,0 +1,17 @@
import { describe, expect, test } from "vitest";
import { getSafeAssistantHttpUrl } from "./assistant-url";
describe("getSafeAssistantHttpUrl", () => {
test("allows ordinary HTTP and HTTPS source URLs", () => {
expect(getSafeAssistantHttpUrl("https://example.com/source?q=homarr")).toBe("https://example.com/source?q=homarr");
expect(getSafeAssistantHttpUrl("http://example.com/source")).toBe("http://example.com/source");
});
test("rejects unsafe protocols, relative URLs, and embedded credentials", () => {
expect(getSafeAssistantHttpUrl("javascript:alert(1)")).toBeNull();
expect(getSafeAssistantHttpUrl("data:text/html;base64,PGgxPkJhZDwvaDE+")).toBeNull();
expect(getSafeAssistantHttpUrl("/manage/custom-widgets")).toBeNull();
expect(getSafeAssistantHttpUrl("https://user:secret@example.com/source")).toBeNull();
});
});
@@ -0,0 +1,11 @@
export const getSafeAssistantHttpUrl = (source: unknown) => {
if (typeof source !== "string" || !source) return null;
try {
const url = new URL(source);
if ((url.protocol !== "http:" && url.protocol !== "https:") || url.username || url.password) return null;
return url.toString();
} catch {
return null;
}
};
@@ -13,6 +13,7 @@ import classes from "./assistant-panel.module.css";
import { AssistantConversationSurface } from "./assistant-panel";
import { getPendingAssistantAction } from "./assistant-pending-action";
import { useAssistantPreferences, useHomarrAssistant } from "./assistant-context";
import { AssistantComposerSurfaceProvider } from "./assistant-runtime-provider";
export const AssistantBoardWidget = (props: WidgetComponentProps<"assistant">) => {
const assistant = useHomarrAssistant();
@@ -21,7 +22,11 @@ export const AssistantBoardWidget = (props: WidgetComponentProps<"assistant">) =
return <UnavailableAssistantWidget description={assistant.unavailableDescription} />;
}
return <EnabledAssistantBoardWidget {...props} />;
return (
<AssistantComposerSurfaceProvider surfaceId={`board-widget-${props.itemId ?? "preview"}`}>
<EnabledAssistantBoardWidget {...props} />
</AssistantComposerSurfaceProvider>
);
};
const EnabledAssistantBoardWidget = ({ options, width, height, isEditMode }: WidgetComponentProps<"assistant">) => {