diff --git a/apps/docs/docs/management/assistant.mdx b/apps/docs/docs/management/assistant.mdx index db5115118..f31619ff2 100644 --- a/apps/docs/docs/management/assistant.mdx +++ b/apps/docs/docs/management/assistant.mdx @@ -129,7 +129,9 @@ Fenced `mermaid` blocks render as diagrams in the current color scheme using Mer invalid diagrams fall back to their source instead of breaking the response. When Homarr's icon search returns an image URL, visual result tables render that actual icon in a compact image column instead of substituting an emoji. Assistant-provided Markdown images accept safe relative, -HTTP, and HTTPS sources and load without forwarding the Homarr page as the referrer. +HTTP, and HTTPS sources and load without forwarding the Homarr page as the referrer. Image message parts +render through assistant-ui's image primitive in a bounded responsive preview with an accessible loading state, +a filename caption when available, and an in-place retry when the source fails. Images up to 1 MB and text, Markdown, CSV, HTML, XML, or JSON documents up to 350 KB can be attached by using the paperclip, dropping files onto the composer, or pasting an image with `Cmd/Ctrl + V`. Attachment previews stay local to the composer where they were added and show upload state, file type, and a removable image @@ -192,7 +194,8 @@ the assistant does not require a second typed confirmation first. Tool cards exp validated input, result, and error state without flooding the conversation with raw JSON by default. Common collection and widget results are summarized inline as compact responsive cards, while the full structured result remains available in the tool card details. Icon searches show the returned image variants, -file formats, and repositories instead of the size of the complete icon catalog. +file formats, and repositories instead of the size of the complete icon catalog. Their trace uses a readable +activity such as **Looking for “homarr” icons** instead of exposing the internal MCP procedure name. Large tool-driven requests can use up to 32 agent steps and a larger response budget within one request, so multi-item dashboard changes normally finish without requiring a follow-up “Continue” message. Independent diff --git a/apps/nextjs/src/app/api/assistant/chat/assistant-tool-loop.spec.ts b/apps/nextjs/src/app/api/assistant/chat/assistant-tool-loop.spec.ts new file mode 100644 index 000000000..9516170b9 --- /dev/null +++ b/apps/nextjs/src/app/api/assistant/chat/assistant-tool-loop.spec.ts @@ -0,0 +1,103 @@ +import { describe, expect, test, vi } from "vitest"; +import { jsonSchema, simulateReadableStream, stepCountIs, streamText, tool } from "ai"; +import { MockLanguageModelV4 } from "ai/test"; + +const usage = { + inputTokens: { total: 10, noCache: 10, cacheRead: 0, cacheWrite: 0 }, + outputTokens: { total: 5, text: 5, reasoning: 0 }, +}; + +const finish = (unified: "stop" | "tool-calls") => ({ + type: "finish" as const, + finishReason: { unified, raw: unified }, + usage, +}); + +describe("Assistant tool loop", () => { + test("keeps a valid icon search alongside another agent tool result", async () => { + const findIcons = vi.fn(() => ({ + countIcons: 1, + icons: [ + { + slug: "homarr-labs/dashboard-icons", + icons: [{ name: "homarr.svg", url: "https://cdn.example.com/homarr.svg" }], + }, + ], + })); + const getBoardSettings = vi.fn(() => ({ id: "board-1", name: "Home" })); + const model = new MockLanguageModelV4({ + doStream: [ + { + stream: simulateReadableStream({ + chunks: [ + { type: "stream-start" as const, warnings: [] }, + { + type: "tool-call" as const, + toolCallId: "icon-call", + toolName: "icon_findIcons", + input: '{"searchText":"homarr"}', + }, + { + type: "tool-call" as const, + toolCallId: "board-call", + toolName: "board_getBoardSettings", + input: '{"id":"board-1"}', + }, + finish("tool-calls"), + ], + chunkDelayInMs: null, + }), + }, + { + stream: simulateReadableStream({ + chunks: [ + { type: "stream-start" as const, warnings: [] }, + { type: "text-start" as const, id: "answer" }, + { type: "text-delta" as const, id: "answer", delta: "Both tools completed." }, + { type: "text-end" as const, id: "answer" }, + finish("stop"), + ], + chunkDelayInMs: null, + }), + }, + ], + }); + const result = streamText({ + model, + prompt: "Find the Homarr icon and inspect my board.", + tools: { + icon_findIcons: tool({ + inputSchema: jsonSchema({ + type: "object", + properties: { + searchText: { type: "string" }, + limitPerGroup: { type: "number", minimum: 1, maximum: 500, default: 12 }, + }, + additionalProperties: false, + }), + execute: findIcons, + }), + board_getBoardSettings: tool({ + inputSchema: jsonSchema({ + type: "object", + properties: { id: { type: "string" } }, + required: ["id"], + additionalProperties: false, + }), + execute: getBoardSettings, + }), + }, + stopWhen: stepCountIs(2), + }); + + await expect(result.text).resolves.toBe("Both tools completed."); + expect(findIcons).toHaveBeenCalledWith( + { searchText: "homarr" }, + expect.objectContaining({ toolCallId: "icon-call" }), + ); + expect(getBoardSettings).toHaveBeenCalledTimes(1); + const nextPrompt = JSON.stringify(model.doStreamCalls[1]?.prompt); + expect(nextPrompt).toContain("homarr.svg"); + expect(nextPrompt).toContain('"id":"board-1"'); + }); +}); diff --git a/apps/nextjs/src/components/assistant/assistant-auto-submit.spec.ts b/apps/nextjs/src/components/assistant/assistant-auto-submit.spec.ts index fadf9f11a..8ab6827ba 100644 --- a/apps/nextjs/src/components/assistant/assistant-auto-submit.spec.ts +++ b/apps/nextjs/src/components/assistant/assistant-auto-submit.spec.ts @@ -55,4 +55,36 @@ describe("shouldAutomaticallyContinueAssistant", () => { }), ).toBe(false); }); + + test("continues a completed browser-tool step without treating an earlier agent step as user input", () => { + expect( + shouldAutomaticallyContinueAssistant({ + messages: [ + { + id: "assistant-message", + role: "assistant", + parts: [ + { type: "step-start" }, + { + type: "dynamic-tool", + toolName: "icon_findIcons", + toolCallId: "icon-1", + input: { searchText: "homarr" }, + state: "input-available", + }, + { type: "step-start" }, + { + type: "dynamic-tool", + toolName: "refresh_current_view", + toolCallId: "refresh-1", + input: {}, + state: "output-available", + output: { success: true }, + }, + ], + }, + ], + }), + ).toBe(true); + }); }); diff --git a/apps/nextjs/src/components/assistant/assistant-image.module.css b/apps/nextjs/src/components/assistant/assistant-image.module.css new file mode 100644 index 000000000..1d284f3ad --- /dev/null +++ b/apps/nextjs/src/components/assistant/assistant-image.module.css @@ -0,0 +1,69 @@ +.assistantImage { + width: fit-content; + max-width: min(100%, 28rem); + margin: 0.65rem 0; +} + +.assistantImageViewport { + position: relative; + display: grid; + min-width: min(12rem, 100%); + min-height: 6rem; + overflow: hidden; + place-items: center; + border: 1px solid var(--mantine-color-default-border); + border-radius: var(--mantine-radius-md); + background: + linear-gradient(45deg, var(--mantine-color-default) 25%, transparent 25%), + linear-gradient(-45deg, var(--mantine-color-default) 25%, transparent 25%), + linear-gradient(45deg, transparent 75%, var(--mantine-color-default) 75%), + linear-gradient(-45deg, transparent 75%, var(--mantine-color-default) 75%); + background-position: + 0 0, + 0 0.45rem, + 0.45rem -0.45rem, + -0.45rem 0; + background-size: 0.9rem 0.9rem; +} + +.assistantImageViewport > img { + display: block; + width: auto; + max-width: 100%; + height: auto; + max-height: 24rem; + margin: 0; + border-radius: 0; + object-fit: contain; +} + +.assistantImageStatus { + position: absolute; + inset: 0; + min-height: 6rem; + color: var(--mantine-color-dimmed); + background: color-mix(in srgb, var(--mantine-color-body) 92%, transparent); +} + +.assistantImageCaption { + max-width: 100%; + padding: 0.3rem 0.15rem 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +:global(td) .assistantImage { + width: 2rem; + margin: 0 auto; +} + +:global(td) .assistantImageViewport, +:global(td) .assistantImageStatus { + min-width: 2rem; + min-height: 2rem; +} + +:global(td) .assistantImageViewport { + border-radius: var(--mantine-radius-sm); +} diff --git a/apps/nextjs/src/components/assistant/assistant-image.spec.tsx b/apps/nextjs/src/components/assistant/assistant-image.spec.tsx new file mode 100644 index 000000000..46f5655eb --- /dev/null +++ b/apps/nextjs/src/components/assistant/assistant-image.spec.tsx @@ -0,0 +1,83 @@ +// @vitest-environment jsdom + +import { act, createElement } from "react"; +import { createRoot } from "react-dom/client"; +import { MantineProvider } from "@mantine/core"; +import { afterEach, beforeAll, describe, expect, test, vi } from "vitest"; + +import { AssistantImage } from "./assistant-image"; + +(globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = true; + +const containers: HTMLDivElement[] = []; + +beforeAll(() => { + Object.defineProperty(window, "matchMedia", { + configurable: true, + value: vi.fn().mockReturnValue({ + matches: false, + addEventListener: vi.fn(), + removeEventListener: vi.fn(), + }), + }); +}); + +afterEach(() => { + for (const container of containers.splice(0)) container.remove(); +}); + +const renderImage = async () => { + const container = document.createElement("div"); + containers.push(container); + document.body.append(container); + const root = createRoot(container); + await act(async () => { + root.render( + createElement( + MantineProvider, + undefined, + createElement(AssistantImage, { + source: "https://cdn.example.com/homarr.png", + alt: "Homarr icon", + loadingLabel: "Loading image", + failedLabel: "Image could not be loaded", + retryLabel: "Retry image", + }), + ), + ); + }); + return { container, root }; +}; + +describe("AssistantImage", () => { + test("shows an accessible loading state and reveals the responsive image after loading", async () => { + const { container, root } = await renderImage(); + expect(container.querySelector("output")?.getAttribute("aria-label")).toBe("Loading image"); + + const image = container.querySelector("img"); + expect(image?.getAttribute("alt")).toBe("Homarr icon"); + expect(image?.getAttribute("referrerpolicy")).toBe("no-referrer"); + await act(async () => image?.dispatchEvent(new Event("load"))); + + expect(container.querySelector("output")).toBeNull(); + expect(container.querySelector("figure")?.getAttribute("data-image-state")).toBe("loaded"); + await act(async () => root.unmount()); + }); + + test("replaces a broken image with a recoverable error and retries it", async () => { + const { container, root } = await renderImage(); + const firstImage = container.querySelector("img") as HTMLImageElement; + await act(async () => firstImage.dispatchEvent(new Event("error"))); + + expect(container.querySelector('[role="alert"]')?.textContent).toContain("Image could not be loaded"); + const retry = Array.from(container.querySelectorAll("button")).find((button) => + button.textContent?.includes("Retry image"), + ); + await act(async () => retry?.click()); + + expect(container.querySelector('[role="alert"]')).toBeNull(); + expect(container.querySelector("output")).not.toBeNull(); + expect(container.querySelector("img")).not.toBe(firstImage); + await act(async () => root.unmount()); + }); +}); diff --git a/apps/nextjs/src/components/assistant/assistant-image.tsx b/apps/nextjs/src/components/assistant/assistant-image.tsx new file mode 100644 index 000000000..c0da2b5d8 --- /dev/null +++ b/apps/nextjs/src/components/assistant/assistant-image.tsx @@ -0,0 +1,97 @@ +"use client"; + +import type { ComponentPropsWithoutRef } from "react"; +import { useEffect, useState } from "react"; +import { MessagePartPrimitive } from "@assistant-ui/react"; +import { Box, Button, Center, Loader, Stack, Text } from "@mantine/core"; +import { IconPhotoOff, IconRefresh } from "@tabler/icons-react"; + +import classes from "./assistant-image.module.css"; + +interface AssistantImageProps extends Omit, "src"> { + source: string; + messagePart?: boolean; + caption?: string; + loadingLabel: string; + failedLabel: string; + retryLabel: string; +} + +/** + * A bounded image preview shared by Markdown and assistant-ui image message parts. Message parts + * deliberately render through MessagePartPrimitive.Image so their source remains connected to the + * assistant-ui part context; the sanitized source prop overrides the raw part value. + */ +export const AssistantImage = ({ + source, + messagePart = false, + caption, + loadingLabel, + failedLabel, + retryLabel, + alt, + className, + ...imageProps +}: AssistantImageProps) => { + const [attempt, setAttempt] = useState(0); + const [state, setState] = useState<"loading" | "loaded" | "failed">("loading"); + + useEffect(() => { + setState("loading"); + setAttempt(0); + }, [source]); + + const sharedProps = { + ...imageProps, + src: source, + className, + loading: "lazy" as const, + decoding: "async" as const, + referrerPolicy: "no-referrer" as const, + onLoad: () => setState("loaded"), + onError: () => setState("failed"), + }; + + return ( + + + {messagePart ? ( + + ) : ( + {alt} + )} + {state === "loading" && ( +
+ +
+ )} + {state === "failed" && ( +
+ + + + {failedLabel} + + + +
+ )} + + {caption && state === "loaded" && ( + + {caption} + + )} + + ); +}; diff --git a/apps/nextjs/src/components/assistant/assistant-panel.tsx b/apps/nextjs/src/components/assistant/assistant-panel.tsx index 4aec558cf..e7401e55c 100644 --- a/apps/nextjs/src/components/assistant/assistant-panel.tsx +++ b/apps/nextjs/src/components/assistant/assistant-panel.tsx @@ -124,6 +124,7 @@ import classes from "./assistant-panel.module.css"; import { getAssistantActivityState } from "./assistant-activity-state"; import { useAssistantAutoApproval, useAssistantAutomaticAction } from "./assistant-auto-approval"; import { AssistantDotMatrix } from "./assistant-dot-matrix"; +import { AssistantImage } from "./assistant-image"; import { getAssistantDirectiveTranslationKey, parseAssistantDirectives } from "./assistant-directives"; import { remarkAssistantDirectives, resolveAssistantDirectiveEntity } from "./assistant-markdown-directives"; import type { AssistantDirectiveEntity } from "./assistant-markdown-directives"; @@ -142,6 +143,7 @@ import type { AssistantReasoningMode, AssistantRuntimeModelOption } from "./assi import { useAssistantReasoningState } from "./assistant-reasoning-state"; import { getNearestTriggerScrollTop } from "./assistant-trigger-scroll"; import { getToolResultPresentation } from "./assistant-tool-result"; +import { getAssistantIconSearchQuery } from "./assistant-tool-label"; import { getAssistantToolTraceTarget } from "./assistant-tool-trace"; import { getSafeAssistantHttpUrl } from "./assistant-url"; @@ -267,18 +269,19 @@ const MarkdownTable = ({ children, ...props }: ComponentPropsWithoutRef<"table"> ); const MarkdownImage = ({ src, alt = "", ...props }: ComponentPropsWithoutRef<"img">) => { + const t = useScopedI18n("common.assistant.image"); const safeSource = getSafeAssistantMarkdownImageSource(src); if (!safeSource) return null; return ( - {alt} ); }; @@ -623,7 +626,18 @@ const ImagePart = ({ image, filename }: ImageMessagePartProps) => { ); } - return ; + return ( + + ); }; const formatToolResultValue = (value: string | number | boolean) => @@ -835,6 +849,13 @@ const ToolPart = ({ const successful = completed && !denied && !failed; const compactPresentation = compact && !awaitingApproval; const traceTarget = getAssistantToolTraceTarget(args); + const iconSearchQuery = getAssistantIconSearchQuery(toolName, args); + const displayName = + iconSearchQuery === null + ? toolName.replaceAll("_", " ") + : iconSearchQuery.length > 0 + ? t("toolActivity.iconSearch", { query: iconSearchQuery }) + : t("toolActivity.iconBrowse"); const duration = timing?.completedAt !== undefined ? Math.max(0, timing.completedAt - timing.startedAt) : undefined; const autoApprovalInProgress = useAssistantAutomaticAction({ toolCallId, @@ -865,7 +886,7 @@ const ToolPart = ({ - {toolName.replaceAll("_", " ")} + {displayName} {compactPresentation && traceTarget && ( diff --git a/apps/nextjs/src/components/assistant/assistant-provider.tsx b/apps/nextjs/src/components/assistant/assistant-provider.tsx index 8d4b29079..bfb033a41 100644 --- a/apps/nextjs/src/components/assistant/assistant-provider.tsx +++ b/apps/nextjs/src/components/assistant/assistant-provider.tsx @@ -12,7 +12,8 @@ import type { Toolkit, } from "@assistant-ui/react"; import { defineToolkit, useAui, useAuiEvent, useAuiState, useRemoteThreadListRuntime } from "@assistant-ui/react"; -import { AssistantChatTransport, useChatRuntime } from "@assistant-ui/react-ai-sdk"; +import { useChat } from "@ai-sdk/react"; +import { AssistantChatTransport, useAISDKRuntime } from "@assistant-ui/react-ai-sdk"; import { useHotkeys } from "@mantine/hooks"; import { createAssistantStream } from "assistant-stream"; @@ -36,6 +37,7 @@ import { AssistantPanel } from "./assistant-panel"; import { getPendingAssistantAction } from "./assistant-pending-action"; import type { AssistantReasoningMode, AssistantRuntimeModelOption } from "./assistant-preferences"; import { resolveAssistantPreferenceModelId, resolveAssistantThreadPreferenceModelId } from "./assistant-preferences"; +import { assistantAiSdkRuntimeOptions } from "./assistant-runtime-options"; import { AssistantComposerSurfaceProvider, AssistantRunFocusPreserver, @@ -321,6 +323,8 @@ const createHistoryAdapter = (threadId: string | undefined): ThreadHistoryAdapte const AssistantThreadRuntime = () => { const t = useScopedI18n("common.assistant"); + const aui = useAui(); + const localThreadId = useAuiState((state) => state.threadListItem.id); const threadId = useAuiState((state) => state.threadListItem.remoteId); const preferences = useAssistantPreferences(); const transport = useMemo( @@ -364,16 +368,25 @@ const AssistantThreadRuntime = () => { [t], ); - return useChatRuntime({ + const chat = useChat({ + id: localThreadId, transport, onError, sendAutomaticallyWhen: shouldAutomaticallyContinueAssistant, + }); + + const runtime = useAISDKRuntime(chat, { + ...assistantAiSdkRuntimeOptions, adapters: { history, attachments, feedback, }, }); + transport.setRuntime(runtime); + // oxlint-disable-next-line no-underscore-dangle -- assistant-ui requires this hook to resolve the persisted remote thread before sending. + transport.__internal_setGetThreadListItem(() => (aui.threadListItem.source ? aui.threadListItem() : undefined)); + return runtime; }; const AssistantRuntime = ({ children }: PropsWithChildren) => { diff --git a/apps/nextjs/src/components/assistant/assistant-runtime-options.spec.ts b/apps/nextjs/src/components/assistant/assistant-runtime-options.spec.ts new file mode 100644 index 000000000..bc5bdaebb --- /dev/null +++ b/apps/nextjs/src/components/assistant/assistant-runtime-options.spec.ts @@ -0,0 +1,9 @@ +import { describe, expect, test } from "vitest"; + +import { assistantAiSdkRuntimeOptions } from "./assistant-runtime-options"; + +describe("assistantAiSdkRuntimeOptions", () => { + test("preserves pending server tools across automatic agent continuations", () => { + expect(assistantAiSdkRuntimeOptions.cancelPendingToolCallsOnSend).toBe(false); + }); +}); diff --git a/apps/nextjs/src/components/assistant/assistant-runtime-options.ts b/apps/nextjs/src/components/assistant/assistant-runtime-options.ts new file mode 100644 index 000000000..77e176905 --- /dev/null +++ b/apps/nextjs/src/components/assistant/assistant-runtime-options.ts @@ -0,0 +1,5 @@ +export const assistantAiSdkRuntimeOptions = { + // Automatic frontend-tool continuations are sends too. Letting assistant-ui cancel every + // pending tool on send incorrectly turns earlier server tools into user cancellations. + cancelPendingToolCallsOnSend: false, +} as const; diff --git a/apps/nextjs/src/components/assistant/assistant-tool-label.spec.ts b/apps/nextjs/src/components/assistant/assistant-tool-label.spec.ts new file mode 100644 index 000000000..d67cc54f3 --- /dev/null +++ b/apps/nextjs/src/components/assistant/assistant-tool-label.spec.ts @@ -0,0 +1,14 @@ +import { describe, expect, test } from "vitest"; + +import { getAssistantIconSearchQuery } from "./assistant-tool-label"; + +describe("getAssistantIconSearchQuery", () => { + test("extracts and trims the streamed icon search target", () => { + expect(getAssistantIconSearchQuery("icon_findIcons", { searchText: " homarr " })).toBe("homarr"); + }); + + test("distinguishes browsing all icons from unrelated tools", () => { + expect(getAssistantIconSearchQuery("icon_findIcons", {})).toBe(""); + expect(getAssistantIconSearchQuery("app_create", { searchText: "homarr" })).toBeNull(); + }); +}); diff --git a/apps/nextjs/src/components/assistant/assistant-tool-label.ts b/apps/nextjs/src/components/assistant/assistant-tool-label.ts new file mode 100644 index 000000000..089e43765 --- /dev/null +++ b/apps/nextjs/src/components/assistant/assistant-tool-label.ts @@ -0,0 +1,8 @@ +const asRecord = (value: unknown): Record | null => + typeof value === "object" && value !== null && !Array.isArray(value) ? (value as Record) : null; + +export const getAssistantIconSearchQuery = (toolName: string, args: unknown) => { + if (toolName !== "icon_findIcons") return null; + const searchText = asRecord(args)?.searchText; + return typeof searchText === "string" && searchText.trim().length > 0 ? searchText.trim() : ""; +}; diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 2bedb78ea..5b8815de3 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -3,6 +3,7 @@ packages: - packages/* - tooling/* catalog: + "@ai-sdk/react": ^4.0.55 "@babel/parser": ^7.29.7 "@codemirror/autocomplete": ^6.20.3 "@codemirror/commands": ^6.10.3