From bd3e4c33f3ffecba27a38f04f7d140b3df575274 Mon Sep 17 00:00:00 2001 From: ajnart Date: Mon, 3 Aug 2026 15:48:41 +0200 Subject: [PATCH] enhance assistant batch responses --- apps/docs/docs/management/assistant.mdx | 11 ++++ .../chat/assistant-execution-policy.spec.ts | 12 +++++ .../chat/assistant-execution-policy.ts | 7 +++ .../src/app/api/assistant/chat/route.ts | 17 ++++-- .../assistant-markdown-image.spec.ts | 19 +++++++ .../assistant/assistant-markdown-image.ts | 12 +++++ .../assistant/assistant-panel.module.css | 11 +++- .../components/assistant/assistant-panel.tsx | 20 ++++++- packages/api/src/router/board.ts | 2 +- packages/api/src/router/icons.ts | 13 +++-- packages/api/src/router/test/icons.spec.ts | 54 +++++++++++++++++++ 11 files changed, 167 insertions(+), 11 deletions(-) create mode 100644 apps/nextjs/src/app/api/assistant/chat/assistant-execution-policy.spec.ts create mode 100644 apps/nextjs/src/app/api/assistant/chat/assistant-execution-policy.ts create mode 100644 apps/nextjs/src/components/assistant/assistant-markdown-image.spec.ts create mode 100644 apps/nextjs/src/components/assistant/assistant-markdown-image.ts create mode 100644 packages/api/src/router/test/icons.spec.ts diff --git a/apps/docs/docs/management/assistant.mdx b/apps/docs/docs/management/assistant.mdx index 79b4a0ac8..80473fd19 100644 --- a/apps/docs/docs/management/assistant.mdx +++ b/apps/docs/docs/management/assistant.mdx @@ -103,6 +103,9 @@ Files returned by a compatible model can be downloaded directly from the message Assistant responses render GitHub-flavored Markdown, including headings, links, lists, task lists, tables, quotes, inline code, and fenced code blocks. Line breaks are preserved, and Homarr repairs common streamed responses where a provider returns escaped line breaks or places a bold-label list on one line. +When Homarr's icon search returns an image URL, visual result tables render that actual icon in a compact +image column instead of substituting an emoji. Assistant-provided Markdown images accept safe relative, +HTTP, and HTTPS sources and load without forwarding the Homarr page as the referrer. Images up to 1 MB and text, Markdown, CSV, HTML, XML, or JSON documents up to 350 KB can be attached by using the paperclip or dropping files onto the composer. A request can contain up to five attachments. Image understanding depends on the selected model; document text is extracted locally by Homarr and sent @@ -143,6 +146,14 @@ Common collection and widget results are summarized inline as compact responsive structured result remains available in the tool card details. Icon searches show the returned image variants, file formats, and repositories instead of the size of the complete icon catalog. +Large tool-driven requests can use up to 32 agent steps and a larger response budget within one request, so +multi-item dashboard changes normally finish without requiring a follow-up “Continue” message. Independent +read-only lookups may be performed together, while every mutation still pauses for its normal approval. + +The assistant can create a formatted dashboard note by adding a **Notebook** widget. Ask it for a note with +headings, lists, tasks, links, emphasis, tables, or images; it generates compatible rich-text content, shows +the standard mutation approval, and places the widget in the next available dashboard position. + Custom-widget tools let an administrator list, inspect, create, and update definitions. The custom-widget form's **Design with AI** action sends a focused request into the current conversation, where the assistant can propose the matching mutation and use the same approval controls. Existing stored secret values are diff --git a/apps/nextjs/src/app/api/assistant/chat/assistant-execution-policy.spec.ts b/apps/nextjs/src/app/api/assistant/chat/assistant-execution-policy.spec.ts new file mode 100644 index 000000000..717b806bd --- /dev/null +++ b/apps/nextjs/src/app/api/assistant/chat/assistant-execution-policy.spec.ts @@ -0,0 +1,12 @@ +import { describe, expect, test } from "vitest"; + +import { assistantExecutionPolicy } from "./assistant-execution-policy"; + +describe("assistantExecutionPolicy", () => { + test("allows a sizeable batch of tool calls to finish in one request", () => { + expect(assistantExecutionPolicy.maxSteps).toBeGreaterThanOrEqual(32); + expect(assistantExecutionPolicy.maxOutputTokens).toBeGreaterThanOrEqual(8_192); + expect(assistantExecutionPolicy.totalTimeoutMs).toBeGreaterThan(assistantExecutionPolicy.stepTimeoutMs); + expect(assistantExecutionPolicy.toolTimeoutMs).toBeGreaterThanOrEqual(60_000); + }); +}); diff --git a/apps/nextjs/src/app/api/assistant/chat/assistant-execution-policy.ts b/apps/nextjs/src/app/api/assistant/chat/assistant-execution-policy.ts new file mode 100644 index 000000000..138aa4b98 --- /dev/null +++ b/apps/nextjs/src/app/api/assistant/chat/assistant-execution-policy.ts @@ -0,0 +1,7 @@ +export const assistantExecutionPolicy = { + maxSteps: 32, + maxOutputTokens: 8_192, + totalTimeoutMs: 240_000, + stepTimeoutMs: 60_000, + toolTimeoutMs: 60_000, +} as const; diff --git a/apps/nextjs/src/app/api/assistant/chat/route.ts b/apps/nextjs/src/app/api/assistant/chat/route.ts index 4d6d95252..dc4f01fc7 100644 --- a/apps/nextjs/src/app/api/assistant/chat/route.ts +++ b/apps/nextjs/src/app/api/assistant/chat/route.ts @@ -32,10 +32,11 @@ import type { import { extractMcpTools } from "../../mcp/_extract-tools"; import { getRequestedMentionIds, sanitizeAttachmentFilename } from "./assistant-chat-input"; import { getAssistantModelLookupStatus } from "./assistant-model-lookup"; +import { assistantExecutionPolicy } from "./assistant-execution-policy"; import { getAssistantStreamErrorMessage } from "./assistant-stream-error"; import { getForcedAssistantToolName, withAssistantToolPolicy } from "./assistant-tool-policy"; -export const maxDuration = 60; +export const maxDuration = 300; const logger = createLogger({ module: "assistant" }); const getToolApprovalSecret = () => @@ -98,9 +99,12 @@ Action rules: - Mutating Homarr tools already pause for native user approval. Calling a mutation only proposes the action; it cannot execute until the user selects Approve and run. Once the requested change is sufficiently specified, call the mutation immediately so the approval UI appears. - A prose response such as "Please confirm if you would like me to proceed", "Would you like me to proceed?", or a parameter summary that asks for confirmation is incorrect. Never ask for a second textual confirmation before an approval-gated tool call. - Do not retry a denied action. -- Before creating an app, call configure_app with the app name and every useful default you can infer, including its description, href, pingUrl, and an icon URL returned by icon_findIcons. The user reviews Homarr's native app form. Its icon picker searches Homarr's local icon repository. Use the returned values for app_create. +- Before creating an app, call configure_app with the app name and every useful default you can infer, including its description, href, pingUrl, and an icon URL returned by icon_findIcons. The user reviews Homarr's native app form. Its icon picker searches Homarr's local icon repository. Use the returned values for app_create. If the user asked to place it on a board, use the appId returned by app_create in a board_addItem call with kind "app" and options { appId } before reporting success. - Before changing a board's custom CSS or visual and behavior settings, call board_getBoardSettings to read the current values, then call configure_board_settings with only the requested proposed changes. The user can edit the complete CSS and settings in Homarr's native form. Use its returned flat object for board_savePartialBoardSettings, which then shows the mutation approval. Never overwrite existing CSS rules unless the user explicitly requests replacement. +- To create a formatted note on a dashboard, call board_addItem with kind "notebook". Put valid Tiptap-compatible HTML in options.content and set options.showToolbar and options.allowReadOnlyCheck when useful. Use semantic paragraphs, headings, lists, task lists, blockquotes, tables, links, emphasis, and images as appropriate. Never include scripts, styles, iframes, event handlers, or unsafe URL protocols. - When choosing an app icon without configure_app, call the Homarr icon findIcons tool first and use one of its returned local icon URLs. Never invent a third-party icon CDN URL. +- When a Homarr tool returns a usable image or icon URL and a visual summary helps, embed that exact URL with Markdown image syntax. App summary tables must use an Icon column such as ![Discord icon](exact-returned-url). Never replace an available returned icon with emoji, and never invent or transform image URLs. +- Complete batch requests in the same run. Use parallel independent read-only calls when supported, reuse results, continue calling tools until every requested item has been attempted, and summarize only after the batch is complete. Do not stop after a partial batch or ask the user to type "Continue". - Browser tools can navigate within Homarr, open existing Homarr UI, or refresh the active view after a completed change. Never navigate to an arbitrary external URL and never refresh before a mutation finishes. - Keep responses concise and lead with the result. Summarize tool output instead of dumping JSON. - Use well-formed GitHub-flavored Markdown. Put each list item on its own line and leave a blank line before lists. @@ -516,9 +520,14 @@ export async function POST(request: Request) { toolChoice: { type: "tool", toolName: forcedToolName }, }; }, - stopWhen: stepCountIs(8), + stopWhen: stepCountIs(assistantExecutionPolicy.maxSteps), abortSignal: request.signal, - timeout: { totalMs: 55_000, stepMs: 30_000, toolMs: 30_000 }, + timeout: { + totalMs: assistantExecutionPolicy.totalTimeoutMs, + stepMs: assistantExecutionPolicy.stepTimeoutMs, + toolMs: assistantExecutionPolicy.toolTimeoutMs, + }, + maxOutputTokens: assistantExecutionPolicy.maxOutputTokens, maxRetries: 2, reasoning: parsed.data.reasoning === "auto" ? undefined : parsed.data.reasoning, providerOptions: diff --git a/apps/nextjs/src/components/assistant/assistant-markdown-image.spec.ts b/apps/nextjs/src/components/assistant/assistant-markdown-image.spec.ts new file mode 100644 index 000000000..c2def988f --- /dev/null +++ b/apps/nextjs/src/components/assistant/assistant-markdown-image.spec.ts @@ -0,0 +1,19 @@ +import { describe, expect, test } from "vitest"; + +import { getSafeAssistantMarkdownImageSource } from "./assistant-markdown-image"; + +describe("getSafeAssistantMarkdownImageSource", () => { + test("allows Homarr-relative and ordinary remote images", () => { + expect(getSafeAssistantMarkdownImageSource("/logo/logo.png")).toBe("/logo/logo.png"); + expect(getSafeAssistantMarkdownImageSource("https://cdn.example.com/icons/discord.svg")).toBe( + "https://cdn.example.com/icons/discord.svg", + ); + }); + + test("rejects unsafe protocols, protocol-relative URLs, and embedded credentials", () => { + expect(getSafeAssistantMarkdownImageSource("javascript:alert(1)")).toBeNull(); + expect(getSafeAssistantMarkdownImageSource("data:image/svg+xml;base64,PHN2Zy8+")).toBeNull(); + expect(getSafeAssistantMarkdownImageSource("//example.com/icon.svg")).toBeNull(); + expect(getSafeAssistantMarkdownImageSource("https://user:secret@example.com/icon.svg")).toBeNull(); + }); +}); diff --git a/apps/nextjs/src/components/assistant/assistant-markdown-image.ts b/apps/nextjs/src/components/assistant/assistant-markdown-image.ts new file mode 100644 index 000000000..e30ffea70 --- /dev/null +++ b/apps/nextjs/src/components/assistant/assistant-markdown-image.ts @@ -0,0 +1,12 @@ +export const getSafeAssistantMarkdownImageSource = (source: string | Blob | undefined) => { + if (typeof source !== "string" || !source) return null; + if (source.startsWith("/") && !source.startsWith("//")) return source; + + try { + const url = new URL(source); + if ((url.protocol !== "http:" && url.protocol !== "https:") || url.username || url.password) return null; + return url.toString(); + } catch { + return null; + } +}; diff --git a/apps/nextjs/src/components/assistant/assistant-panel.module.css b/apps/nextjs/src/components/assistant/assistant-panel.module.css index 8dc7e413e..03fc906d3 100644 --- a/apps/nextjs/src/components/assistant/assistant-panel.module.css +++ b/apps/nextjs/src/components/assistant/assistant-panel.module.css @@ -466,11 +466,13 @@ border-top: 1px solid var(--mantine-color-default-border); } -.messageMarkdown img { +.markdownImage { display: block; max-width: 100%; + max-height: 24rem; height: auto; margin-block: 0.65rem; + object-fit: contain; border-radius: var(--mantine-radius-md); } @@ -511,6 +513,13 @@ border-bottom: 0; } +.markdownTableWrap .markdownImage { + width: 2rem; + height: 2rem; + margin: 0 auto; + border-radius: var(--mantine-radius-sm); +} + .messageQuote { margin: 0 0 0.55rem; padding-inline-start: 0.65rem; diff --git a/apps/nextjs/src/components/assistant/assistant-panel.tsx b/apps/nextjs/src/components/assistant/assistant-panel.tsx index 41d541701..86cf6204e 100644 --- a/apps/nextjs/src/components/assistant/assistant-panel.tsx +++ b/apps/nextjs/src/components/assistant/assistant-panel.tsx @@ -104,6 +104,7 @@ import { useScopedI18n } from "@homarr/translation/client"; import classes from "./assistant-panel.module.css"; import { useAssistantAutoApproval, useAssistantAutomaticAction } from "./assistant-auto-approval"; import { normalizeAssistantMarkdown } from "./assistant-markdown"; +import { getSafeAssistantMarkdownImageSource } from "./assistant-markdown-image"; import { getAssistantTelemetry, getAssistantUsage } from "./assistant-message-metadata"; import type { AssistantPendingAction } from "./assistant-pending-action"; import { AssistantQuestionPortalProvider } from "./assistant-question-portal"; @@ -153,12 +154,29 @@ const MarkdownTable = ({ children, ...props }: ComponentPropsWithoutRef<"table"> ); +const MarkdownImage = ({ src, alt = "", ...props }: ComponentPropsWithoutRef<"img">) => { + const safeSource = getSafeAssistantMarkdownImageSource(src); + if (!safeSource) return null; + + return ( + {alt} + ); +}; + const AssistantTextPart = () => ( ); diff --git a/packages/api/src/router/board.ts b/packages/api/src/router/board.ts index 3af921dcc..d8c3cbe17 100644 --- a/packages/api/src/router/board.ts +++ b/packages/api/src/router/board.ts @@ -1485,7 +1485,7 @@ export const boardRouter = createTRPCRouter({ mcp: { enabled: true, description: - "Add a widget/app item to a board. Automatically places it in the first empty section at the next free grid position. Provide boardId (from board_getAllBoards), kind (widget type like 'app', 'weather', etc.), optional options map, and optional integrationIds array. Returns { itemId }", + "Add a widget/app item to a board. Automatically places it in the first empty section at the next free grid position. Provide boardId (from board_getAllBoards), kind (widget type like 'app', 'weather', etc.), optional options map, and optional integrationIds array. To create a formatted dashboard note, use kind 'notebook' with options { content: Tiptap-compatible HTML, showToolbar: boolean, allowReadOnlyCheck: boolean }. Notebook content may use semantic paragraphs, headings, lists, task lists, blockquotes, tables, links, emphasis, and images, but must not contain scripts, styles, iframes, event handlers, or unsafe URL protocols. Returns { itemId }", }, }) .input(addItemToBoardSchema) diff --git a/packages/api/src/router/icons.ts b/packages/api/src/router/icons.ts index 6758dd012..214ee894b 100644 --- a/packages/api/src/router/icons.ts +++ b/packages/api/src/router/icons.ts @@ -10,7 +10,7 @@ export const iconsRouter = createTRPCRouter({ mcp: { enabled: true, description: - "Search for icons by name across all icon repositories. OPTIONAL: searchText (string to filter), limitPerGroup (number 1-500, default 12). Call with no arguments to browse all icons", + "Search for icons by name across all icon repositories. Returns matching icon names and exact image URLs; use those URLs directly for app icons and Markdown image previews. OPTIONAL: searchText (string to filter), limitPerGroup (number 1-500, default 12). Call with no arguments to browse all icons", }, }) .input(iconsFindSchema) @@ -23,8 +23,8 @@ export const iconsRouter = createTRPCRouter({ whereCondition = and(...keywords.map((keyword) => like(icons.name, `%${keyword}%`))); } - return { - icons: await ctx.db.query.iconRepositories.findMany({ + const [repositories, countIcons] = await Promise.all([ + ctx.db.query.iconRepositories.findMany({ with: { icons: { columns: { id: true, name: true, url: true }, @@ -47,7 +47,12 @@ export const iconsRouter = createTRPCRouter({ }, }, }), - countIcons: await ctx.db.$count(icons), + ctx.db.$count(icons), + ]); + + return { + icons: term.length > 0 ? repositories.filter((repository) => repository.icons.length > 0) : repositories, + countIcons, }; }), }); diff --git a/packages/api/src/router/test/icons.spec.ts b/packages/api/src/router/test/icons.spec.ts new file mode 100644 index 000000000..e38f67b2d --- /dev/null +++ b/packages/api/src/router/test/icons.spec.ts @@ -0,0 +1,54 @@ +// @vitest-environment node + +import { describe, expect, test } from "vitest"; + +import { iconRepositories, icons } from "@homarr/db/schema"; +import { createDb } from "@homarr/db/test"; + +import { iconsRouter } from "../icons"; + +describe("findIcons", () => { + test("returns only repositories containing matches for a search", async () => { + const db = createDb(); + await db.insert(iconRepositories).values([ + { id: "dashboard-icons", slug: "homarr-labs/dashboard-icons" }, + { id: "other-icons", slug: "example/other-icons" }, + ]); + await db.insert(icons).values([ + { + id: "discord-svg", + name: "discord.svg", + url: "https://cdn.example.com/discord.svg", + checksum: "discord", + iconRepositoryId: "dashboard-icons", + }, + { + id: "homarr-svg", + name: "homarr.svg", + url: "https://cdn.example.com/homarr.svg", + checksum: "homarr", + iconRepositoryId: "other-icons", + }, + ]); + const caller = iconsRouter.createCaller({ db, deviceType: undefined, session: null }); + + const result = await caller.findIcons({ searchText: "discord", limitPerGroup: 6 }); + + expect(result.countIcons).toBe(2); + expect(result.icons).toHaveLength(1); + expect(result.icons[0]).toMatchObject({ + slug: "homarr-labs/dashboard-icons", + icons: [{ name: "discord.svg", url: "https://cdn.example.com/discord.svg" }], + }); + }); + + test("keeps repositories when browsing without a search", async () => { + const db = createDb(); + await db.insert(iconRepositories).values({ id: "empty", slug: "example/empty" }); + const caller = iconsRouter.createCaller({ db, deviceType: undefined, session: null }); + + const result = await caller.findIcons({ limitPerGroup: 6 }); + + expect(result.icons).toMatchObject([{ slug: "example/empty", icons: [] }]); + }); +});