From f01969b9fcb706d49a1399e64ddf23a6c24c6eb9 Mon Sep 17 00:00:00 2001 From: ajnart Date: Sun, 9 Aug 2026 13:04:51 +0200 Subject: [PATCH] fix: stabilize custom widget authoring --- apps/docs/docs/management/assistant.mdx | 9 +- .../chat/assistant-stream-error.spec.ts | 21 +- .../assistant/chat/assistant-stream-error.ts | 17 +- .../chat/assistant-tool-input-repair.spec.ts | 23 ++ .../chat/assistant-tool-input-repair.ts | 29 +- .../chat/assistant-tool-policy.spec.ts | 2 + .../assistant/chat/assistant-tool-policy.ts | 2 +- .../custom-widget-authoring-context.spec.ts | 265 ++++++++++++++++++ .../chat/custom-widget-authoring-context.ts | 176 ++++++++++++ .../chat/custom-widget-tool-loop.spec.ts | 229 +++++++++++++++ .../src/app/api/assistant/chat/route.ts | 50 +++- 11 files changed, 796 insertions(+), 27 deletions(-) create mode 100644 apps/nextjs/src/app/api/assistant/chat/custom-widget-authoring-context.spec.ts create mode 100644 apps/nextjs/src/app/api/assistant/chat/custom-widget-authoring-context.ts create mode 100644 apps/nextjs/src/app/api/assistant/chat/custom-widget-tool-loop.spec.ts diff --git a/apps/docs/docs/management/assistant.mdx b/apps/docs/docs/management/assistant.mdx index 439025a92..d52bdcbbf 100644 --- a/apps/docs/docs/management/assistant.mdx +++ b/apps/docs/docs/management/assistant.mdx @@ -207,9 +207,12 @@ Custom-widget tools let an administrator list, inspect, create, and update defin form's **Design with AI** action sends a focused request into the current conversation, where the assistant can propose the matching mutation and use the same approval controls. Existing stored secret values are reported only as configured/not configured and are preserved on update unless replacements are explicitly -provided. Before authoring, the assistant lazily loads the complete installed Custom Widget skill, including -all bundled schema, runtime, and security references. Multiline JSX uses `templateLines` to avoid fragile streamed -JSON escaping. The assistant validates the candidate, creates a preview, executes every returned preview query, +provided. For Custom Widget requests, Homarr lazily adds the complete installed skill, all bundled schema, +runtime and security references, and the current JSON Schema to the trusted authoring context. This avoids an +extra provider round trip for the large `customWidget_getSkill` and `customWidget_schema` results. If the model +discovers the Custom Widget workflow mid-response, Homarr promotes the same resources before its next step and +removes the now-redundant resource calls from provider history. Multiline JSX uses `templateLines` to avoid fragile +streamed JSON escaping. The assistant validates the candidate, creates a preview, executes every returned preview query, checks the actual HTTP status and response shape, and iterates before saving. Preview and saved-definition results include safe Homarr paths that open in the assistant without a page reload. When placement is still a choice, the assistant uses the structured question UI instead of ending with a prose question. diff --git a/apps/nextjs/src/app/api/assistant/chat/assistant-stream-error.spec.ts b/apps/nextjs/src/app/api/assistant/chat/assistant-stream-error.spec.ts index 9612defb9..27b499431 100644 --- a/apps/nextjs/src/app/api/assistant/chat/assistant-stream-error.spec.ts +++ b/apps/nextjs/src/app/api/assistant/chat/assistant-stream-error.spec.ts @@ -3,9 +3,18 @@ import { describe, expect, test } from "vitest"; import { getAssistantStreamErrorMessage } from "./assistant-stream-error"; describe("assistant stream errors", () => { - test("explains malformed streamed tool input without implying the action ran", () => { + test("explains a provider-interrupted input stream without blaming tool input", () => { expect(getAssistantStreamErrorMessage(new Error("Error in input stream"))).toBe( - "The model produced incomplete tool input, so Homarr did not run the action. Try again; multiline custom-widget JSX will be sent as templateLines.", + "The model provider interrupted the streamed response before the assistant could finish. Try again.", + ); + }); + + test("explains malformed tool input without implying the action ran", () => { + expect(getAssistantStreamErrorMessage(new Error("AI_InvalidToolInputError"))).toBe( + "The model produced invalid tool input, so Homarr did not run the action. Try again.", + ); + expect(getAssistantStreamErrorMessage(new Error("Invalid input for tool customWidget_previewCreate"))).toBe( + "The model produced incomplete Custom Widget input, so Homarr did not run the action. Try again; multiline JSX will be sent as templateLines.", ); }); @@ -37,4 +46,12 @@ describe("assistant stream errors", () => { "The model endpoint took too long to respond. Try again.", ); }); + + test.each([ + [401, "The provider rejected the configured credentials. Ask an administrator to update the API key."], + [429, "The model endpoint is rate limited. Wait a moment and try again."], + [503, "The model provider is temporarily unavailable. Try again later."], + ])("prefers actionable status %s over a generic input-stream phrase", (statusCode, expected) => { + expect(getAssistantStreamErrorMessage({ statusCode, responseBody: "Error in input stream" })).toBe(expected); + }); }); diff --git a/apps/nextjs/src/app/api/assistant/chat/assistant-stream-error.ts b/apps/nextjs/src/app/api/assistant/chat/assistant-stream-error.ts index da43c52bf..6bb1e2a8a 100644 --- a/apps/nextjs/src/app/api/assistant/chat/assistant-stream-error.ts +++ b/apps/nextjs/src/app/api/assistant/chat/assistant-stream-error.ts @@ -29,18 +29,12 @@ export const getAssistantStreamErrorMessage = (error: unknown) => { const statusCode = getStatusCode(error); const message = getErrorMessage(error); - if (/AI_InvalidToolInputError|Invalid input for tool|\bError in input stream\b/iu.test(message)) { - return "The model produced incomplete tool input, so Homarr did not run the action. Try again; multiline custom-widget JSX will be sent as templateLines."; - } if ( /\bmodel(?:\s+id)?\b/iu.test(message) && /\b(invalid|unknown|unavailable|not found|not a valid)\b/iu.test(message) ) { return "The provider rejected the selected model. Ask an administrator to select a valid model ID."; } - if (statusCode === 400) { - return "The provider rejected the request. The selected model may not support the requested input or tools."; - } if (statusCode === 401 || statusCode === 403) { return "The provider rejected the configured credentials. Ask an administrator to update the API key."; } @@ -59,6 +53,17 @@ export const getAssistantStreamErrorMessage = (error: unknown) => { if (statusCode !== undefined && statusCode >= 500) { return "The model provider is temporarily unavailable. Try again later."; } + if (/AI_InvalidToolInputError|Invalid input for tool/iu.test(message)) { + return /customWidget_|custom[\s-]+widget/iu.test(message) + ? "The model produced incomplete Custom Widget input, so Homarr did not run the action. Try again; multiline JSX will be sent as templateLines." + : "The model produced invalid tool input, so Homarr did not run the action. Try again."; + } + if (statusCode === 400) { + return "The provider rejected the request. The selected model may not support the requested input or tools."; + } + if (/\bError in input stream\b/iu.test(message)) { + return "The model provider interrupted the streamed response before the assistant could finish. Try again."; + } return "The model endpoint stopped the response. Try again, or ask an administrator to verify its URL, model, and credentials."; }; diff --git a/apps/nextjs/src/app/api/assistant/chat/assistant-tool-input-repair.spec.ts b/apps/nextjs/src/app/api/assistant/chat/assistant-tool-input-repair.spec.ts index 1220acecc..ea2f3748a 100644 --- a/apps/nextjs/src/app/api/assistant/chat/assistant-tool-input-repair.spec.ts +++ b/apps/nextjs/src/app/api/assistant/chat/assistant-tool-input-repair.spec.ts @@ -15,10 +15,33 @@ describe("repairCustomWidgetToolInput", () => { }); }); + test("repairs every JSON-forbidden control character inside a Custom Widget string", () => { + const repaired = repairCustomWidgetToolInput({ + toolName: "customWidget_validate", + input: '{"value":"before\u0000\b\fafter"}', + }); + + expect(JSON.parse(repaired?.input ?? "")).toEqual({ value: "before\u0000\b\fafter" }); + }); + test("does not alter unrelated tools or guess at truncated JSON", () => { expect(repairCustomWidgetToolInput({ toolName: "app_create", input: '{"name":"Wiki\npedia"}' })).toBeNull(); expect( repairCustomWidgetToolInput({ toolName: "customWidget_create", input: '{"template":"broken' }), ).toBeNull(); }); + + test.each([ + "customWidget_getSkill", + "customWidget_schema", + "customWidget_getAuthoringPrompt", + "customWidget_getComponentCatalog", + "customWidget_list", + ])("normalizes malformed or provider-specific no-input arguments for %s", (toolName) => { + expect(repairCustomWidgetToolInput({ toolName, input: "null" })?.input).toBe("{}"); + expect(repairCustomWidgetToolInput({ toolName, input: "[]" })?.input).toBe("{}"); + expect(repairCustomWidgetToolInput({ toolName, input: '{"unexpected":true}' })?.input).toBe("{}"); + expect(repairCustomWidgetToolInput({ toolName, input: '"' })?.input).toBe("{}"); + expect(repairCustomWidgetToolInput({ toolName, input: "{}" })).toBeNull(); + }); }); diff --git a/apps/nextjs/src/app/api/assistant/chat/assistant-tool-input-repair.ts b/apps/nextjs/src/app/api/assistant/chat/assistant-tool-input-repair.ts index 2494aedd9..7f82ef41c 100644 --- a/apps/nextjs/src/app/api/assistant/chat/assistant-tool-input-repair.ts +++ b/apps/nextjs/src/app/api/assistant/chat/assistant-tool-input-repair.ts @@ -3,6 +3,14 @@ interface AssistantToolCallInput { input: string; } +const customWidgetNoInputToolNames = new Set([ + "customWidget_schema", + "customWidget_getAuthoringPrompt", + "customWidget_getSkill", + "customWidget_getComponentCatalog", + "customWidget_list", +]); + const escapeControlCharactersInsideJsonStrings = (value: string) => { let result = ""; let insideString = false; @@ -38,8 +46,15 @@ const escapeControlCharactersInsideJsonStrings = (value: string) => { case "\t": result += "\\t"; break; + case "\b": + result += "\\b"; + break; + case "\f": + result += "\\f"; + break; default: - result += character; + result += + character.charCodeAt(0) < 0x20 ? `\\u${character.charCodeAt(0).toString(16).padStart(4, "0")}` : character; } } return result; @@ -47,6 +62,18 @@ const escapeControlCharactersInsideJsonStrings = (value: string) => { export const repairCustomWidgetToolInput = (toolCall: T): T | null => { if (!toolCall.toolName.startsWith("customWidget_")) return null; + if (customWidgetNoInputToolNames.has(toolCall.toolName)) { + try { + const input = JSON.parse(toolCall.input) as unknown; + if (typeof input === "object" && input !== null && !Array.isArray(input) && Object.keys(input).length === 0) { + return null; + } + } catch { + // These read-only resource tools accept no input, so malformed provider arguments are + // unambiguous and can be replaced without guessing at user data. + } + return { ...toolCall, input: "{}" }; + } const repairedInput = escapeControlCharactersInsideJsonStrings(toolCall.input); if (repairedInput === toolCall.input) return null; try { diff --git a/apps/nextjs/src/app/api/assistant/chat/assistant-tool-policy.spec.ts b/apps/nextjs/src/app/api/assistant/chat/assistant-tool-policy.spec.ts index 6ac8d0c1b..26d16b669 100644 --- a/apps/nextjs/src/app/api/assistant/chat/assistant-tool-policy.spec.ts +++ b/apps/nextjs/src/app/api/assistant/chat/assistant-tool-policy.spec.ts @@ -30,6 +30,8 @@ describe("withAssistantToolPolicy", () => { describe("customWidgetAssistantInstructions", () => { test("requires the complete skill and real preview-query data before creation", () => { + expect(customWidgetAssistantInstructions).toContain("when they are preloaded below"); + expect(customWidgetAssistantInstructions).toContain("do not call customWidget_getSkill or customWidget_schema"); expect(customWidgetAssistantInstructions).toContain("first call customWidget_getSkill"); expect(customWidgetAssistantInstructions).toContain("customWidget_schema"); expect(customWidgetAssistantInstructions).toContain("customWidget_getComponentCatalog"); diff --git a/apps/nextjs/src/app/api/assistant/chat/assistant-tool-policy.ts b/apps/nextjs/src/app/api/assistant/chat/assistant-tool-policy.ts index 249841564..63a67a033 100644 --- a/apps/nextjs/src/app/api/assistant/chat/assistant-tool-policy.ts +++ b/apps/nextjs/src/app/api/assistant/chat/assistant-tool-policy.ts @@ -7,7 +7,7 @@ const mutationApprovalInstruction = export const customWidgetAssistantInstructions = ` Custom Widget authoring rules: -- For every request to create, repair, validate, preview, or explain a Custom JSX widget, first call customWidget_getSkill. Its response includes the complete installed skill and all references in one lazy tool result. Follow that result as authoritative. Then call customWidget_schema and the compact customWidget_getComponentCatalog for the installed release. Fetch only the relevant named component documentation with customWidget_getComponent and, when its interaction pattern applies, one named example with customWidget_getExample before authoring. Component results contain only component-specific props; if the design needs shared props such as spacing or color, collect their names from the catalog and call customWidget_getSharedProps once with only those names. +- For every request to create, repair, validate, preview, or explain a Custom JSX widget, use the complete installed skill and JSON Schema from the trusted Custom Widget authoring context when they are preloaded below; do not call customWidget_getSkill or customWidget_schema in that case. When they are not preloaded, first call customWidget_getSkill, whose response contains the complete installed skill and all references, then call customWidget_schema. Follow those resources as authoritative. Next call the compact customWidget_getComponentCatalog for the installed release. Fetch only the relevant named component documentation with customWidget_getComponent and, when its interaction pattern applies, one named example with customWidget_getExample before authoring. Component results contain only component-specific props; if the design needs shared props such as spacing or color, collect their names from the catalog and call customWidget_getSharedProps once with only those names. - Use templateLines for multiline JSX tool input. Do not send a heavily escaped multiline template string when templateLines is accepted. - Never skip the authoring lifecycle: customWidget_validate, repair all issues, customWidget_previewCreate, then customWidget_previewQuery for every query listed by the preview. Inspect the real status and returned data shape, correct request paths and template bindings, and repeat validation and preview after changes. Call customWidget_create only after the definition and its preview queries succeed. - A preview session is not evidence that its API queries work. Never claim live data works until customWidget_previewQuery returned it. Use the returned previewPath when the user should inspect the rendered result. diff --git a/apps/nextjs/src/app/api/assistant/chat/custom-widget-authoring-context.spec.ts b/apps/nextjs/src/app/api/assistant/chat/custom-widget-authoring-context.spec.ts new file mode 100644 index 000000000..9c713d1ad --- /dev/null +++ b/apps/nextjs/src/app/api/assistant/chat/custom-widget-authoring-context.spec.ts @@ -0,0 +1,265 @@ +import { convertToModelMessages } from "ai"; +import type { UIMessage } from "ai"; +import { describe, expect, test } from "vitest"; + +import { getCustomWidgetSkill, getCustomWidgetSkillContent } from "@homarr/custom-widgets/authoring-resources"; +import { getCustomWidgetJsonSchema } from "@homarr/custom-widgets/core"; + +import { + createCustomWidgetDynamicContextController, + getCustomWidgetAuthoringContext, + isCustomWidgetAuthoringToolName, + needsCustomWidgetAuthoringContext, + preloadedCustomWidgetToolNames, + prunePreloadedCustomWidgetModelMessages, +} from "./custom-widget-authoring-context"; + +const userMessage = (text: string): UIMessage => ({ + id: crypto.randomUUID(), + role: "user", + parts: [{ type: "text", text }], +}); + +describe("needsCustomWidgetAuthoringContext", () => { + test.each([ + "Create a custom widget for these fixtures", + "Create custom widgets for these services", + "Repair this custom-widget", + "Validate this Custom JSX definition", + 'Fix this {"$schema":"homarr-custom-widget-v2"}', + ])("detects explicit authoring intent: %s", (text) => { + expect(needsCustomWidgetAuthoringContext([userMessage(text)])).toBe(true); + }); + + test("continues loading authoring context after a Custom Widget tool call", () => { + const messages: UIMessage[] = [ + userMessage("Continue"), + { + id: "assistant-1", + role: "assistant", + parts: [ + { + type: "dynamic-tool", + toolName: "customWidget_previewCreate", + toolCallId: "preview-1", + state: "output-available", + input: { name: "Fixtures" }, + output: { previewId: "preview-1" }, + }, + ], + }, + ]; + + expect(needsCustomWidgetAuthoringContext(messages)).toBe(true); + }); + + test("keeps context for a user follow-up immediately after a Custom Widget tool call", () => { + const messages: UIMessage[] = [ + { + id: "assistant-1", + role: "assistant", + parts: [ + { + type: "dynamic-tool", + toolName: "customWidget_previewCreate", + toolCallId: "preview-1", + state: "output-available", + input: { name: "Fixtures" }, + output: { previewId: "preview-1" }, + }, + ], + }, + userMessage("Make the colors more compact"), + ]; + + expect(needsCustomWidgetAuthoringContext(messages)).toBe(true); + }); + + test("does not keep loading context because of stale Custom Widget history", () => { + const messages: UIMessage[] = [ + userMessage("Explain custom widgets"), + { + id: "assistant-1", + role: "assistant", + parts: [{ type: "text", text: "Custom Widgets render safe JSX." }], + }, + userMessage("How healthy is my media server?"), + ]; + + expect(needsCustomWidgetAuthoringContext(messages)).toBe(false); + }); + + test("ignores malformed unknown message parts", () => { + const malformedMessage = { + id: "malformed", + role: "user", + parts: [null, {}, { type: 1 }, { type: "text", text: null }], + } as unknown as UIMessage; + + expect(needsCustomWidgetAuthoringContext([malformedMessage])).toBe(false); + }); + + test("does not load authoring context for ordinary widget requests or assistant prose", () => { + const messages: UIMessage[] = [ + userMessage("Add my media server widget to the home board"), + { + id: "assistant-1", + role: "assistant", + parts: [{ type: "text", text: "I could also build a custom widget." }], + }, + ]; + + expect(needsCustomWidgetAuthoringContext(messages)).toBe(false); + expect(getCustomWidgetAuthoringContext(messages, true)).toEqual({ systemContext: "", omittedToolNames: [] }); + expect(Buffer.byteLength(getCustomWidgetAuthoringContext(messages, true).systemContext, "utf8")).toBe(0); + }); +}); + +describe("getCustomWidgetAuthoringContext", () => { + test("does not expose administrator-only authoring resources to non-admins", () => { + expect(getCustomWidgetAuthoringContext([userMessage("Create a custom widget")], false)).toEqual({ + systemContext: "", + omittedToolNames: [], + }); + }); + + test("preloads the complete installed skill and every exact bundled reference", () => { + const context = getCustomWidgetAuthoringContext([userMessage("Create a custom widget")], true); + const skill = getCustomWidgetSkill(); + + expect(context.systemContext).toContain(`## Installed skill: ${skill.name} ${skill.version}`); + expect(context.systemContext).toContain(skill.skillMd.trim()); + expect(context.systemContext).toContain(getCustomWidgetSkillContent()); + for (const [file, content] of Object.entries(skill.references)) { + expect(context.systemContext).toContain(`# Bundled file: ${file}`); + expect(context.systemContext).toContain(content.trim()); + } + }); + + test("preloads a valid current JSON Schema and omits only the redundant resource tools", () => { + const context = getCustomWidgetAuthoringContext([userMessage("Validate my custom widget")], true); + const schemaBlock = context.systemContext.match( + /## Current Custom Widget JSON Schema\n\n```json\n(?[\s\S]+)\n```$/u, + ); + + expect(schemaBlock?.groups?.schema).toBeDefined(); + const schema = JSON.parse(schemaBlock?.groups?.schema ?? "null") as Record; + expect(schema).toMatchObject({ + $schema: expect.any(String), + title: "Homarr Custom JSX v2 widget", + type: "object", + }); + expect(schema).toEqual(getCustomWidgetJsonSchema()); + expect(JSON.stringify(schema)).toContain("homarr-custom-widget-v2"); + expect(context.omittedToolNames).toEqual(preloadedCustomWidgetToolNames); + expect(context.systemContext).toContain( + "customWidget_getSkill and customWidget_schema are already preloaded below", + ); + expect(context.systemContext).toContain("intentionally omitted from the available tools"); + }); + + test("keeps the complete trusted context within a bounded request budget", () => { + const context = getCustomWidgetAuthoringContext([userMessage("Create a custom widget")], true); + + expect(Buffer.byteLength(context.systemContext, "utf8")).toBeLessThan(100_000); + }); +}); + +describe("createCustomWidgetDynamicContextController", () => { + test("activates only for authoring tools, not lightweight management or Workshop inspection", () => { + expect(isCustomWidgetAuthoringToolName("customWidget_getComponentCatalog")).toBe(true); + expect(isCustomWidgetAuthoringToolName("customWidget_validate")).toBe(true); + expect(isCustomWidgetAuthoringToolName("customWidget_previewCreate")).toBe(true); + expect(isCustomWidgetAuthoringToolName("customWidget_list")).toBe(false); + expect(isCustomWidgetAuthoringToolName("customWidget_get")).toBe(false); + expect(isCustomWidgetAuthoringToolName("customWidget_workshopSearch")).toBe(false); + }); + + test("does not promote protected resources after a non-admin tool attempt", () => { + const prepareContext = createCustomWidgetDynamicContextController({ + isAdmin: false, + baseInstructions: "Base instructions", + availableToolNames: ["customWidget_getSkill", "customWidget_schema"], + }); + + expect( + prepareContext({ + instructions: "Base instructions", + messages: [], + steps: [{ toolCalls: [{ toolName: "customWidget_getSkill" }] }], + }), + ).toBeUndefined(); + }); +}); + +describe("prunePreloadedCustomWidgetModelMessages", () => { + test("removes only preloaded resource call-result pairs and preserves the rest of the turn", async () => { + const messages: UIMessage[] = [ + userMessage("Create a custom widget"), + { + id: "assistant-1", + role: "assistant", + parts: [ + { + type: "dynamic-tool", + toolName: "customWidget_getSkill", + toolCallId: "skill-1", + state: "output-available", + input: {}, + output: { name: "homarr-custom-widget" }, + }, + { + type: "dynamic-tool", + toolName: "customWidget_schema", + toolCallId: "schema-1", + state: "output-available", + input: {}, + output: { type: "object" }, + }, + { type: "text", text: "I loaded the authoring resources." }, + { + type: "dynamic-tool", + toolName: "customWidget_previewCreate", + toolCallId: "preview-1", + state: "output-available", + input: { widget: { name: "Fixtures" } }, + output: { previewId: "preview-1" }, + }, + ], + }, + ]; + + const modelMessages = await convertToModelMessages(messages); + const result = prunePreloadedCustomWidgetModelMessages(modelMessages); + const serialized = JSON.stringify(result); + + expect(serialized).not.toContain("customWidget_getSkill"); + expect(serialized).not.toContain("customWidget_schema"); + expect(serialized).toContain("customWidget_previewCreate"); + expect(serialized).toContain("I loaded the authoring resources."); + }); + + test("removes empty assistant and tool messages left by resource-only history", async () => { + const modelMessages = await convertToModelMessages([ + userMessage("Create a custom widget"), + { + id: "assistant-1", + role: "assistant", + parts: [ + { + type: "dynamic-tool", + toolName: "customWidget_getSkill", + toolCallId: "skill-1", + state: "output-available", + input: {}, + output: { name: "homarr-custom-widget" }, + }, + ], + }, + ]); + + expect(prunePreloadedCustomWidgetModelMessages(modelMessages)).toEqual([ + { role: "user", content: [{ type: "text", text: "Create a custom widget" }] }, + ]); + }); +}); diff --git a/apps/nextjs/src/app/api/assistant/chat/custom-widget-authoring-context.ts b/apps/nextjs/src/app/api/assistant/chat/custom-widget-authoring-context.ts new file mode 100644 index 000000000..463e9f11c --- /dev/null +++ b/apps/nextjs/src/app/api/assistant/chat/custom-widget-authoring-context.ts @@ -0,0 +1,176 @@ +import { pruneMessages } from "ai"; +import type { Instructions, ModelMessage, UIMessage } from "ai"; + +import { getCustomWidgetSkill, getCustomWidgetSkillContent } from "@homarr/custom-widgets/authoring-resources"; +import { getCustomWidgetJsonSchema } from "@homarr/custom-widgets/core"; + +export const preloadedCustomWidgetToolNames = ["customWidget_getSkill", "customWidget_schema"] as const; +const preloadedCustomWidgetToolNameSet = new Set(preloadedCustomWidgetToolNames); +const customWidgetAuthoringToolNameSet = new Set([ + ...preloadedCustomWidgetToolNames, + "customWidget_getAuthoringPrompt", + "customWidget_getComponentCatalog", + "customWidget_getComponent", + "customWidget_getSharedProps", + "customWidget_getExample", + "customWidget_validate", + "customWidget_previewCreate", + "customWidget_previewQuery", + "customWidget_previewAction", + "customWidget_previewJournal", + "customWidget_create", + "customWidget_update", + "customWidget_templatePatch", + "customWidget_legacyMigrationPrompt", + "customWidget_migrateLegacy", +]); + +export const isPreloadedCustomWidgetToolName = (toolName: string) => preloadedCustomWidgetToolNameSet.has(toolName); +export const isCustomWidgetAuthoringToolName = (toolName: string) => customWidgetAuthoringToolNameSet.has(toolName); + +export type CustomWidgetAuthoringContext = { + systemContext: string; + omittedToolNames: readonly (typeof preloadedCustomWidgetToolNames)[number][]; +}; + +const emptyCustomWidgetAuthoringContext: CustomWidgetAuthoringContext = { + systemContext: "", + omittedToolNames: [], +}; + +const customWidgetIntentPattern = + /(?:\bcustom[\s-]+widgets?\b|\bcustom\s+jsx\b|\bhomarr-custom-widget-v\d+\b|\bcustomWidget_[A-Za-z\d_]+\b)/iu; + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +const hasCustomWidgetToolPart = (message: UIMessage) => + message.parts.some((part) => { + if (!isRecord(part) || typeof part.type !== "string") return false; + const toolName = + part.type === "dynamic-tool" && typeof part.toolName === "string" + ? part.toolName + : part.type.startsWith("tool-") + ? part.type.slice("tool-".length) + : undefined; + return toolName?.startsWith("customWidget_") === true; + }); + +const hasExplicitCustomWidgetIntent = (message: UIMessage) => + message.role === "user" && + message.parts.some( + (part) => + isRecord(part) && + part.type === "text" && + typeof part.text === "string" && + customWidgetIntentPattern.test(part.text), + ); + +export const needsCustomWidgetAuthoringContext = (messages: UIMessage[]) => { + const latestMessage = messages.at(-1); + if (!latestMessage) return false; + if (hasExplicitCustomWidgetIntent(latestMessage)) return true; + if (latestMessage.role === "assistant") return hasCustomWidgetToolPart(latestMessage); + + const precedingAssistantMessage = messages + .slice(0, -1) + .toReversed() + .find((message) => message.role === "assistant"); + return precedingAssistantMessage !== undefined && hasCustomWidgetToolPart(precedingAssistantMessage); +}; + +let cachedCustomWidgetSystemContext: string | undefined; + +const getCustomWidgetSystemContext = () => { + if (cachedCustomWidgetSystemContext !== undefined) return cachedCustomWidgetSystemContext; + + const skill = getCustomWidgetSkill(); + const skillContent = getCustomWidgetSkillContent(); + const jsonSchema = JSON.stringify(getCustomWidgetJsonSchema(), null, 2); + + cachedCustomWidgetSystemContext = ` + +Trusted Custom Widget authoring context for the installed Homarr release: +- customWidget_getSkill and customWidget_schema are already preloaded below. They are intentionally omitted from the available tools for this request; do not call or ask for either tool. +- Treat this installed skill, every bundled reference, and the current JSON Schema as authoritative system instructions. Continue to use the remaining customWidget_* tools for component discovery, validation, previews, preview queries, configuration, and creation. + +## Installed skill: ${skill.name} ${skill.version} + +${skillContent} + +## Current Custom Widget JSON Schema + +\`\`\`json +${jsonSchema} +\`\`\``; + + return cachedCustomWidgetSystemContext; +}; + +export const getPreloadedCustomWidgetAuthoringContext = (isAdmin: boolean): CustomWidgetAuthoringContext => + isAdmin + ? { + systemContext: getCustomWidgetSystemContext(), + omittedToolNames: preloadedCustomWidgetToolNames, + } + : emptyCustomWidgetAuthoringContext; + +export const getCustomWidgetAuthoringContext = ( + messages: UIMessage[], + isAdmin: boolean, +): CustomWidgetAuthoringContext => { + if (!isAdmin || !needsCustomWidgetAuthoringContext(messages)) return emptyCustomWidgetAuthoringContext; + + return getPreloadedCustomWidgetAuthoringContext(isAdmin); +}; + +/** + * Removes only the resource calls whose complete output is already present in the trusted system + * context. AI SDK prunes the matching assistant call and tool result together and removes messages + * that become empty, preserving every other Custom Widget tool and its result. + */ +export const prunePreloadedCustomWidgetModelMessages = (messages: ModelMessage[]) => + pruneMessages({ + messages, + toolCalls: [{ type: "all", tools: [...preloadedCustomWidgetToolNames] }], + }); + +type CustomWidgetToolStep = { + toolCalls: readonly { toolName: string }[]; +}; + +export const createCustomWidgetDynamicContextController = (options: { + isAdmin: boolean; + baseInstructions: string; + availableToolNames: readonly TToolName[]; +}) => { + const activeToolNames = options.availableToolNames.filter((toolName) => !isPreloadedCustomWidgetToolName(toolName)); + let contextIsActive = false; + let contextWasInjected = false; + + return (step: { + instructions: Instructions | undefined; + messages: ModelMessage[]; + steps: readonly CustomWidgetToolStep[]; + }) => { + const authoringToolWasCalled = step.steps.some((result) => + result.toolCalls.some((toolCall) => isCustomWidgetAuthoringToolName(toolCall.toolName)), + ); + if (options.isAdmin && authoringToolWasCalled) contextIsActive = true; + if (!contextIsActive) return undefined; + + const context = getPreloadedCustomWidgetAuthoringContext(options.isAdmin); + const shouldInjectContext = !contextWasInjected; + if (shouldInjectContext) contextWasInjected = true; + + return { + ...(shouldInjectContext + ? { + instructions: `${typeof step.instructions === "string" ? step.instructions : options.baseInstructions}${context.systemContext}`, + } + : {}), + messages: prunePreloadedCustomWidgetModelMessages(step.messages), + activeTools: activeToolNames, + }; + }; +}; diff --git a/apps/nextjs/src/app/api/assistant/chat/custom-widget-tool-loop.spec.ts b/apps/nextjs/src/app/api/assistant/chat/custom-widget-tool-loop.spec.ts new file mode 100644 index 000000000..84d9766df --- /dev/null +++ b/apps/nextjs/src/app/api/assistant/chat/custom-widget-tool-loop.spec.ts @@ -0,0 +1,229 @@ +import { describe, expect, it, vi } from "vitest"; +import { jsonSchema, simulateReadableStream, stepCountIs, streamText, tool } from "ai"; +import { MockLanguageModelV4 } from "ai/test"; +import { parse, stringify } from "superjson"; + +import { getCustomWidgetSkill } from "@homarr/custom-widgets/authoring-resources"; +import { getCustomWidgetJsonSchema } from "@homarr/custom-widgets/core"; + +import { createCustomWidgetDynamicContextController } from "./custom-widget-authoring-context"; +import { repairCustomWidgetToolInput } from "./assistant-tool-input-repair"; + +const usage = { + inputTokens: { total: 10, noCache: 10, cacheRead: 0, cacheWrite: 0 }, + outputTokens: { total: 5, text: 5, reasoning: 0 }, +}; + +const finish = (unified: "stop" | "tool-calls") => ({ + type: "finish" as const, + finishReason: { unified, raw: unified }, + usage, +}); + +describe("Custom Widget assistant tool loop", () => { + it("feeds the complete skill and schema into the next streamed model step", async () => { + const streamErrors = vi.fn(); + const model = new MockLanguageModelV4({ + doStream: [ + { + stream: simulateReadableStream({ + chunks: [ + { type: "stream-start" as const, warnings: [] }, + { + type: "tool-call" as const, + toolCallId: "skill-call", + toolName: "customWidget_getSkill", + input: "{}", + }, + { + type: "tool-call" as const, + toolCallId: "schema-call", + toolName: "customWidget_schema", + input: "{}", + }, + finish("tool-calls"), + ], + chunkDelayInMs: null, + }), + }, + { + stream: simulateReadableStream({ + chunks: [ + { type: "stream-start" as const, warnings: [] }, + { type: "text-start" as const, id: "answer" }, + { type: "text-delta" as const, id: "answer", delta: "Custom Widget resources loaded." }, + { type: "text-end" as const, id: "answer" }, + finish("stop"), + ], + chunkDelayInMs: null, + }), + }, + ], + }); + + const noInputSchema = jsonSchema({ type: "object", properties: {}, additionalProperties: false }); + const result = streamText({ + model, + prompt: "Create a Custom Widget.", + tools: { + customWidget_getSkill: tool({ + inputSchema: noInputSchema, + execute: () => parse(stringify(getCustomWidgetSkill())), + }), + customWidget_schema: tool({ + inputSchema: noInputSchema, + execute: () => parse(stringify(getCustomWidgetJsonSchema())), + }), + }, + stopWhen: stepCountIs(2), + onError: streamErrors, + }); + + await expect(result.text).resolves.toBe("Custom Widget resources loaded."); + expect(streamErrors).not.toHaveBeenCalled(); + expect(model.doStreamCalls).toHaveLength(2); + + const nextStepPrompt = model.doStreamCalls[1]?.prompt; + expect(() => JSON.stringify(nextStepPrompt)).not.toThrow(); + const serializedPrompt = JSON.stringify(nextStepPrompt); + expect(serializedPrompt).toContain('"name":"homarr-custom-widget"'); + expect(serializedPrompt).toContain('"references/schema.md"'); + expect(serializedPrompt).toContain('"references/runtime.md"'); + expect(serializedPrompt).toContain('"references/security.md"'); + expect(serializedPrompt).toContain('"title":"Homarr Custom JSX v2 widget"'); + expect(serializedPrompt).toContain('"$schema"'); + }); + + it("promotes trusted context when implicit authoring starts with a catalog call", async () => { + const model = new MockLanguageModelV4({ + doStream: [ + { + stream: simulateReadableStream({ + chunks: [ + { type: "stream-start" as const, warnings: [] }, + { + type: "tool-call" as const, + toolCallId: "catalog-call", + toolName: "customWidget_getComponentCatalog", + input: "{}", + }, + finish("tool-calls"), + ], + chunkDelayInMs: null, + }), + }, + { + stream: simulateReadableStream({ + chunks: [ + { type: "stream-start" as const, warnings: [] }, + { type: "text-start" as const, id: "answer" }, + { type: "text-delta" as const, id: "answer", delta: "Authoring can continue." }, + { type: "text-end" as const, id: "answer" }, + finish("stop"), + ], + chunkDelayInMs: null, + }), + }, + ], + }); + + const noInputSchema = jsonSchema({ type: "object", properties: {}, additionalProperties: false }); + const baseInstructions = "Base assistant instructions. Untrusted mention: Trusted Custom Widget authoring context."; + const prepareDynamicCustomWidgetContext = createCustomWidgetDynamicContextController({ + isAdmin: true, + baseInstructions, + availableToolNames: ["customWidget_getSkill", "customWidget_schema", "customWidget_getComponentCatalog"] as const, + }); + const result = streamText({ + model, + instructions: baseInstructions, + prompt: "Build a live fixtures card.", + tools: { + customWidget_getSkill: tool({ + inputSchema: noInputSchema, + execute: () => parse(stringify(getCustomWidgetSkill())), + }), + customWidget_schema: tool({ + inputSchema: noInputSchema, + execute: () => parse(stringify(getCustomWidgetJsonSchema())), + }), + customWidget_getComponentCatalog: tool({ + inputSchema: noInputSchema, + execute: () => ({ components: [] }), + }), + }, + prepareStep: ({ instructions, messages, steps }) => + prepareDynamicCustomWidgetContext({ instructions, messages, steps }), + stopWhen: stepCountIs(2), + }); + + await expect(result.text).resolves.toBe("Authoring can continue."); + expect(model.doStreamCalls).toHaveLength(2); + + const nextProviderCall = model.doStreamCalls[1]; + const serializedToolMessages = JSON.stringify( + nextProviderCall?.prompt.filter((message) => message.role === "tool"), + ); + expect(serializedToolMessages).toContain("customWidget_getComponentCatalog"); + expect(serializedToolMessages).not.toContain("customWidget_getSkill"); + expect(serializedToolMessages).not.toContain("customWidget_schema"); + const systemContent = nextProviderCall?.prompt.find((message) => message.role === "system")?.content; + expect(systemContent).toEqual(expect.any(String)); + expect(systemContent).toContain("# Bundled file: references/schema.md"); + expect(systemContent).toContain('"title": "Homarr Custom JSX v2 widget"'); + expect(nextProviderCall?.tools?.map((entry) => entry.name)).toEqual(["customWidget_getComponentCatalog"]); + }); + + it("repairs malformed no-input provider arguments and executes the resource tool", async () => { + const execute = vi.fn((_input: unknown) => ({ name: "homarr-custom-widget" })); + const model = new MockLanguageModelV4({ + doStream: [ + { + stream: simulateReadableStream({ + chunks: [ + { type: "stream-start" as const, warnings: [] }, + { + type: "tool-call" as const, + toolCallId: "skill-call", + toolName: "customWidget_getSkill", + input: "{", + }, + finish("tool-calls"), + ], + chunkDelayInMs: null, + }), + }, + { + stream: simulateReadableStream({ + chunks: [ + { type: "stream-start" as const, warnings: [] }, + { type: "text-start" as const, id: "answer" }, + { type: "text-delta" as const, id: "answer", delta: "Repaired." }, + { type: "text-end" as const, id: "answer" }, + finish("stop"), + ], + chunkDelayInMs: null, + }), + }, + ], + }); + + const result = streamText({ + model, + prompt: "Load the skill.", + tools: { + customWidget_getSkill: tool({ + inputSchema: jsonSchema({ type: "object", properties: {}, additionalProperties: false }), + execute, + }), + }, + experimental_repairToolCall: ({ toolCall }) => Promise.resolve(repairCustomWidgetToolInput(toolCall)), + stopWhen: stepCountIs(2), + }); + + await expect(result.text).resolves.toBe("Repaired."); + expect(execute).toHaveBeenCalledTimes(1); + expect(execute.mock.calls[0]?.[0]).toEqual({}); + expect(model.doStreamCalls[1]?.prompt.some((message) => message.role === "tool")).toBe(true); + }); +}); diff --git a/apps/nextjs/src/app/api/assistant/chat/route.ts b/apps/nextjs/src/app/api/assistant/chat/route.ts index 659ef8061..835378975 100644 --- a/apps/nextjs/src/app/api/assistant/chat/route.ts +++ b/apps/nextjs/src/app/api/assistant/chat/route.ts @@ -57,6 +57,11 @@ import { assistantExecutionPolicy } from "./assistant-execution-policy"; import { getAssistantStreamErrorMessage } from "./assistant-stream-error"; import { getSafeAssistantToolError } from "./assistant-tool-error"; import { repairCustomWidgetToolInput } from "./assistant-tool-input-repair"; +import { + createCustomWidgetDynamicContextController, + getCustomWidgetAuthoringContext, + prunePreloadedCustomWidgetModelMessages, +} from "./custom-widget-authoring-context"; import { customWidgetAssistantInstructions, getForcedAssistantToolName, @@ -343,8 +348,9 @@ const getProcedureTypeMap = () => { ); }; +const waitForDemoChunkAsync = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + const createDemoAssistantResponse = (request: z.infer) => { - const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); const text = "This is the Homarr demo assistant. It is running in preview/mock mode, so it responds with a canned answer instead of calling a real language model. In a real deployment, an administrator configures a provider and a model for full conversations."; const responseId = crypto.randomUUID(); @@ -358,7 +364,7 @@ const createDemoAssistantResponse = (request: z.infer) => writer.write({ type: "text-start", id: partId }); for (const chunk of chunkTexts) { writer.write({ type: "text-delta", id: partId, delta: chunk }); - await delay(18); + await waitForDemoChunkAsync(18); } writer.write({ type: "text-end", id: partId }); writer.write({ type: "finish", finishReason: "stop" }); @@ -401,6 +407,7 @@ export async function POST(request: Request) { { status: 413 }, ); } + const incomingMessages = parsed.data.messages as UIMessage[]; const configuration = await db.query.assistantConfigurations.findFirst({ where: eq(assistantConfigurations.id, "default"), @@ -469,9 +476,12 @@ export async function POST(request: Request) { ) { return Response.json({ error: "The selected model does not support image input." }, { status: 400 }); } + const canAuthorCustomWidgets = session.user.permissions.includes("admin"); + const customWidgetAuthoringContext = getCustomWidgetAuthoringContext(incomingMessages, canAuthorCustomWidgets); const caller = mcpRouter.createCaller(context); const procedureTypes = getProcedureTypeMap(); - const mcpTools = extractMcpTools(); + const omittedCustomWidgetTools = new Set(customWidgetAuthoringContext.omittedToolNames); + const mcpTools = extractMcpTools().filter((mcpTool) => !omittedCustomWidgetTools.has(mcpTool.name)); const homarrTools = Object.fromEntries( mcpTools.map((mcpTool) => { @@ -535,7 +545,7 @@ export async function POST(request: Request) { }), ) satisfies ToolSet; const availableTools = { ...homarrTools, ...frontendTools }; - const forcedToolName = getForcedAssistantToolName(parsed.data.messages as UIMessage[]); + const forcedToolName = getForcedAssistantToolName(incomingMessages); const openRouterServerToolsEnabled = configuration.webSearchEnabled && assistantProviderCanUseOpenRouterServerTools(configuration.provider); @@ -571,27 +581,39 @@ export async function POST(request: Request) { let hasReportedCost = false; let upstreamCost = 0; let hasUpstreamCost = false; + const initialModelMessages = await convertToModelMessages(prepareMessagesForModel(incomingMessages)); + const modelMessages = + customWidgetAuthoringContext.systemContext.length > 0 + ? prunePreloadedCustomWidgetModelMessages(initialModelMessages) + : initialModelMessages; + const baseInstructions = `${assistantInstructions}${customWidgetAssistantInstructions}${openRouterServerToolsEnabled ? webSearchInstructions : ""}${mentionContext}${customWidgetAuthoringContext.systemContext}`; + const prepareDynamicCustomWidgetContext = createCustomWidgetDynamicContextController({ + isAdmin: canAuthorCustomWidgets, + baseInstructions, + availableToolNames: Object.keys(availableTools), + }); const result = streamText({ model: provider(modelId), - instructions: `${assistantInstructions}${customWidgetAssistantInstructions}${openRouterServerToolsEnabled ? webSearchInstructions : ""}${mentionContext}`, - messages: await convertToModelMessages(prepareMessagesForModel(parsed.data.messages as UIMessage[])), + instructions: baseInstructions, + messages: modelMessages, tools: availableTools, - prepareStep: ({ stepNumber, steps, responseMessages }) => { + prepareStep: ({ instructions, messages, stepNumber, steps, responseMessages }) => { + const dynamicStepOverrides = prepareDynamicCustomWidgetContext({ instructions, messages, steps }); + const stepOverrides = dynamicStepOverrides ?? {}; if (stepNumber === 0 && forcedToolName !== undefined && forcedToolName in availableTools) { return { + ...stepOverrides, activeTools: [forcedToolName], toolChoice: { type: "tool", toolName: forcedToolName }, }; } - const requiredToolNames = getRequiredAssistantToolNames( - parsed.data.messages as UIMessage[], - steps, - responseMessages, - ).filter((toolName) => toolName in availableTools); + const requiredToolNames = getRequiredAssistantToolNames(incomingMessages, steps, responseMessages).filter( + (toolName) => toolName in availableTools, + ); if (requiredToolNames.length > 0) { - return { activeTools: requiredToolNames, toolChoice: "required" }; + return { ...stepOverrides, activeTools: requiredToolNames, toolChoice: "required" }; } - return undefined; + return dynamicStepOverrides; }, stopWhen: stepCountIs(assistantExecutionPolicy.maxSteps), abortSignal: request.signal,