diff --git a/apps/docs/docs/management/mcp.mdx b/apps/docs/docs/management/mcp.mdx index 606be3554..2b0de0f5e 100644 --- a/apps/docs/docs/management/mcp.mdx +++ b/apps/docs/docs/management/mcp.mdx @@ -201,8 +201,8 @@ Load the installed skill before authoring. `customWidget_getSkill` returns its c reference index. Load a named schema, runtime, or security reference with `customWidget_getReference` only when the design needs it. Read the live JSON Schema or component resources when their detail is required by the design. -1. Call `customWidget_getSkill`. In Homarr Assistant, use `homarr_findTools` to activate later authoring tools only when - they are needed. +1. Call `customWidget_getSkill`. Homarr Assistant stages the next typed authoring tools automatically as the widget + moves through context, validation, preview, evidence, and persistence phases. 2. Load the live schema and applicable named references on demand. Prefer `customWidget_findComponents` for a small release-matched result set; reserve the full catalog for broad exploration. Fetch only relevant installed component documentation with `customWidget_getComponents` (or `customWidget_getComponent` for one repair) and, when useful, diff --git a/apps/nextjs/src/app/api/assistant/chat/assistant-execution-policy.spec.ts b/apps/nextjs/src/app/api/assistant/chat/assistant-execution-policy.spec.ts index fcb6226f1..1f468faaa 100644 --- a/apps/nextjs/src/app/api/assistant/chat/assistant-execution-policy.spec.ts +++ b/apps/nextjs/src/app/api/assistant/chat/assistant-execution-policy.spec.ts @@ -22,7 +22,7 @@ describe("createCustomWidgetToolStepGate", () => { gate.begin(0); expect(gate.claim("customWidget_getSkill")).toBe(true); expect(gate.claim("web_search")).toBe(true); - expect(gate.claim("homarr_findTools")).toBe(true); + expect(gate.claim("homarr_enableToolGroups")).toBe(true); expect(gate.claim("customWidget_getReference")).toBe(true); expect(gate.claim("customWidget_getComponent")).toBe(true); expect(gate.claim("customWidget_getComponent")).toBe(false); @@ -30,7 +30,7 @@ describe("createCustomWidgetToolStepGate", () => { gate.begin(1); expect(gate.claim("customWidget_validateTemplate")).toBe(true); - expect(gate.claim("homarr_findTools")).toBe(false); + expect(gate.claim("homarr_enableToolGroups")).toBe(false); expect(gate.claim("customWidget_previewCreate")).toBe(false); gate.begin(2); diff --git a/apps/nextjs/src/app/api/assistant/chat/assistant-message-conversion.spec.ts b/apps/nextjs/src/app/api/assistant/chat/assistant-message-conversion.spec.ts index e3260eb55..2b81f3b4b 100644 --- a/apps/nextjs/src/app/api/assistant/chat/assistant-message-conversion.spec.ts +++ b/apps/nextjs/src/app/api/assistant/chat/assistant-message-conversion.spec.ts @@ -1,10 +1,7 @@ import { describe, expect, test } from "vitest"; import type { ModelMessage, UIMessage } from "ai"; -import { - compactAssistantStepMessages, - convertAssistantMessagesToModelMessages, -} from "./assistant-message-conversion"; +import { compactAssistantStepMessages, convertAssistantMessagesToModelMessages } from "./assistant-message-conversion"; describe("convertAssistantMessagesToModelMessages", () => { test("drops an interrupted tool call while preserving completed tool results", async () => { @@ -205,8 +202,8 @@ describe("convertAssistantMessagesToModelMessages", () => { { type: "tool-call", toolCallId: "regular-tool", - toolName: "homarr_findTools", - input: { query: "boards" }, + toolName: "homarr_enableToolGroups", + input: { groups: ["board"] }, }, ], }, @@ -228,8 +225,8 @@ describe("convertAssistantMessagesToModelMessages", () => { { type: "tool-result", toolCallId: "regular-tool", - toolName: "homarr_findTools", - output: { type: "json", value: { matches: [] } }, + toolName: "homarr_enableToolGroups", + output: { type: "json", value: { enabledGroups: ["board"] } }, }, ], }, diff --git a/apps/nextjs/src/app/api/assistant/chat/assistant-tool-discovery.spec.ts b/apps/nextjs/src/app/api/assistant/chat/assistant-tool-discovery.spec.ts deleted file mode 100644 index a1ec765c7..000000000 --- a/apps/nextjs/src/app/api/assistant/chat/assistant-tool-discovery.spec.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { describe, expect, test } from "vitest"; -import { z } from "zod/v4"; - -import type { McpTool } from "@homarr/api/mcp"; - -import { filterAssistantMcpToolMatches, findAssistantMcpTools } from "./assistant-tool-discovery"; - -const createTool = (name: string, description: string, pathInRouter = name.split("_")): McpTool => ({ - name, - description, - pathInRouter, - inputSchema: { type: "object", properties: {} }, - inputValidator: z.object({}), -}); - -describe("Assistant tool discovery", () => { - const tools = [ - createTool("customWidget_previewCreate", "Validate a complete Custom Widget and create its preview."), - createTool("customWidget_previewQuery", "Test a query from a Custom Widget preview."), - createTool("customWidget_createFromPreview", "Persist the exact tested Custom Widget preview."), - createTool("mediaRequests_search", "Search Seerr media requests."), - createTool("board_getAllBoards", "List Homarr boards."), - ]; - - test("finds only the small relevant lifecycle subset", () => { - expect(findAssistantMcpTools(tools, "custom widget preview and persist", 3).map(({ name }) => name)).toEqual([ - "customWidget_createFromPreview", - "customWidget_previewCreate", - "customWidget_previewQuery", - ]); - }); - - test("uses service aliases without loading unrelated schemas", () => { - expect(findAssistantMcpTools(tools, "Seerr movie requests").map(({ name }) => name)).toEqual([ - "mediaRequests_search", - ]); - }); - - test("returns no tools when a query contains only ignored terms", () => { - expect(findAssistantMcpTools(tools, "a an and the to")).toEqual([]); - }); - - test("keeps heavyweight authoring resources hidden unless explicitly requested", () => { - const matches = [ - createTool("customWidget_getComponentCatalog", "Get the complete component catalog."), - createTool("customWidget_schema", "Get the complete JSON Schema."), - createTool("customWidget_findComponents", "Find focused components."), - ]; - - expect(filterAssistantMcpToolMatches(matches, "custom widget", true).map(({ name }) => name)).toEqual([ - "customWidget_findComponents", - ]); - expect(filterAssistantMcpToolMatches(matches, "full component catalog and live schema", true).map(({ name }) => - name, - )).toEqual(["customWidget_getComponentCatalog", "customWidget_findComponents"]); - expect(filterAssistantMcpToolMatches(matches, "custom widget", false)).toEqual(matches); - }); -}); diff --git a/apps/nextjs/src/app/api/assistant/chat/assistant-tool-discovery.ts b/apps/nextjs/src/app/api/assistant/chat/assistant-tool-discovery.ts deleted file mode 100644 index e96d7827e..000000000 --- a/apps/nextjs/src/app/api/assistant/chat/assistant-tool-discovery.ts +++ /dev/null @@ -1,110 +0,0 @@ -import type { UIMessage } from "ai"; - -import type { McpTool } from "@homarr/api/mcp"; - -const ignoredSearchTerms = new Set([ - "a", - "an", - "and", - "can", - "create", - "for", - "from", - "get", - "in", - "list", - "make", - "my", - "of", - "on", - "please", - "the", - "to", - "want", - "with", -]); - -const searchTermAliases: Record = { - account: ["user"], - dashboard: ["board"], - k8s: ["kubernetes"], - movie: ["media", "radarr", "request"], - seerr: ["overseerr", "jellyseerr", "media", "request"], - show: ["media", "sonarr", "request"], - streaming: ["stream", "media", "server"], -}; - -const normalizeSearchText = (value: string) => - value - .replaceAll(/([a-z\d])([A-Z])/gu, "$1 $2") - .normalize("NFKD") - .toLowerCase() - .replaceAll(/[^a-z\d]+/gu, " ") - .trim(); - -const getSearchTerms = (query: string) => { - const terms = normalizeSearchText(query) - .split(" ") - .filter((term) => term.length > 1 && !ignoredSearchTerms.has(term)); - return [...new Set(terms.flatMap((term) => [term, ...(searchTermAliases[term] ?? [])]))]; -}; - -const scoreTool = (tool: McpTool, query: string, terms: readonly string[]) => { - const normalizedName = normalizeSearchText(tool.name); - const normalizedPath = normalizeSearchText(tool.pathInRouter.join(" ")); - const normalizedDescription = normalizeSearchText(tool.description); - const searchable = `${normalizedName} ${normalizedPath} ${normalizedDescription}`; - let score = 0; - if (normalizedName.includes(query)) score += 40; - if (normalizedDescription.includes(query)) score += 20; - for (const term of terms) { - if (normalizedName.split(" ").includes(term)) score += 12; - else if (normalizedName.includes(term)) score += 7; - if (normalizedPath.split(" ").includes(term)) score += 6; - if (normalizedDescription.split(" ").includes(term)) score += 4; - else if (searchable.includes(term)) score += 1; - } - return score; -}; - -export const findAssistantMcpTools = (tools: readonly McpTool[], query: string, limit = 8) => { - const normalizedQuery = normalizeSearchText(query); - const terms = getSearchTerms(query); - if (!normalizedQuery || terms.length === 0) return []; - - return tools - .map((tool) => ({ tool, score: scoreTool(tool, normalizedQuery, terms) })) - .filter(({ score }) => score > 0) - .toSorted((left, right) => right.score - left.score || left.tool.name.localeCompare(right.tool.name)) - .slice(0, limit) - .map(({ tool }) => tool); -}; - -export const filterAssistantMcpToolMatches = >( - tools: readonly TTool[], - query: string, - customWidgetAuthoringActive: boolean, -) => { - if (!customWidgetAuthoringActive) return [...tools]; - const normalizedQuery = normalizeSearchText(query); - const requestsCompleteCatalog = - normalizedQuery.includes("all components") || - ["full component catalog", "complete component catalog"].some((phrase) => normalizedQuery.includes(phrase)); - return tools.filter((tool) => { - if (tool.name === "customWidget_getComponentCatalog") return requestsCompleteCatalog; - if (tool.name === "customWidget_schema") return false; - return true; - }); -}; - -export const getLatestAssistantUserText = (messages: UIMessage[]) => { - const latestUserMessage = messages.toReversed().find((message) => message.role === "user"); - if (!latestUserMessage) return ""; - return latestUserMessage.parts - .flatMap((part) => { - if (typeof part !== "object" || part === null || !("type" in part) || part.type !== "text") return []; - if (!("text" in part) || typeof part.text !== "string") return []; - return [part.text]; - }) - .join("\n"); -}; diff --git a/apps/nextjs/src/app/api/assistant/chat/assistant-tool-groups.spec.ts b/apps/nextjs/src/app/api/assistant/chat/assistant-tool-groups.spec.ts new file mode 100644 index 000000000..f92ec78be --- /dev/null +++ b/apps/nextjs/src/app/api/assistant/chat/assistant-tool-groups.spec.ts @@ -0,0 +1,40 @@ +import { describe, expect, test } from "vitest"; + +import { createAssistantMcpToolGroups } from "./assistant-tool-groups"; + +const createTool = (name: string, group: string) => ({ + name, + description: `${name} description`, + pathInRouter: [group, name], +}); + +describe("Assistant MCP tool groups", () => { + test("derives deterministic groups from router paths", () => { + const groups = createAssistantMcpToolGroups([ + createTool("board_search", "board"), + createTool("mediaRequests_search", "mediaRequests"), + createTool("board_getAll", "board"), + ]); + + expect(groups.ids).toEqual(["board", "mediaRequests"]); + expect(groups.resolve(["board"])[0]?.tools.map(({ name }) => name)).toEqual(["board_getAll", "board_search"]); + }); + + test("deduplicates requested groups while preserving request order", () => { + const groups = createAssistantMcpToolGroups([ + createTool("board_getAll", "board"), + createTool("docker_getContainers", "docker"), + ]); + + expect(groups.resolve(["docker", "board", "docker"]).map(({ id }) => id)).toEqual(["docker", "board"]); + }); + + test("rejects tools without a router group and unknown group IDs", () => { + expect(() => createAssistantMcpToolGroups([{ ...createTool("info_get", "info"), pathInRouter: [] }])).toThrow( + "has no router group", + ); + + const groups = createAssistantMcpToolGroups([createTool("info_get", "info")]); + expect(() => groups.resolve(["missing"])).toThrow("Unknown Assistant MCP tool group 'missing'"); + }); +}); diff --git a/apps/nextjs/src/app/api/assistant/chat/assistant-tool-groups.ts b/apps/nextjs/src/app/api/assistant/chat/assistant-tool-groups.ts new file mode 100644 index 000000000..4578b74c5 --- /dev/null +++ b/apps/nextjs/src/app/api/assistant/chat/assistant-tool-groups.ts @@ -0,0 +1,38 @@ +import type { McpTool } from "@homarr/api/mcp"; + +type AssistantMcpTool = Pick; + +export interface AssistantMcpToolGroup { + id: string; + tools: readonly TTool[]; +} + +export const createAssistantMcpToolGroups = (tools: readonly TTool[]) => { + const toolsByGroup = new Map(); + for (const tool of tools) { + const groupId = tool.pathInRouter[0]; + if (!groupId) throw new Error(`Assistant MCP tool '${tool.name}' has no router group`); + const groupedTools = toolsByGroup.get(groupId) ?? []; + groupedTools.push(tool); + toolsByGroup.set(groupId, groupedTools); + } + + const groups = [...toolsByGroup.entries()] + .toSorted(([left], [right]) => left.localeCompare(right)) + .map(([id, groupedTools]) => ({ + id, + tools: groupedTools.toSorted((left, right) => left.name.localeCompare(right.name)), + })); + const groupsById = new Map(groups.map((group) => [group.id, group])); + + return { + ids: groups.map(({ id }) => id), + resolve(groupIds: readonly string[]): AssistantMcpToolGroup[] { + return [...new Set(groupIds)].map((groupId) => { + const group = groupsById.get(groupId); + if (!group) throw new Error(`Unknown Assistant MCP tool group '${groupId}'`); + return group; + }); + }, + }; +}; diff --git a/apps/nextjs/src/app/api/assistant/chat/assistant-tool-policy.spec.ts b/apps/nextjs/src/app/api/assistant/chat/assistant-tool-policy.spec.ts index fa2dcc5c9..2a273a068 100644 --- a/apps/nextjs/src/app/api/assistant/chat/assistant-tool-policy.spec.ts +++ b/apps/nextjs/src/app/api/assistant/chat/assistant-tool-policy.spec.ts @@ -30,7 +30,7 @@ describe("customWidgetAssistantInstructions", () => { test("loads authoring resources lazily and verifies every final preview", () => { expect(customWidgetAssistantInstructions.length).toBeLessThan(4_100); expect(customWidgetAssistantInstructions).toContain("customWidget_getSkill"); - expect(customWidgetAssistantInstructions).toContain("homarr_findTools"); + expect(customWidgetAssistantInstructions).toContain("staged by the authoring lifecycle"); expect(customWidgetAssistantInstructions).toContain("task-needed"); expect(customWidgetAssistantInstructions).toContain("Reuse loaded context"); expect(customWidgetAssistantInstructions).toContain("customWidget_findComponents"); diff --git a/apps/nextjs/src/app/api/assistant/chat/assistant-tool-policy.ts b/apps/nextjs/src/app/api/assistant/chat/assistant-tool-policy.ts index fc5813bf6..a2c2c42dc 100644 --- a/apps/nextjs/src/app/api/assistant/chat/assistant-tool-policy.ts +++ b/apps/nextjs/src/app/api/assistant/chat/assistant-tool-policy.ts @@ -3,13 +3,13 @@ import type { UIMessage } from "ai"; import { CUSTOM_WIDGET_ASSISTANT_POLICY, - CUSTOM_WIDGET_LAZY_TOOL_DISCOVERY_INSTRUCTION, + CUSTOM_WIDGET_TOOL_STAGING_INSTRUCTION, } from "@homarr/custom-widgets/authoring-prompt"; const mutationApprovalInstruction = "Uses Homarr's native approval UI; call when inputs are ready without separate prose confirmation."; -export const customWidgetAssistantInstructions = `\n\n${CUSTOM_WIDGET_LAZY_TOOL_DISCOVERY_INSTRUCTION}\n\n${CUSTOM_WIDGET_ASSISTANT_POLICY}`; +export const customWidgetAssistantInstructions = `\n\n${CUSTOM_WIDGET_TOOL_STAGING_INSTRUCTION}\n\n${CUSTOM_WIDGET_ASSISTANT_POLICY}`; export const withAssistantToolPolicy = (description: string | undefined, requiresApproval: boolean) => { if (!requiresApproval) return description; diff --git a/apps/nextjs/src/app/api/assistant/chat/custom-widget-authoring-context.spec.ts b/apps/nextjs/src/app/api/assistant/chat/custom-widget-authoring-context.spec.ts index aa922dfe3..093e3ce8e 100644 --- a/apps/nextjs/src/app/api/assistant/chat/custom-widget-authoring-context.spec.ts +++ b/apps/nextjs/src/app/api/assistant/chat/custom-widget-authoring-context.spec.ts @@ -26,6 +26,13 @@ describe("Custom Widget authoring context", () => { expect(needsCustomWidgetAuthoringContext([userMessage(text)])).toBe(true); }); + test.each(["List my custom widgets", "Delete a custom widget", "Explain custom widgets"])( + "leaves management intent to the general Custom Widget MCP group: %s", + (text) => { + expect(needsCustomWidgetAuthoringContext([userMessage(text)])).toBe(false); + }, + ); + test("continues after a Custom Widget tool call", () => { const messages: UIMessage[] = [ userMessage("Continue"), @@ -68,6 +75,8 @@ describe("Custom Widget authoring context", () => { "customWidget_schema", "customWidget_getComponentCatalog", "customWidget_getReference", + "customWidget_findComponents", + "customWidget_getComponents", "customWidget_getComponent", "customWidget_getSharedProps", "customWidget_getExample", @@ -78,7 +87,6 @@ describe("Custom Widget authoring context", () => { expect(getActiveCustomWidgetToolNames(tools, [userMessage("Create two custom widgets")], true)).toEqual([ "customWidget_getSkill", - "customWidget_validateTemplate", ]); expect(getActiveCustomWidgetToolNames(tools, [userMessage("Create a custom widget")], false)).toEqual([]); }); @@ -100,11 +108,7 @@ describe("Custom Widget authoring context", () => { getCustomWidgetPhaseToolNames(tools, [ { toolResults: [{ toolName: "customWidget_validateTemplate", output: { valid: true } }] }, ]), - ).toEqual([ - "customWidget_validateTemplate", - "customWidget_previewCreate", - "customWidget_previewReviseTemplate", - ]); + ).toEqual(["customWidget_validateTemplate", "customWidget_previewCreate", "customWidget_previewReviseTemplate"]); expect( getCustomWidgetPhaseToolNames(tools, [ { toolResults: [{ toolName: "customWidget_previewCreate", output: { success: true } }] }, @@ -135,6 +139,30 @@ describe("Custom Widget authoring context", () => { ).toBeNull(); }); + test("exposes focused context tools only after the skill entrypoint is loaded", () => { + const tools = [ + "customWidget_getSkill", + "customWidget_getReference", + "customWidget_findComponents", + "customWidget_getComponents", + "customWidget_getExample", + "customWidget_validateTemplate", + "customWidget_previewCreate", + ]; + + expect( + getCustomWidgetPhaseToolNames(tools, [ + { toolResults: [{ toolName: "customWidget_getSkill", output: { content: "skill" } }] }, + ]), + ).toEqual([ + "customWidget_getReference", + "customWidget_findComponents", + "customWidget_getComponents", + "customWidget_getExample", + "customWidget_validateTemplate", + ]); + }); + test("keeps one concrete component repair path open for validation warnings", () => { const tools = [ "customWidget_findComponents", @@ -179,16 +207,10 @@ describe("Custom Widget authoring context", () => { ], }, { - toolResults: [ - { toolName: "customWidget_getComponent", output: { name: "RefreshButton", props: [] } }, - ], + toolResults: [{ toolName: "customWidget_getComponent", output: { name: "RefreshButton", props: [] } }], }, ]), - ).toEqual([ - "customWidget_validateTemplate", - "customWidget_previewCreate", - "customWidget_previewReviseTemplate", - ]); + ).toEqual(["customWidget_validateTemplate", "customWidget_previewCreate", "customWidget_previewReviseTemplate"]); }); test("keeps evidence tools until every preview request succeeds, then allows correction or persistence", () => { @@ -257,11 +279,7 @@ describe("Custom Widget authoring context", () => { ...completeEvidence, { toolResults: [{ toolName: "customWidget_validateTemplate", output: { valid: true } }] }, ]), - ).toEqual([ - "customWidget_validateTemplate", - "customWidget_previewCreate", - "customWidget_previewReviseTemplate", - ]); + ).toEqual(["customWidget_validateTemplate", "customWidget_previewCreate", "customWidget_previewReviseTemplate"]); const revisedPreview = [ ...completeEvidence, @@ -310,9 +328,7 @@ describe("Custom Widget authoring context", () => { ], }, ]; - expect(getCustomWidgetPhaseToolNames(tools, completeRevisedEvidence)).toEqual([ - "customWidget_createFromPreview", - ]); + expect(getCustomWidgetPhaseToolNames(tools, completeRevisedEvidence)).toEqual(["customWidget_createFromPreview"]); }); test("closes focused discovery after four searches until validation", () => { diff --git a/apps/nextjs/src/app/api/assistant/chat/custom-widget-authoring-context.ts b/apps/nextjs/src/app/api/assistant/chat/custom-widget-authoring-context.ts index 5bd9b7f0c..a99ab821e 100644 --- a/apps/nextjs/src/app/api/assistant/chat/custom-widget-authoring-context.ts +++ b/apps/nextjs/src/app/api/assistant/chat/custom-widget-authoring-context.ts @@ -5,9 +5,9 @@ import { getCustomWidgetPhaseToolNames } from "@homarr/custom-widgets/core"; export { getCustomWidgetPhaseToolNames }; const customWidgetIntentPattern = - /(?:\bcustom[\s-]+widgets?\b|\bcustom\s+jsx\b|\bhomarr-custom-widget-v\d+\b|\bcustomWidget_[A-Za-z\d_]+\b|\b(?:build|create|design|make)\b[^\n]{0,80}\bwidgets?\s+(?:for|using|with)\b|\b(?:i|we)\s+(?:need|want)\b[^\n]{0,60}\bwidgets?\s+(?:for|using|with)\b)/iu; + /(?:\bcustom\s+jsx\b|\bhomarr-custom-widget-v\d+\b|\b(?:build|create|design|edit|fix|make|repair|update|validate)\b[^\n]{0,80}\bcustom[\s-]+widgets?\b|\b(?:build|create|design|make)\b[^\n]{0,80}\bwidgets?\s+(?:for|using|with)\b|\b(?:i|we)\s+(?:need|want)\b[^\n]{0,60}\bwidgets?\s+(?:for|using|with)\b)/iu; -const customWidgetBootstrapToolNames = new Set(["customWidget_getSkill", "customWidget_validateTemplate"]); +const customWidgetBootstrapToolNames = new Set(["customWidget_getSkill"]); const maxFocusedComponentSearchesPerPhase = 4; const customWidgetContextToolBudgets: Readonly> = { customWidget_findComponents: maxFocusedComponentSearchesPerPhase, diff --git a/apps/nextjs/src/app/api/assistant/chat/route.ts b/apps/nextjs/src/app/api/assistant/chat/route.ts index c55e2fff8..a694f709f 100644 --- a/apps/nextjs/src/app/api/assistant/chat/route.ts +++ b/apps/nextjs/src/app/api/assistant/chat/route.ts @@ -49,10 +49,7 @@ import { sanitizeAttachmentFilename, } from "./assistant-chat-input"; import { getAssistantModelLookupStatus } from "./assistant-model-lookup"; -import { - compactAssistantStepMessages, - convertAssistantMessagesToModelMessages, -} from "./assistant-message-conversion"; +import { compactAssistantStepMessages, convertAssistantMessagesToModelMessages } from "./assistant-message-conversion"; import { getOpenRouterWebSearchRequests, getOpenRouterWebSearchSources, @@ -77,11 +74,7 @@ import { getCustomWidgetPhaseToolNames, needsCustomWidgetAuthoringContext, } from "./custom-widget-authoring-context"; -import { - filterAssistantMcpToolMatches, - findAssistantMcpTools, - getLatestAssistantUserText, -} from "./assistant-tool-discovery"; +import { createAssistantMcpToolGroups } from "./assistant-tool-groups"; import { customWidgetAssistantInstructions, getForcedAssistantToolName, @@ -157,20 +150,12 @@ const requestSchema = z.object({ .optional(), }); -const assistantToolDiscoverySchema = z.object({ - query: z - .string() - .trim() - .min(2) - .max(240) - .describe("A concise Homarr capability or action, such as 'search media requests' or 'list Kubernetes pods'."), -}); - -const assistantToolDiscoveryName = "homarr_findTools"; +const assistantToolGroupActivationName = "homarr_enableToolGroups"; +const maxAssistantToolGroupsPerActivation = 4; const assistantInstructions = `You are Homarr Assistant, embedded in the user's self-hosted Homarr dashboard. -Use Homarr tools for live instance data or actions; never invent resources, IDs, state, or results. Only a small relevant tool set is initially visible. Call ${assistantToolDiscoveryName} with a concise capability when a needed Homarr tool is not visible, then use the activated tool. Follow each tool's description and use integration IDs returned by discovery tools. +Use Homarr tools for live instance data or actions; never invent resources, IDs, state, or results. Homarr tools are grouped by their MCP router namespace. Call ${assistantToolGroupActivationName} with the task-needed group when its typed tools are not visible, then use the activated tool. Follow each tool's description and use integration IDs returned by Homarr tools. Homarr permissions are authoritative. Explain denied access without suggesting a bypass. Read before changing when current state matters. Mutations use Homarr's native approval UI: when inputs are sufficient, call the mutation immediately and never ask for duplicate prose confirmation or retry a denial. Use ask_user only when a missing choice blocks the next action; do not end with a prose question expecting a reply. @@ -547,10 +532,7 @@ export async function POST(request: Request) { input !== null && !Array.isArray(input) ) { - executionInput = normalizeCustomWidgetLifecycleToolInput( - mcpTool.name, - input as Record, - ); + executionInput = normalizeCustomWidgetLifecycleToolInput(mcpTool.name, input as Record); } const contextRequestKey = getCustomWidgetContextRequestKey(mcpTool.name, executionInput); if (contextRequestKey !== null && loadedCustomWidgetContextRequests.has(contextRequestKey)) { @@ -621,45 +603,41 @@ export async function POST(request: Request) { ]; }), ) satisfies ToolSet; - const discoveredToolNames = new Set(); - const latestUserText = getLatestAssistantUserText(incomingMessages); - const automaticToolCandidates = customWidgetAuthoringActive + const groupedToolCandidates = customWidgetAuthoringActive ? mcpTools.filter((mcpTool) => !mcpTool.name.startsWith("customWidget_")) : mcpTools; - const automaticallySelectedTools = findAssistantMcpTools(automaticToolCandidates, latestUserText, 6).map( - ({ name }) => name, - ); - const toolDiscovery = tool({ + const assistantToolGroups = createAssistantMcpToolGroups(groupedToolCandidates); + const assistantToolGroupIds = assistantToolGroups.ids as [string, ...string[]]; + const assistantToolGroupActivationSchema = z.object({ + groups: z + .array(z.enum(assistantToolGroupIds)) + .min(1) + .max(maxAssistantToolGroupsPerActivation) + .describe("MCP router groups whose typed Homarr tools are needed for the current task."), + }); + const enabledToolGroupIds = new Set(); + const toolGroupActivation = tool({ description: - "Find Homarr tools by capability without loading the complete tool catalog. Matching tools become callable on the next step. Use a focused query and call again with a different capability when needed.", - inputSchema: jsonSchema(z.toJSONSchema(assistantToolDiscoverySchema) as Parameters[0]), + "Enable task-needed typed Homarr tools by MCP router group. Enabled groups remain available on later steps.", + inputSchema: jsonSchema(z.toJSONSchema(assistantToolGroupActivationSchema) as Parameters[0]), execute: (value) => { - if (customWidgetAuthoringActive && !customWidgetToolStepGate.claim(assistantToolDiscoveryName)) { + if (customWidgetAuthoringActive && !customWidgetToolStepGate.claim(assistantToolGroupActivationName)) { return { - error: - "A Custom Widget tool must run in its own model step. Use that result before calling another tool.", + error: "A Custom Widget tool must run in its own model step. Use that result before calling another tool.", }; } - const input = assistantToolDiscoverySchema.parse(value); - const matches = filterAssistantMcpToolMatches( - findAssistantMcpTools(mcpTools, input.query), - input.query, - customWidgetAuthoringActive, - ); - for (const match of matches) discoveredToolNames.add(match.name); + const input = assistantToolGroupActivationSchema.parse(value); + const groups = assistantToolGroups.resolve(input.groups); + for (const group of groups) enabledToolGroupIds.add(group.id); return { - query: input.query, - matches: matches.map((match) => ({ name: match.name, description: match.description })), - nextStep: - matches.length > 0 - ? "Use one or more matching tools now. Search again only for a different capability." - : "No matching Homarr tool was found. Refine the capability or explain that it is unavailable.", + enabledGroups: groups.map(({ id }) => id), + nextStep: "Use one or more enabled typed tools now. Enable another group only when the task needs it.", }; }, }); const availableTools: ToolSet = { ...homarrTools, - [assistantToolDiscoveryName]: toolDiscovery, + [assistantToolGroupActivationName]: toolGroupActivation, ...frontendTools, }; const frontendToolNames = Object.keys(frontendTools); @@ -669,14 +647,17 @@ export async function POST(request: Request) { canAuthorCustomWidgets, ); const getActiveToolNames = (steps: Parameters[1] = []) => { + const enabledToolNames = assistantToolGroups + .resolve([...enabledToolGroupIds]) + .flatMap((group) => group.tools.map(({ name }) => name)); const phaseToolNames = customWidgetAuthoringActive ? getCustomWidgetPhaseToolNames(Object.keys(homarrTools), steps) : null; - if (phaseToolNames) return [...frontendToolNames, ...phaseToolNames]; + if (phaseToolNames) return [...frontendToolNames, ...enabledToolNames, ...phaseToolNames]; return [ - assistantToolDiscoveryName, + assistantToolGroupActivationName, ...frontendToolNames, - ...new Set([...automaticallySelectedTools, ...discoveredToolNames, ...activeCustomWidgetToolNames]), + ...new Set([...enabledToolNames, ...activeCustomWidgetToolNames]), ]; }; const forcedToolName = getForcedAssistantToolName(incomingMessages); diff --git a/packages/custom-widgets/scripts/ai-assistant-evaluation.ts b/packages/custom-widgets/scripts/ai-assistant-evaluation.ts index d21ad0088..5b03110ee 100644 --- a/packages/custom-widgets/scripts/ai-assistant-evaluation.ts +++ b/packages/custom-widgets/scripts/ai-assistant-evaluation.ts @@ -27,7 +27,7 @@ import { selectSequentialCustomWidgetToolCalls, } from "../src/core/assistant-tool-step"; import type { HomarrCustomWidgetV2 } from "../src/core/custom-jsx-schema"; -import { CUSTOM_WIDGET_ASSISTANT_POLICY, CUSTOM_WIDGET_LAZY_TOOL_DISCOVERY_INSTRUCTION } from "../src/core/ai-prompt"; +import { CUSTOM_WIDGET_ASSISTANT_POLICY, CUSTOM_WIDGET_TOOL_STAGING_INSTRUCTION } from "../src/core/ai-prompt"; import { getCustomWidgetJsonSchema } from "../src/core/schema"; import { addCustomJsxDiagnosticSourceExcerpts, validateCustomJsxTemplate } from "../src/jsx"; import type { CustomWidgetAiEvaluationCase } from "./ai-evaluation-cases"; @@ -76,7 +76,6 @@ type OpenRouterMessage = const assistantEvaluationContextMaxCharacters = 48_000; const reloadableEvaluationToolNames = new Set([ - "homarr_findTools", "customWidget_schema", "customWidget_getComponentCatalog", "customWidget_findComponents", @@ -210,7 +209,6 @@ interface AssistantEvaluationToolCall { export interface AssistantAttemptState { calledTools: string[]; toolCalls: AssistantEvaluationToolCall[]; - activatedTools: Set; validatedTemplates: Set; previews: Map; completedPreviewSignatures: Set; @@ -253,15 +251,6 @@ const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null && !Array.isArray(value); export const customWidgetAssistantEvaluationToolDefinitions: ToolDefinition[] = [ - { - type: "function", - function: { - name: "homarr_findTools", - description: - "Find Homarr tools by capability without loading every tool schema. Matching tools become callable on the next step.", - parameters: objectSchema({ query: { type: "string", minLength: 2, maxLength: 240 } }, ["query"]), - }, - }, { type: "function", function: { @@ -453,12 +442,7 @@ export const customWidgetAssistantEvaluationToolDefinitions: ToolDefinition[] = }, ]; -const initiallyActiveAssistantEvaluationTools = new Set([ - "homarr_findTools", - "web_search", - "customWidget_getSkill", - "customWidget_validateTemplate", -]); +const initiallyActiveAssistantEvaluationTools = new Set(["web_search", "customWidget_getSkill"]); const maxFocusedComponentSearchesPerPhase = 4; const customWidgetContextToolBudgets: Readonly> = { customWidget_findComponents: maxFocusedComponentSearchesPerPhase, @@ -468,46 +452,6 @@ const customWidgetContextToolBudgets: Readonly> = { customWidget_getExample: 1, }; -const normalizeToolSearchText = (value: string) => - value - .replaceAll(/([a-z\d])([A-Z])/gu, "$1 $2") - .toLowerCase() - .replaceAll(/[^a-z\d]+/gu, " ") - .trim(); - -const findAssistantEvaluationTools = (query: string) => { - const terms = [ - ...new Set( - normalizeToolSearchText(query) - .split(" ") - .filter((term) => term.length > 2), - ), - ]; - const normalizedQuery = normalizeToolSearchText(query); - const requestsCompleteCatalog = - normalizedQuery.includes("all components") || - normalizedQuery.includes("full component catalog") || - normalizedQuery.includes("complete component catalog"); - return customWidgetAssistantEvaluationToolDefinitions - .filter(({ function: definition }) => !initiallyActiveAssistantEvaluationTools.has(definition.name)) - .filter(({ function: definition }) => { - if (definition.name === "customWidget_getComponentCatalog") return requestsCompleteCatalog; - if (definition.name === "customWidget_schema") return false; - return true; - }) - .map((tool) => { - const searchable = normalizeToolSearchText(`${tool.function.name} ${tool.function.description}`); - const score = terms.reduce((total, term) => total + (searchable.includes(term) ? 1 : 0), 0); - return { tool, score }; - }) - .filter(({ score }) => score > 0) - .toSorted( - (left, right) => right.score - left.score || left.tool.function.name.localeCompare(right.tool.function.name), - ) - .slice(0, 8) - .map(({ tool }) => tool); -}; - export const getActiveAssistantEvaluationToolDefinitions = (state: AssistantAttemptState) => { const availableNames = customWidgetAssistantEvaluationToolDefinitions.map( ({ function: definition }) => definition.name, @@ -516,7 +460,7 @@ export const getActiveAssistantEvaluationToolDefinitions = (state: AssistantAtte toolResults: [{ toolName: toolCall.name, output: toolCall.output }], })); const phaseToolNames = getCustomWidgetPhaseToolNames(availableNames, steps); - const activeToolNames = new Set(phaseToolNames ?? state.activatedTools); + const activeToolNames = new Set(phaseToolNames ?? initiallyActiveAssistantEvaluationTools); return customWidgetAssistantEvaluationToolDefinitions.filter(({ function: definition }) => activeToolNames.has(definition.name), ); @@ -693,22 +637,6 @@ const executeAssistantEvaluationToolCore = ( if (contextBudget !== undefined && getContextToolCallsInCurrentPhase(state, name) >= contextBudget) { return getContextPhaseCompleteOutput(name); } - if (name === "homarr_findTools") { - const query = typeof input.query === "string" ? input.query : ""; - const matches = findAssistantEvaluationTools(query); - for (const match of matches) state.activatedTools.add(match.function.name); - return { - query, - matches: matches.map(({ function: definition }) => ({ - name: definition.name, - description: definition.description, - })), - nextStep: - matches.length > 0 - ? "Use the matching tools now. Search again only for a different capability." - : "No matching Homarr tool was found. Refine the capability.", - }; - } if (name === "web_search") { return { query: typeof input.query === "string" ? input.query : "", @@ -1067,7 +995,7 @@ export function getAssistantEvaluationEfficiencyIssues( if (skillCall && skillCall.outputCharacters > 10_000) { issues.push(`The compact skill entrypoint used ${skillCall.outputCharacters} output characters.`); } - const otherReusableContextTools = new Set(["homarr_findTools", "web_search"]); + const otherReusableContextTools = new Set(["web_search"]); const seenContextCalls = new Set(); for (const toolCall of state.toolCalls) { if (toolCall.phaseLimited) continue; @@ -1154,7 +1082,6 @@ export function createAssistantEvaluationState(): AssistantAttemptState { return { calledTools: [], toolCalls: [], - activatedTools: new Set(initiallyActiveAssistantEvaluationTools), validatedTemplates: new Set(), previews: new Map(), completedPreviewSignatures: new Set(), @@ -1260,7 +1187,7 @@ async function runAssistantAttempt(args: { const messages: OpenRouterMessage[] = [ { role: "system", - content: `${CUSTOM_WIDGET_LAZY_TOOL_DISCOVERY_INSTRUCTION}\n\n${CUSTOM_WIDGET_ASSISTANT_POLICY}\n\nThis is an unassisted tool-use evaluation. No tool will be forced for you. Complete and persist all ${getExpectedWidgetCount(args.testCase)} requested widget jobs before returning prose.`, + content: `${CUSTOM_WIDGET_TOOL_STAGING_INSTRUCTION}\n\n${CUSTOM_WIDGET_ASSISTANT_POLICY}\n\nThis is an unassisted tool-use evaluation. No tool will be forced for you. Complete and persist all ${getExpectedWidgetCount(args.testCase)} requested widget jobs before returning prose.`, }, { role: "user", content: buildAssistantPrompt(args.testCase, args.feedback) }, ]; diff --git a/packages/custom-widgets/src/core/ai-prompt.ts b/packages/custom-widgets/src/core/ai-prompt.ts index a6df8c2f1..3fb3a12f7 100644 --- a/packages/custom-widgets/src/core/ai-prompt.ts +++ b/packages/custom-widgets/src/core/ai-prompt.ts @@ -110,8 +110,8 @@ Use customWidget_validateTemplate for focused JSX repair without resending the m export const CUSTOM_WIDGET_AUTHORING_PROMPT = AUTHORING_PROMPT; -export const CUSTOM_WIDGET_LAZY_TOOL_DISCOVERY_INSTRUCTION = - "Use homarr_findTools for task-needed component, reference, lifecycle, configuration, or placement tools; external MCP keeps live schema and Assistant context compact."; +export const CUSTOM_WIDGET_TOOL_STAGING_INSTRUCTION = + "Custom Widget tools are staged by the authoring lifecycle. Use only visible task-needed tools; successful phases expose the next typed tools without loading the full catalog."; export const CUSTOM_WIDGET_ASSISTANT_POLICY = `Custom Widget work: - Use customWidget tools; never substitute prose. diff --git a/packages/custom-widgets/src/core/assistant-authoring-phase.ts b/packages/custom-widgets/src/core/assistant-authoring-phase.ts index 856d4880f..dcab5d3bd 100644 --- a/packages/custom-widgets/src/core/assistant-authoring-phase.ts +++ b/packages/custom-widgets/src/core/assistant-authoring-phase.ts @@ -31,6 +31,15 @@ const customWidgetDraftPhaseToolNames = new Set([ "customWidget_getSharedProps", "customWidget_validateTemplate", ]); +const customWidgetContextPhaseToolNames = new Set([ + "customWidget_getReference", + "customWidget_findComponents", + "customWidget_getComponents", + "customWidget_getComponent", + "customWidget_getSharedProps", + "customWidget_getExample", + "customWidget_validateTemplate", +]); const maxFocusedComponentSearchesPerPhase = 4; export interface CustomWidgetToolStep { @@ -182,5 +191,11 @@ export const getCustomWidgetPhaseToolNames = ( if (focusedSearches >= maxFocusedComponentSearchesPerPhase) { return availableToolNames.filter((toolName) => customWidgetDraftPhaseToolNames.has(toolName)); } + const skillLoaded = steps.some((step) => + step.toolResults.some((result) => result.toolName === "customWidget_getSkill"), + ); + if (skillLoaded) { + return availableToolNames.filter((toolName) => customWidgetContextPhaseToolNames.has(toolName)); + } return null; }; diff --git a/packages/custom-widgets/src/test/ai-assistant-evaluation.spec.ts b/packages/custom-widgets/src/test/ai-assistant-evaluation.spec.ts index 929896ae0..7f33ffd60 100644 --- a/packages/custom-widgets/src/test/ai-assistant-evaluation.spec.ts +++ b/packages/custom-widgets/src/test/ai-assistant-evaluation.spec.ts @@ -336,50 +336,38 @@ describe("Custom Widget assistant live evaluation harness", () => { expect(names).not.toContain("customWidget_validate"); }); - it("starts with only the authoring bootstrap and discovers later lifecycle tools on demand", () => { + it("starts with compact authoring context tools and stages later lifecycle tools by phase", () => { const testCase = getCase("seerr-media-workflows"); const state = createAssistantEvaluationState(); const getActiveNames = () => getActiveAssistantEvaluationToolDefinitions(state).map(({ function: definition }) => definition.name); + expect(getActiveNames()).toEqual(["web_search", "customWidget_getSkill"]); + executeAssistantEvaluationTool(testCase, state, "customWidget_getSkill", {}); expect(getActiveNames()).toEqual([ - "homarr_findTools", - "web_search", - "customWidget_getSkill", + "customWidget_getReference", + "customWidget_findComponents", + "customWidget_getComponents", + "customWidget_getComponent", + "customWidget_getSharedProps", + "customWidget_getExample", "customWidget_validateTemplate", ]); - expect( - executeAssistantEvaluationTool(testCase, state, "homarr_findTools", { - query: "preview create revise template query action persist journal", - }), - ).toMatchObject({ - matches: expect.arrayContaining([ - expect.objectContaining({ name: "customWidget_previewCreate" }), - expect.objectContaining({ name: "customWidget_previewReviseTemplate" }), - expect.objectContaining({ name: "customWidget_previewQuery" }), - expect.objectContaining({ name: "customWidget_previewAction" }), - expect.objectContaining({ name: "customWidget_createFromPreview" }), - ]), - }); - expect(getActiveNames()).toEqual( - expect.arrayContaining([ - "customWidget_previewCreate", - "customWidget_previewReviseTemplate", - "customWidget_previewQuery", - "customWidget_previewAction", - "customWidget_createFromPreview", - ]), + expect(getActiveNames()).not.toEqual( + expect.arrayContaining(["customWidget_previewCreate", "customWidget_createFromPreview"]), ); - expect( - executeAssistantEvaluationTool(testCase, state, "homarr_findTools", { - query: "customWidget_findComponents focused component search and selected component documentation batch", - }), - ).toMatchObject({ - matches: expect.arrayContaining([ - expect.objectContaining({ name: "customWidget_findComponents" }), - expect.objectContaining({ name: "customWidget_getComponents" }), - ]), + + executeAssistantEvaluationTool(testCase, state, "customWidget_getComponents", { + names: ["Stack", "TextInput", "Button"], }); + expect(getActiveNames()).toEqual(["customWidget_getReference", "customWidget_validateTemplate"]); + + validateTemplate(testCase, state, mediaResearchWidget); + expect(getActiveNames()).toEqual([ + "customWidget_validateTemplate", + "customWidget_previewCreate", + "customWidget_previewReviseTemplate", + ]); }); it("compacts obsolete documentation and validation calls but retains the current preview", () => { @@ -855,9 +843,7 @@ describe("Custom Widget assistant live evaluation harness", () => { const testCase = getCase("fake-service-health"); const state = createAssistantEvaluationState(); - executeAssistantEvaluationTool(testCase, state, "homarr_findTools", { - query: "focused component search and selected component documentation batch", - }); + executeAssistantEvaluationTool(testCase, state, "customWidget_getSkill", {}); executeAssistantEvaluationTool(testCase, state, "customWidget_findComponents", { query: "health status layout", }); diff --git a/packages/custom-widgets/src/test/ai-prompt.spec.ts b/packages/custom-widgets/src/test/ai-prompt.spec.ts index e2a4eb6fd..222ab1a8d 100644 --- a/packages/custom-widgets/src/test/ai-prompt.spec.ts +++ b/packages/custom-widgets/src/test/ai-prompt.spec.ts @@ -5,7 +5,7 @@ import { buildCustomWidgetMcpPrompt, CUSTOM_WIDGET_ASSISTANT_POLICY, CUSTOM_WIDGET_FINAL_OUTPUT_INSTRUCTION, - CUSTOM_WIDGET_LAZY_TOOL_DISCOVERY_INSTRUCTION, + CUSTOM_WIDGET_TOOL_STAGING_INSTRUCTION, CUSTOM_WIDGET_MCP_AUTHORING_PROMPT, } from "../core/ai-prompt"; @@ -54,8 +54,8 @@ describe("AI prompt", () => { expect(CUSTOM_WIDGET_ASSISTANT_POLICY).toContain("change the active phase"); expect(CUSTOM_WIDGET_ASSISTANT_POLICY).toContain("customWidget_getComponents"); expect(CUSTOM_WIDGET_ASSISTANT_POLICY).toContain("Do not load the full catalog"); - expect(CUSTOM_WIDGET_LAZY_TOOL_DISCOVERY_INSTRUCTION).toContain("homarr_findTools"); - expect(CUSTOM_WIDGET_LAZY_TOOL_DISCOVERY_INSTRUCTION).toContain("task-needed"); + expect(CUSTOM_WIDGET_TOOL_STAGING_INSTRUCTION).toContain("staged by the authoring lifecycle"); + expect(CUSTOM_WIDGET_TOOL_STAGING_INSTRUCTION).toContain("task-needed"); expect(CUSTOM_WIDGET_ASSISTANT_POLICY).toContain("Reuse loaded context"); expect(CUSTOM_WIDGET_ASSISTANT_POLICY).toContain("no arbitrary documentation or creativity cap"); expect(CUSTOM_WIDGET_ASSISTANT_POLICY).toContain("one focused component search per widget job");