diff --git a/apps/nextjs/src/app/api/assistant/chat/assistant-tool-policy.spec.ts b/apps/nextjs/src/app/api/assistant/chat/assistant-tool-policy.spec.ts new file mode 100644 index 000000000..0fb365986 --- /dev/null +++ b/apps/nextjs/src/app/api/assistant/chat/assistant-tool-policy.spec.ts @@ -0,0 +1,84 @@ +import { describe, expect, test } from "vitest"; +import type { UIMessage } from "ai"; + +import { getForcedAssistantToolName, withAssistantToolPolicy } from "./assistant-tool-policy"; + +const assistantMessage = (...parts: UIMessage["parts"]): UIMessage => ({ + id: "assistant-message", + role: "assistant", + parts, +}); + +describe("withAssistantToolPolicy", () => { + test("tells models to call approval-gated mutations without a prose confirmation", () => { + const description = withAssistantToolPolicy("Create a Homarr app.", true); + + expect(description).toContain("call this tool immediately"); + expect(description).toContain("does not execute until the user selects Approve and run"); + expect(description).toContain("Never ask for confirmation in prose"); + }); + + test("does not alter read-only tool descriptions", () => { + expect(withAssistantToolPolicy("List all Homarr apps.", false)).toBe("List all Homarr apps."); + }); +}); + +describe("getForcedAssistantToolName", () => { + test("continues directly from the reviewed app form to app creation", () => { + expect( + getForcedAssistantToolName([ + assistantMessage({ + type: "dynamic-tool", + toolName: "configure_app", + toolCallId: "configure-1", + input: { name: "YouTube" }, + state: "output-available", + output: { + name: "YouTube", + iconUrl: "/api/icons/youtube.svg", + href: "https://youtube.com", + }, + }), + ]), + ).toBe("app_create"); + }); + + test("waits for the user to finish reviewing the app form", () => { + expect( + getForcedAssistantToolName([ + assistantMessage({ + type: "dynamic-tool", + toolName: "configure_app", + toolCallId: "configure-1", + input: { name: "YouTube" }, + state: "input-available", + }), + ]), + ).toBeUndefined(); + }); + + test("does not repeat app creation after the mutation tool has been called", () => { + expect( + getForcedAssistantToolName([ + assistantMessage( + { + type: "dynamic-tool", + toolName: "configure_app", + toolCallId: "configure-1", + input: { name: "YouTube" }, + state: "output-available", + output: { name: "YouTube", href: "https://youtube.com" }, + }, + { + type: "dynamic-tool", + toolName: "app_create", + toolCallId: "create-1", + input: { name: "YouTube", href: "https://youtube.com" }, + state: "approval-requested", + approval: { id: "approval-1" }, + }, + ), + ]), + ).toBeUndefined(); + }); +}); diff --git a/apps/nextjs/src/app/api/assistant/chat/assistant-tool-policy.ts b/apps/nextjs/src/app/api/assistant/chat/assistant-tool-policy.ts new file mode 100644 index 000000000..56975c90b --- /dev/null +++ b/apps/nextjs/src/app/api/assistant/chat/assistant-tool-policy.ts @@ -0,0 +1,26 @@ +import { getToolName, isToolUIPart } from "ai"; +import type { UIMessage } from "ai"; + +const mutationApprovalInstruction = + "This action is protected by Homarr's native approval UI. When the user's request and the required inputs are clear, call this tool immediately. The tool call only requests approval; it does not execute until the user selects Approve and run. Never ask for confirmation in prose before calling it."; + +export const withAssistantToolPolicy = (description: string | undefined, requiresApproval: boolean) => { + if (!requiresApproval) return description; + return description ? `${description}\n\n${mutationApprovalInstruction}` : mutationApprovalInstruction; +}; + +export const getForcedAssistantToolName = (messages: UIMessage[]) => { + const latestMessage = messages.at(-1); + if (latestMessage?.role !== "assistant") return undefined; + + const latestToolPart = latestMessage.parts.toReversed().find((part) => isToolUIPart(part)); + if ( + latestToolPart === undefined || + getToolName(latestToolPart) !== "configure_app" || + latestToolPart.state !== "output-available" + ) { + return undefined; + } + + return "app_create" as const; +}; diff --git a/apps/nextjs/src/app/api/assistant/chat/route.ts b/apps/nextjs/src/app/api/assistant/chat/route.ts index bd805701f..e7dd274e0 100644 --- a/apps/nextjs/src/app/api/assistant/chat/route.ts +++ b/apps/nextjs/src/app/api/assistant/chat/route.ts @@ -28,6 +28,7 @@ import type { import { extractMcpTools } from "../../mcp/_extract-tools"; import { getRequestedMentionIds, sanitizeAttachmentFilename } from "./assistant-chat-input"; import { getAssistantStreamErrorMessage } from "./assistant-stream-error"; +import { getForcedAssistantToolName, withAssistantToolPolicy } from "./assistant-tool-policy"; export const maxDuration = 60; @@ -88,15 +89,18 @@ Homarr concepts: Action rules: - Prefer read-only tools before actions. -- When required information is missing or the user must choose between meaningful alternatives, call ask_user. Do not ask the question only in prose and do not tell the user to type yes or no. For a confirmation-style question, offer Yes, No, and Alternative options and leave the freeform Other answer enabled. -- Mutating Homarr tools already pause for native user approval. Once the requested change is sufficiently specified, call the mutation immediately so the approval UI appears. Never ask for a second textual confirmation first. +- When required information is missing or the user must choose between meaningful alternatives, call ask_user. Do not ask the question only in prose. Offer distinct choices and leave the freeform Other answer enabled. +- Mutating Homarr tools already pause for native user approval. Calling a mutation only proposes the action; it cannot execute until the user selects Approve and run. Once the requested change is sufficiently specified, call the mutation immediately so the approval UI appears. +- A prose response such as "Please confirm if you would like me to proceed", "Would you like me to proceed?", or a parameter summary that asks for confirmation is incorrect. Never ask for a second textual confirmation before an approval-gated tool call. - Do not retry a denied action. - Before creating an app, call configure_app with the best defaults so the user can review Homarr's native app form. Its icon picker searches Homarr's local icon repository. Use the returned values for app_create. - When choosing an app icon without configure_app, call the Homarr icon findIcons tool first and use one of its returned local icon URLs. Never invent a third-party icon CDN URL. - Browser tools can navigate within Homarr or open existing Homarr UI. Never navigate to an arbitrary external URL. - Keep responses concise and lead with the result. Summarize tool output instead of dumping JSON. - Use well-formed GitHub-flavored Markdown. Put each list item on its own line and leave a blank line before lists. -- If configuration or a service is unavailable, say what the user or administrator can do next.`; +- If configuration or a service is unavailable, say what the user or administrator can do next. + +Critical approval protocol: when a mutation's inputs are known, your next response must contain the mutation tool call, not a confirmation question. Homarr itself obtains approval after the tool call and before execution.`; const asRecord = (value: unknown): Record | null => typeof value === "object" && value !== null && !Array.isArray(value) ? (value as Record) : null; @@ -355,10 +359,11 @@ export async function POST(request: Request) { const homarrTools = Object.fromEntries( mcpTools.map((mcpTool) => { + const requiresApproval = procedureTypes.get(mcpTool.pathInRouter.join(".")) === "mutation"; return [ mcpTool.name, tool({ - description: mcpTool.description, + description: withAssistantToolPolicy(mcpTool.description, requiresApproval), inputSchema: jsonSchema( (mcpTool.inputSchema ?? { type: "object", properties: {} }) as Parameters[0], ), @@ -413,6 +418,8 @@ export async function POST(request: Request) { ]; }), ) satisfies ToolSet; + const availableTools = { ...homarrTools, ...frontendTools }; + const forcedToolName = getForcedAssistantToolName(parsed.data.messages as UIMessage[]); try { const customHeaders = configuration.encryptedHeaders @@ -443,7 +450,14 @@ export async function POST(request: Request) { model: provider(modelId), instructions: `${assistantInstructions}${mentionContext}`, messages: await convertToModelMessages(prepareMessagesForModel(parsed.data.messages as UIMessage[])), - tools: { ...homarrTools, ...frontendTools }, + tools: availableTools, + prepareStep: ({ stepNumber }) => { + if (stepNumber !== 0 || forcedToolName === undefined || !(forcedToolName in availableTools)) return undefined; + return { + activeTools: [forcedToolName], + toolChoice: { type: "tool", toolName: forcedToolName }, + }; + }, stopWhen: stepCountIs(8), abortSignal: request.signal, timeout: { totalMs: 55_000, stepMs: 30_000, toolMs: 30_000 },