diff --git a/apps/docs/docs/management/assistant.mdx b/apps/docs/docs/management/assistant.mdx index f08ecf484..b0ffcabcd 100644 --- a/apps/docs/docs/management/assistant.mdx +++ b/apps/docs/docs/management/assistant.mdx @@ -102,10 +102,19 @@ calculated value as estimated. Missing provider data is shown as not reported ra The assistant can use the same internal tool definitions as the [Homarr MCP server](./mcp). Each tool call runs as the signed-in user, so normal Homarr permission checks still apply. Read-only procedures can run during the agent loop. Mutating procedures require an explicit approval in the -conversation before they execute. Tool cards expose their status, timing, validated input, result, and -error state without flooding the conversation with raw JSON by default. Common collection and widget -results are summarized inline as compact responsive cards, while the full structured result remains -available in the tool card details. +conversation before they execute. The approval appears as a large **Approve and run** or **Deny** action; +the assistant does not require a second typed confirmation first. Tool cards expose their status, timing, +validated input, result, and error state without flooding the conversation with raw JSON by default. +Common collection and widget results are summarized inline as compact responsive cards, while the full +structured result remains available in the tool card details. + +When the assistant genuinely needs missing information or a choice, it can pause with a structured +question. Users can select one of the proposed answers or choose **Other** and enter their own response. +Submitting the answer resumes the same agent run automatically. + +App creation uses Homarr's native app form inside the conversation. The form reuses the local icon picker, +including automatic matching and searches through the `icon.findIcons` procedure, before the final +`app.create` approval is requested. The model is instructed not to invent external icon CDN URLs. Homarr also exposes a small set of browser-only actions to the assistant. These actions can navigate to safe internal routes and open Homarr interfaces such as the command menu and media request search. diff --git a/apps/nextjs/src/app/api/assistant/chat/route.ts b/apps/nextjs/src/app/api/assistant/chat/route.ts index f5c4d6917..bd805701f 100644 --- a/apps/nextjs/src/app/api/assistant/chat/route.ts +++ b/apps/nextjs/src/app/api/assistant/chat/route.ts @@ -88,8 +88,11 @@ Homarr concepts: Action rules: - Prefer read-only tools before actions. -- Mutating Homarr tools require user approval. Present the exact intended change and wait for the approval flow. +- When required information is missing or the user must choose between meaningful alternatives, call ask_user. Do not ask the question only in prose and do not tell the user to type yes or no. For a confirmation-style question, offer Yes, No, and Alternative options and leave the freeform Other answer enabled. +- Mutating Homarr tools already pause for native user approval. Once the requested change is sufficiently specified, call the mutation immediately so the approval UI appears. Never ask for a second textual confirmation first. - Do not retry a denied action. +- Before creating an app, call configure_app with the best defaults so the user can review Homarr's native app form. Its icon picker searches Homarr's local icon repository. Use the returned values for app_create. +- When choosing an app icon without configure_app, call the Homarr icon findIcons tool first and use one of its returned local icon URLs. Never invent a third-party icon CDN URL. - Browser tools can navigate within Homarr or open existing Homarr UI. Never navigate to an arbitrary external URL. - Keep responses concise and lead with the result. Summarize tool output instead of dumping JSON. - Use well-formed GitHub-flavored Markdown. Put each list item on its own line and leave a blank line before lists. diff --git a/apps/nextjs/src/components/assistant/assistant-auto-submit.spec.ts b/apps/nextjs/src/components/assistant/assistant-auto-submit.spec.ts new file mode 100644 index 000000000..fadf9f11a --- /dev/null +++ b/apps/nextjs/src/components/assistant/assistant-auto-submit.spec.ts @@ -0,0 +1,58 @@ +import { describe, expect, test } from "vitest"; +import type { UIMessage } from "ai"; + +import { shouldAutomaticallyContinueAssistant } from "./assistant-auto-submit"; + +const messagesWithToolPart = (part: UIMessage["parts"][number]): UIMessage[] => [ + { + id: "assistant-message", + role: "assistant", + parts: [{ type: "step-start" }, part], + }, +]; + +describe("shouldAutomaticallyContinueAssistant", () => { + test("continues after a human or frontend tool supplies its result", () => { + expect( + shouldAutomaticallyContinueAssistant({ + messages: messagesWithToolPart({ + type: "dynamic-tool", + toolName: "ask_user", + toolCallId: "ask-1", + input: { question: "Continue?" }, + state: "output-available", + output: { answer: "Yes" }, + }), + }), + ).toBe(true); + }); + + test("continues after a mutation approval response", () => { + expect( + shouldAutomaticallyContinueAssistant({ + messages: messagesWithToolPart({ + type: "dynamic-tool", + toolName: "app_create", + toolCallId: "create-1", + input: { name: "YouTube" }, + state: "approval-responded", + approval: { id: "approval-1", approved: true }, + }), + }), + ).toBe(true); + }); + + test("waits while a human tool has no result", () => { + expect( + shouldAutomaticallyContinueAssistant({ + messages: messagesWithToolPart({ + type: "dynamic-tool", + toolName: "ask_user", + toolCallId: "ask-1", + input: { question: "Continue?" }, + state: "input-available", + }), + }), + ).toBe(false); + }); +}); diff --git a/apps/nextjs/src/components/assistant/assistant-auto-submit.ts b/apps/nextjs/src/components/assistant/assistant-auto-submit.ts new file mode 100644 index 000000000..eacd4914f --- /dev/null +++ b/apps/nextjs/src/components/assistant/assistant-auto-submit.ts @@ -0,0 +1,6 @@ +import { lastAssistantMessageIsCompleteWithApprovalResponses, lastAssistantMessageIsCompleteWithToolCalls } from "ai"; + +type AutoSubmitOptions = Parameters[0]; + +export const shouldAutomaticallyContinueAssistant = (options: AutoSubmitOptions) => + lastAssistantMessageIsCompleteWithApprovalResponses(options) || lastAssistantMessageIsCompleteWithToolCalls(options); diff --git a/apps/nextjs/src/components/assistant/assistant-human-tools.tsx b/apps/nextjs/src/components/assistant/assistant-human-tools.tsx new file mode 100644 index 000000000..4548ee8e6 --- /dev/null +++ b/apps/nextjs/src/components/assistant/assistant-human-tools.tsx @@ -0,0 +1,217 @@ +"use client"; + +import { useEffect, useRef, useState } from "react"; +import type { ToolCallMessagePartProps } from "@assistant-ui/react"; +import { Box, Button, Group, Skeleton, Stack, Text, TextInput, ThemeIcon } from "@mantine/core"; +import { IconCheck, IconMessageQuestion, IconPencil, IconX } from "@tabler/icons-react"; +import type { z } from "zod/v4"; + +import { AppForm } from "@homarr/forms-collection"; +import { useScopedI18n } from "@homarr/translation/client"; +import type { appManageSchema } from "@homarr/validation/app"; + +import classes from "./assistant-panel.module.css"; +import { + type AskUserArgs, + type AskUserResult, + type ConfigureAppArgs, + normalizeAssistantAppIconUrl, +} from "./assistant-tool-contracts"; + +type AppValues = z.infer; + +const toAppValues = (args: ConfigureAppArgs | undefined): AppValues => ({ + name: args?.name ?? "", + description: args?.description ?? "", + iconUrl: normalizeAssistantAppIconUrl(args?.iconUrl), + href: args?.href ?? "", + pingUrl: args?.pingUrl ?? "", +}); + +export const AssistantAskUserTool = ({ + args, + result, + addResult, +}: ToolCallMessagePartProps) => { + const t = useScopedI18n("common.assistant.askUser"); + const [showOther, setShowOther] = useState(false); + const [other, setOther] = useState(""); + const otherInputRef = useRef(null); + const options = Array.isArray(args?.options) ? args.options : []; + + useEffect(() => { + if (showOther) otherInputRef.current?.focus(); + }, [showOther]); + + if (result) { + return ( + + + + + + + {t("answered")} + + + {result.answer} + + + + ); + } + + if (!args?.question || options.length < 2) { + return ( + + + + + + + + ); + } + + const submitOther = () => { + const answer = other.trim(); + if (!answer) return; + addResult({ answer, source: "other" }); + }; + + return ( + + + + + + + + {args.question} + + {args.description && ( + + {args.description} + + )} + + + + + {options.map((option, index) => ( + + ))} + + {args.allowOther !== false && + (showOther ? ( + + setOther(event.currentTarget.value)} + onKeyDown={(event) => { + if (event.key === "Enter" && !event.nativeEvent.isComposing) { + event.preventDefault(); + submitOther(); + } + }} + label={t("otherLabel")} + placeholder={t("otherPlaceholder")} + /> + + + + ) : ( + + ))} + + + ); +}; + +export const AssistantConfigureAppTool = ({ + args, + result, + addResult, +}: ToolCallMessagePartProps) => { + const t = useScopedI18n("common.assistant.configureApp"); + + if (result) { + return ( + + + + + + + {t("ready")} + + + {result.name} + + + + ); + } + + return ( + + + {t("title")} + + {t("description")} + + + addResult(values)} + isPending={false} + /> + + ); +}; diff --git a/apps/nextjs/src/components/assistant/assistant-panel.module.css b/apps/nextjs/src/components/assistant/assistant-panel.module.css index 7b7e6dd34..961bce411 100644 --- a/apps/nextjs/src/components/assistant/assistant-panel.module.css +++ b/apps/nextjs/src/components/assistant/assistant-panel.module.css @@ -336,6 +336,61 @@ text-align: start; } +.approvalPanel { + margin-top: 0.8rem; + padding-top: 0.8rem; + border-top: 1px solid var(--mantine-color-default-border); +} + +.approvalActions { + margin-top: 0.85rem; +} + +.humanTool, +.appTool { + width: 100%; + margin-block: 0.65rem; + padding: 1rem; + border: 1px solid var(--mantine-color-default-border); + border-radius: var(--mantine-radius-md); + background: var(--mantine-color-default); +} + +.humanToolQuestion { + line-height: 1.35; + text-wrap: pretty; +} + +.humanToolOption { + height: auto; + min-height: 3.25rem; + padding-block: 0.55rem; + padding-inline: 0.8rem; + white-space: normal; +} + +.humanToolOption :global(.mantine-Button-inner), +.humanToolOption :global(.mantine-Button-label) { + width: 100%; +} + +.humanToolOtherInput { + flex: 1 1 auto; + min-width: 0; +} + +.humanToolCompleted { + display: flex; + width: 100%; + align-items: center; + gap: 0.65rem; + margin-block: 0.5rem; + padding: 0.7rem 0.8rem; + border: 1px solid var(--mantine-color-default-border); + border-radius: var(--mantine-radius-md); + background: var(--mantine-color-default); +} + .toolResultText, .toolResultProperties, .toolResultCollection { @@ -918,6 +973,15 @@ .reasoningSelect { width: 7.75rem; } + + .approvalActions { + flex-direction: column; + } + + .humanTool, + .appTool { + padding: 0.8rem; + } } @media (prefers-reduced-motion: reduce) { diff --git a/apps/nextjs/src/components/assistant/assistant-panel.tsx b/apps/nextjs/src/components/assistant/assistant-panel.tsx index e760faaf1..ccbde59ac 100644 --- a/apps/nextjs/src/components/assistant/assistant-panel.tsx +++ b/apps/nextjs/src/components/assistant/assistant-panel.tsx @@ -357,7 +357,7 @@ const ToolPart = ({ setOpened((current) => !current)} - aria-expanded={opened || awaitingApproval} + aria-expanded={opened} > @@ -399,20 +399,34 @@ const ToolPart = ({ {awaitingApproval && ( - - {t("approvalDescription")} - - - - + )} {successful && result !== undefined && } - + diff --git a/apps/nextjs/src/components/assistant/assistant-provider.tsx b/apps/nextjs/src/components/assistant/assistant-provider.tsx index a38b7c3b6..1e3d1f79d 100644 --- a/apps/nextjs/src/components/assistant/assistant-provider.tsx +++ b/apps/nextjs/src/components/assistant/assistant-provider.tsx @@ -23,7 +23,6 @@ import { import { AssistantChatTransport, useChatRuntime } from "@assistant-ui/react-ai-sdk"; import { useHotkeys } from "@mantine/hooks"; import { createAssistantStream } from "assistant-stream"; -import { lastAssistantMessageIsCompleteWithApprovalResponses } from "ai"; import { clientApi, fetchApi } from "@homarr/api/client"; import { useSession } from "@homarr/auth/client"; @@ -38,6 +37,8 @@ import { useRegisterSpotlightContextResults, } from "@homarr/spotlight"; +import { shouldAutomaticallyContinueAssistant } from "./assistant-auto-submit"; +import { AssistantAskUserTool, AssistantConfigureAppTool } from "./assistant-human-tools"; import { AssistantPanel } from "./assistant-panel"; import type { AssistantReasoningMode, AssistantRuntimeModelOption } from "./assistant-preferences"; import { sendAssistantPrompt as sendPromptThroughComposer } from "./assistant-send"; @@ -391,7 +392,7 @@ const AssistantThreadRuntime = () => { return useChatRuntime({ transport, onError, - sendAutomaticallyWhen: lastAssistantMessageIsCompleteWithApprovalResponses, + sendAutomaticallyWhen: shouldAutomaticallyContinueAssistant, adapters: { history, attachments, @@ -410,6 +411,18 @@ const AssistantRuntime = ({ children }: PropsWithChildren) => { const toolkit = useMemo( () => defineToolkit({ + ask_user: { + type: "human", + display: "standalone", + ...browserToolContracts.ask_user, + render: AssistantAskUserTool, + }, + configure_app: { + type: "human", + display: "standalone", + ...browserToolContracts.configure_app, + render: AssistantConfigureAppTool, + }, navigate_to_route: { type: "frontend", ...browserToolContracts.navigate_to_route, diff --git a/apps/nextjs/src/components/assistant/assistant-tool-contracts.spec.ts b/apps/nextjs/src/components/assistant/assistant-tool-contracts.spec.ts new file mode 100644 index 000000000..5d8d19ff6 --- /dev/null +++ b/apps/nextjs/src/components/assistant/assistant-tool-contracts.spec.ts @@ -0,0 +1,42 @@ +import { describe, expect, test } from "vitest"; + +import { browserToolContracts, normalizeAssistantAppIconUrl } from "./assistant-tool-contracts"; + +describe("assistant human tool contracts", () => { + test("accepts a structured question with reusable choices", () => { + expect( + browserToolContracts.ask_user.parameters.safeParse({ + question: "How should the app be added?", + options: [ + { id: "yes", label: "Yes" }, + { id: "no", label: "No" }, + { id: "alternative", label: "Choose an alternative", description: "Adjust the proposed app first." }, + ], + }).success, + ).toBe(true); + }); + + test("rejects questions without enough choices", () => { + expect( + browserToolContracts.ask_user.parameters.safeParse({ + question: "Continue?", + options: [{ id: "yes", label: "Yes" }], + }).success, + ).toBe(false); + }); + + test("allows partial app defaults for the native form", () => { + expect( + browserToolContracts.configure_app.parameters.safeParse({ + name: "YouTube", + href: "https://youtube.com", + }).success, + ).toBe(true); + }); + + test("only preserves usable app icon URLs", () => { + expect(normalizeAssistantAppIconUrl("youtube")).toBe(""); + expect(normalizeAssistantAppIconUrl("/api/media/youtube.svg")).toBe("/api/media/youtube.svg"); + expect(normalizeAssistantAppIconUrl("https://icons.example/youtube.svg")).toBe("https://icons.example/youtube.svg"); + }); +}); diff --git a/apps/nextjs/src/components/assistant/assistant-tool-contracts.ts b/apps/nextjs/src/components/assistant/assistant-tool-contracts.ts index 9dccd7fe6..f6dc359ac 100644 --- a/apps/nextjs/src/components/assistant/assistant-tool-contracts.ts +++ b/apps/nextjs/src/components/assistant/assistant-tool-contracts.ts @@ -1,6 +1,40 @@ import { z } from "zod/v4"; +const askUserOptionSchema = z.object({ + id: z.string().trim().min(1).max(48), + label: z.string().trim().min(1).max(80), + description: z.string().trim().max(180).optional(), +}); + export const browserToolContracts = { + ask_user: { + description: + "Pause and ask the user one concise structured question. Use this for missing information or a meaningful choice, never as a second confirmation before a mutating tool. Provide 2-4 distinct options; the UI adds a freeform Other choice when allowOther is not false.", + parameters: z.object({ + question: z.string().trim().min(1).max(240), + description: z.string().trim().max(400).optional(), + options: z.array(askUserOptionSchema).min(2).max(4), + allowOther: z.boolean().optional(), + }), + }, + configure_app: { + description: + "Open Homarr's native app form so the user can review or complete app details. Its icon picker searches the local Homarr icon repository. Use this before app.create instead of inventing an external icon URL.", + parameters: z.object({ + name: z.string().trim().max(64).optional(), + description: z.string().trim().max(512).nullable().optional(), + iconUrl: z + .string() + .trim() + .max(2_048) + .optional() + .describe( + "An exact icon URL returned by Homarr's icon_findIcons tool. Omit this field if no icon tool result is available; the native form will search locally from the app name.", + ), + href: z.string().trim().max(2_048).nullable().optional(), + pingUrl: z.string().trim().max(2_048).nullable().optional(), + }), + }, navigate_to_route: { description: "Navigate the current Homarr tab to a safe internal route. Only same-origin paths beginning with a single slash are accepted.", @@ -17,3 +51,16 @@ export const browserToolContracts = { parameters: z.object({}), }, } as const; + +export type AskUserArgs = z.infer<(typeof browserToolContracts)["ask_user"]["parameters"]>; +export type AskUserResult = { + answer: string; + optionId?: string; + source: "option" | "other"; +}; +export type ConfigureAppArgs = z.infer<(typeof browserToolContracts)["configure_app"]["parameters"]>; + +export const normalizeAssistantAppIconUrl = (value: string | undefined) => { + if (!value) return ""; + return value.startsWith("/") || URL.canParse(value) ? value : ""; +}; diff --git a/packages/spotlight/src/components/spotlight.tsx b/packages/spotlight/src/components/spotlight.tsx index 3e285da9a..b73539e56 100644 --- a/packages/spotlight/src/components/spotlight.tsx +++ b/packages/spotlight/src/components/spotlight.tsx @@ -129,6 +129,9 @@ const SpotlightWithActiveMode = ({ modeState, queryState, activeMode }: Spotligh shortcut={hotkeys.openSpotlight} yOffset={8} overlayProps={{ blur: 2, backgroundOpacity: 0.2 }} + onSpotlightOpen={() => { + requestAnimationFrame(() => inputRef.current?.focus()); + }} onSpotlightClose={() => { setMode(defaultMode); clearChildrenStack(); diff --git a/packages/translation/src/lang/en.json b/packages/translation/src/lang/en.json index 2ddb38e7f..0c758ba34 100644 --- a/packages/translation/src/lang/en.json +++ b/packages/translation/src/lang/en.json @@ -2124,13 +2124,28 @@ "dismiss": "Dismiss assistant activity" }, "scrollToLatest": "Scroll to latest", + "askUser": { + "preparing": "Preparing question", + "answered": "Your answer", + "other": "Other", + "otherLabel": "Your answer", + "otherPlaceholder": "Type another answer…", + "submit": "Send answer", + "cancelOther": "Cancel other answer" + }, + "configureApp": { + "title": "Review app details", + "description": "Choose a local icon and adjust anything before the assistant requests creation approval.", + "continue": "Use these details", + "ready": "App details ready" + }, "approvalRequired": "Approval required", "complete": "Complete", "denied": "Denied", "failed": "Failed", "working": "Working", - "approvalDescription": "This action changes Homarr. Review the input before allowing it.", - "allowOnce": "Allow once", + "approvalDescription": "Review this change before it runs.", + "approveAndRun": "Approve and run", "deny": "Deny" }, "dangerZone": "Danger zone",