fix assistant custom widget creation

This commit is contained in:
ajnart committed 2026-08-14 11:32:11 +02:00
1 parent 474d9968b5
commit ceed0979a1
21 files changed
+513 -47

No files matched your search

@@ -30,8 +30,13 @@ describe("withAssistantToolPolicy", () => {
describe("customWidgetAssistantInstructions", () => {
test("requires the complete skill and real preview-query data before creation", () => {
expect(customWidgetAssistantInstructions).toContain("Custom Widget tools are mandatory");
expect(customWidgetAssistantInstructions).toContain("only a manifest, JSX, instructions");
expect(customWidgetAssistantInstructions).toContain("when they are preloaded below");
expect(customWidgetAssistantInstructions).toContain("do not call customWidget_getSkill or customWidget_schema");
expect(customWidgetAssistantInstructions).toContain(
"already loaded their complete contents into this system prompt",
);
expect(customWidgetAssistantInstructions).toContain("first call customWidget_getSkill");
expect(customWidgetAssistantInstructions).toContain("customWidget_schema");
expect(customWidgetAssistantInstructions).toContain("customWidget_getComponentCatalog");
@@ -39,8 +44,15 @@ describe("customWidgetAssistantInstructions", () => {
expect(customWidgetAssistantInstructions).toContain("customWidget_getSharedProps once");
expect(customWidgetAssistantInstructions).toContain("customWidget_getExample");
expect(customWidgetAssistantInstructions).toContain("templateLines");
expect(customWidgetAssistantInstructions).toContain("customWidget_previewQuery for every query");
expect(customWidgetAssistantInstructions).toContain("Call customWidget_create only after");
expect(customWidgetAssistantInstructions).toContain("installed `pokedex` example");
expect(customWidgetAssistantInstructions).toContain("Honor an explicit iteration count");
expect(customWidgetAssistantInstructions).toContain("customWidget_validate with the complete definition");
expect(customWidgetAssistantInstructions).toContain("customWidget_previewQuery for every query returned");
expect(customWidgetAssistantInstructions).toContain("the previous validation and preview evidence is stale");
expect(customWidgetAssistantInstructions).toContain("Never substitute a later unvalidated version");
expect(customWidgetAssistantInstructions).toContain("customWidget_createFromPreview");
expect(customWidgetAssistantInstructions).toContain("Never say the widget is created, updated, placed");
expect(customWidgetAssistantInstructions).toContain("corresponding customWidget_create, customWidget_update");
expect(customWidgetAssistantInstructions).toContain("managementPath");
});
@@ -191,6 +203,28 @@ describe("getRequiredAssistantToolNames", () => {
toolCallId: "create-widget-1",
input: { name: "PSG fixtures" },
state: "output-available",
output: {
id: "widget-1",
managementPath: "/manage/custom-widgets/edit/widget-1",
nextAction: {
type: "place-custom-widget",
targetBoardId: "board-hey",
},
},
}),
]),
).toEqual(["configure_widget"]);
});
test("requires placement after persisting the final tested preview", () => {
expect(
getRequiredAssistantToolNames([
assistantMessage({
type: "dynamic-tool",
toolName: "customWidget_createFromPreview",
toolCallId: "create-widget-from-preview-1",
input: { previewSessionId: "preview-3" },
state: "output-available",
output: {
id: "widget-1",
managementPath: "/manage/custom-widgets/edit/widget-1",
@@ -7,10 +7,19 @@ const mutationApprovalInstruction =
export const customWidgetAssistantInstructions = `
Custom Widget authoring rules:
- For every request to create, repair, validate, preview, or explain a Custom JSX widget, use the complete installed skill and JSON Schema from the trusted Custom Widget authoring context when they are preloaded below; do not call customWidget_getSkill or customWidget_schema in that case. When they are not preloaded, first call customWidget_getSkill, whose response contains the complete installed skill and all references, then call customWidget_schema. Follow those resources as authoritative. Next call the compact customWidget_getComponentCatalog for the installed release. Fetch only the relevant named component documentation with customWidget_getComponent and, when its interaction pattern applies, one named example with customWidget_getExample before authoring. Component results contain only component-specific props; if the design needs shared props such as spacing or color, collect their names from the catalog and call customWidget_getSharedProps once with only those names.
- Custom Widget tools are mandatory for Custom JSX work. Never answer a request to create, repair, or materially change a Custom Widget with only a manifest, JSX, instructions, or an unsupported claim that it should work.
- For every request to create, repair, validate, preview, or explain a Custom JSX widget, use the complete installed skill and JSON Schema from the trusted Custom Widget authoring context when they are preloaded below; do not call customWidget_getSkill or customWidget_schema in that case because Homarr has already loaded their complete contents into this system prompt. When they are not preloaded, first call customWidget_getSkill, whose response contains the complete installed skill and all references, then call customWidget_schema. Do not merely say that you loaded them: use their rules for the subsequent tool inputs. Follow those resources as authoritative. Next call the compact customWidget_getComponentCatalog for the installed release. Fetch only the relevant named component documentation with customWidget_getComponent and, when its interaction pattern applies, one named example with customWidget_getExample before authoring. Component results contain only component-specific props; if the design needs shared props such as spacing or color, collect their names from the catalog and call customWidget_getSharedProps once with only those names.
- Use templateLines for multiline JSX tool input. Do not send a heavily escaped multiline template string when templateLines is accepted.
- Never skip the authoring lifecycle: customWidget_validate, repair all issues, customWidget_previewCreate, then customWidget_previewQuery for every query listed by the preview. Inspect the real status and returned data shape, correct request paths and template bindings, and repeat validation and preview after changes. Call customWidget_create only after the definition and its preview queries succeed.
- A preview session is not evidence that its API queries work. Never claim live data works until customWidget_previewQuery returned it. Use the returned previewPath when the user should inspect the rendered result.
- Keep each tool call within a reliable streamed size. For Pokédex or PokéAPI requests, start from the installed \`pokedex\` example returned by customWidget_getExample and improve it through validated iterations instead of generating one enormous untested JSX argument from scratch. Completeness comes from working interactions and states, not gratuitous line count.
- Honor an explicit iteration count. Each requested iteration must make a meaningful improvement and complete its own validation, preview, and required preview-query checks. Persist only the last tested preview session.
- Never skip or reorder the evidence checkpoints for the exact candidate that will be saved:
1. Call customWidget_validate with the complete definition. Repair every reported issue and call it again until it succeeds.
2. Call customWidget_previewCreate with that same validated definition.
3. Call customWidget_previewQuery for every query returned by the preview. Inspect the real HTTP status and returned data shape. Test relevant simulated actions with customWidget_previewAction when the widget contains actions.
4. If the JSX template, request definitions, sources, options, or bindings change materially at any point, the previous validation and preview evidence is stale. Start again at customWidget_validate with the complete revised definition, create a fresh preview, and retest every returned query.
5. For a new widget, call customWidget_createFromPreview with the final tested preview session ID so the approved mutation persists that exact candidate without streaming the complete JSX again. If the user already named a target board, resolve its ID before creation and include targetBoardId; after creation call configure_widget directly and do not ask the user to choose again. Use customWidget_create only when no preview session can be reused. For an update, call customWidget_update only with the final definition whose latest validation, preview, and required query checks succeeded. Never substitute a later unvalidated version.
- A successful syntax validation proves only that the definition is valid. A preview session proves only that it can be rendered. Never claim live data works until every customWidget_previewQuery succeeded with the expected response shape. Never say the widget is created, updated, placed, or fully working unless the corresponding customWidget_create, customWidget_update, or board placement tool returned success. If a tool fails, report the specific failed checkpoint and continue repairing it when possible instead of describing the widget as done.
- Progress updates may say which checkpoint is running, but they must not claim success before the tool result exists. Use the returned previewPath when the user should inspect the rendered result.
- After creating a widget, include the returned managementPath as a Markdown link so the user can open and edit the exact widget in Homarr.
- If the user must choose what happens next, such as whether to add the created widget to a board or leave it unplaced, call ask_user with explicit choices. Never end a custom-widget response with a prose question that expects a typed answer. Open-ended conversation and purely rhetorical questions do not require ask_user; state optional next steps declaratively instead of asking about them.`;
@@ -58,7 +67,16 @@ type AssistantToolResponseMessage = {
content: unknown;
};
const isSuccessfulCustomWidgetCreateOutput = (output: unknown) =>
const customWidgetCreateToolNames = new Set(["customWidget_create", "customWidget_createFromPreview"]);
type SuccessfulCustomWidgetCreateOutput = {
id: string;
managementPath: string;
nextAction?: unknown;
error?: undefined;
};
const isSuccessfulCustomWidgetCreateOutput = (output: unknown): output is SuccessfulCustomWidgetCreateOutput =>
typeof output === "object" &&
output !== null &&
"id" in output &&
@@ -69,6 +87,18 @@ const isSuccessfulCustomWidgetCreateOutput = (output: unknown) =>
output.managementPath.length > 0 &&
(!("error" in output) || output.error === undefined);
const getCustomWidgetCreateFollowup = (output: unknown) => {
if (!isSuccessfulCustomWidgetCreateOutput(output)) return [];
const nextAction = output.nextAction;
const hasKnownTarget =
typeof nextAction === "object" &&
nextAction !== null &&
"targetBoardId" in nextAction &&
typeof nextAction.targetBoardId === "string" &&
nextAction.targetBoardId.length > 0;
return hasKnownTarget ? (["configure_widget"] as const) : (["configure_widget", "ask_user"] as const);
};
const getCustomWidgetCreateOutputFromResponseMessages = (responseMessages: readonly AssistantToolResponseMessage[]) => {
for (const message of responseMessages.toReversed()) {
if (message.role !== "tool" || !Array.isArray(message.content)) continue;
@@ -79,7 +109,7 @@ const getCustomWidgetCreateOutputFromResponseMessages = (responseMessages: reado
!("type" in part) ||
part.type !== "tool-result" ||
!("toolName" in part) ||
part.toolName !== "customWidget_create" ||
!customWidgetCreateToolNames.has(String(part.toolName)) ||
!("output" in part)
) {
continue;
@@ -113,10 +143,8 @@ export const getRequiredAssistantToolNames = (
const latestCreateResult = completedSteps
.at(-1)
?.toolResults.toReversed()
.find((result) => result.toolName === "customWidget_create");
return latestCreateResult && isSuccessfulCustomWidgetCreateOutput(latestCreateResult.output)
? (["configure_widget", "ask_user"] as const)
: [];
.find((result) => customWidgetCreateToolNames.has(result.toolName));
return latestCreateResult ? getCustomWidgetCreateFollowup(latestCreateResult.output) : [];
}
// Approval-gated tools execute immediately before the first model step of the follow-up request.
@@ -124,9 +152,7 @@ export const getRequiredAssistantToolNames = (
// UI history, so inspect them before falling back to the persisted UI part.
const responseCreateOutput = getCustomWidgetCreateOutputFromResponseMessages(responseMessages);
if (responseCreateOutput !== undefined) {
return isSuccessfulCustomWidgetCreateOutput(responseCreateOutput)
? (["configure_widget", "ask_user"] as const)
: [];
return getCustomWidgetCreateFollowup(responseCreateOutput);
}
// Approval-gated tools finish in a later HTTP request. At step zero their result is already in
@@ -137,14 +163,12 @@ export const getRequiredAssistantToolNames = (
if (
latestToolPart === undefined ||
latestToolPart.state !== "output-available" ||
getToolName(latestToolPart) !== "customWidget_create"
!customWidgetCreateToolNames.has(getToolName(latestToolPart))
) {
return [];
}
if (!isSuccessfulCustomWidgetCreateOutput(latestToolPart.output)) return [];
// A successful definition creation always has one structured follow-up: place the widget when
// a target is already known, or ask whether it should remain unplaced. Requiring one of these
// tools prevents the model from falling back to a prose "Want me to add it?" question.
return ["configure_widget", "ask_user"] as const;
return getCustomWidgetCreateFollowup(latestToolPart.output);
};
@@ -2,7 +2,7 @@
import { act, createElement, Fragment } from "react";
import { createRoot } from "react-dom/client";
import { afterEach, describe, expect, test } from "vitest";
import { afterEach, describe, expect, test, vi } from "vitest";
import {
AssistantAutoApprovalProvider,
@@ -67,6 +67,19 @@ describe("assistant automatic approval tracker", () => {
expect(tracker.claim("app-create-1")).toBe(true);
});
test("bounds automatic retries and permits a fresh claim after completion", () => {
const tracker = createAssistantAutoApprovalTracker(2);
expect(tracker.claim("custom-widget-create-1")).toBe(true);
tracker.release("custom-widget-create-1");
expect(tracker.claim("custom-widget-create-1")).toBe(true);
tracker.release("custom-widget-create-1");
expect(tracker.claim("custom-widget-create-1")).toBe(false);
tracker.complete("custom-widget-create-1");
expect(tracker.claim("custom-widget-create-1")).toBe(true);
});
test("keeps the setting for the same local conversation and resets it on a real switch", async () => {
const container = document.createElement("div");
containers.push(container);
@@ -185,4 +198,48 @@ describe("assistant automatic approval tracker", () => {
await act(async () => root.unmount());
});
test("retries an approval response that never settles and restores manual controls after two attempts", async () => {
vi.useFakeTimers();
const container = document.createElement("div");
containers.push(container);
document.body.append(container);
const root = createRoot(container);
let confirmationCount = 0;
await act(async () => {
root.render(
createElement(
AssistantAutoApprovalProvider,
{ conversationId: "local-thread-1" },
createElement(
Fragment,
null,
createElement(AutoApprovalProbe),
createElement(AutomaticActionProbe, {
ready: true,
completed: false,
confirm: () => {
confirmationCount += 1;
},
}),
),
),
);
});
act(() => currentAutoApproval?.setEnabled(true));
expect(confirmationCount).toBe(1);
expect(automaticActionInProgress).toBe(true);
await act(async () => vi.advanceTimersByTime(5_000));
expect(confirmationCount).toBe(2);
expect(automaticActionInProgress).toBe(true);
await act(async () => vi.advanceTimersByTime(5_000));
expect(confirmationCount).toBe(2);
expect(automaticActionInProgress).toBe(false);
await act(async () => root.unmount());
vi.useRealTimers();
});
});
@@ -3,20 +3,32 @@
import type { PropsWithChildren } from "react";
import { createContext, useCallback, useContext, useEffect, useMemo, useRef, useState } from "react";
export const createAssistantAutoApprovalTracker = () => {
const AUTO_APPROVAL_RETRY_TIMEOUT_MS = 5_000;
const AUTO_APPROVAL_MAX_ATTEMPTS = 2;
export const createAssistantAutoApprovalTracker = (maximumAttempts = Number.POSITIVE_INFINITY) => {
const claimedToolCalls = new Set<string>();
const attemptsByToolCall = new Map<string, number>();
return {
claim(toolCallId: string) {
if (claimedToolCalls.has(toolCallId)) return false;
const attempts = attemptsByToolCall.get(toolCallId) ?? 0;
if (attempts >= maximumAttempts) return false;
claimedToolCalls.add(toolCallId);
attemptsByToolCall.set(toolCallId, attempts + 1);
return true;
},
release(toolCallId: string) {
claimedToolCalls.delete(toolCallId);
},
complete(toolCallId: string) {
claimedToolCalls.delete(toolCallId);
attemptsByToolCall.delete(toolCallId);
},
clear() {
claimedToolCalls.clear();
attemptsByToolCall.clear();
},
};
};
@@ -25,6 +37,8 @@ interface AssistantAutoApprovalContextValue {
enabled: boolean;
setEnabled: (enabled: boolean) => void;
requestAction: (toolCallId: string, confirm: () => void) => boolean;
completeAction: (toolCallId: string) => void;
retryRevision: number;
}
const AssistantAutoApprovalContext = createContext<AssistantAutoApprovalContextValue | null>(null);
@@ -34,22 +48,41 @@ export const AssistantAutoApprovalProvider = ({
conversationId,
}: PropsWithChildren<{ conversationId: string }>) => {
const [enabled, setEnabledState] = useState(false);
const trackerRef = useRef(createAssistantAutoApprovalTracker());
const [retryRevision, setRetryRevision] = useState(0);
const trackerRef = useRef(createAssistantAutoApprovalTracker(AUTO_APPROVAL_MAX_ATTEMPTS));
const retryTimersRef = useRef(new Map<string, number>());
const previousConversationIdRef = useRef(conversationId);
const setEnabled = useCallback((nextEnabled: boolean) => {
if (!nextEnabled) trackerRef.current.clear();
setEnabledState(nextEnabled);
const clearRetryTimer = useCallback((toolCallId: string) => {
const timer = retryTimersRef.current.get(toolCallId);
if (timer !== undefined) window.clearTimeout(timer);
retryTimersRef.current.delete(toolCallId);
}, []);
const clearAutomaticActions = useCallback(() => {
for (const timer of retryTimersRef.current.values()) window.clearTimeout(timer);
retryTimersRef.current.clear();
trackerRef.current.clear();
}, []);
const setEnabled = useCallback(
(nextEnabled: boolean) => {
if (!nextEnabled) clearAutomaticActions();
setEnabledState(nextEnabled);
},
[clearAutomaticActions],
);
useEffect(() => {
const previousConversationId = previousConversationIdRef.current;
previousConversationIdRef.current = conversationId;
if (previousConversationId === conversationId) return;
trackerRef.current.clear();
clearAutomaticActions();
setEnabledState(false);
}, [conversationId]);
}, [clearAutomaticActions, conversationId]);
useEffect(() => clearAutomaticActions, [clearAutomaticActions]);
const requestAction = useCallback(
(toolCallId: string, confirm: () => void) => {
@@ -57,16 +90,36 @@ export const AssistantAutoApprovalProvider = ({
try {
confirm();
clearRetryTimer(toolCallId);
retryTimersRef.current.set(
toolCallId,
window.setTimeout(() => {
retryTimersRef.current.delete(toolCallId);
trackerRef.current.release(toolCallId);
setRetryRevision((revision) => revision + 1);
}, AUTO_APPROVAL_RETRY_TIMEOUT_MS),
);
return true;
} catch {
trackerRef.current.release(toolCallId);
return false;
}
},
[enabled],
[clearRetryTimer, enabled],
);
const value = useMemo(() => ({ enabled, setEnabled, requestAction }), [enabled, requestAction, setEnabled]);
const completeAction = useCallback(
(toolCallId: string) => {
clearRetryTimer(toolCallId);
trackerRef.current.complete(toolCallId);
},
[clearRetryTimer],
);
const value = useMemo(
() => ({ enabled, setEnabled, requestAction, completeAction, retryRevision }),
[completeAction, enabled, requestAction, retryRevision, setEnabled],
);
return <AssistantAutoApprovalContext.Provider value={value}>{children}</AssistantAutoApprovalContext.Provider>;
};
@@ -88,20 +141,21 @@ export const useAssistantAutomaticAction = ({
completed: boolean;
confirm: () => void;
}) => {
const { enabled, requestAction } = useAssistantAutoApproval();
const { completeAction, enabled, requestAction, retryRevision } = useAssistantAutoApproval();
const [inProgress, setInProgress] = useState(false);
const confirmRef = useRef(confirm);
confirmRef.current = confirm;
useEffect(() => {
if (!ready || completed || !enabled) {
if (completed) completeAction(toolCallId);
setInProgress(false);
return;
}
const requested = requestAction(toolCallId, () => confirmRef.current());
setInProgress(requested);
}, [completed, enabled, ready, requestAction, toolCallId]);
}, [completeAction, completed, enabled, ready, requestAction, retryRevision, toolCallId]);
return inProgress;
};
@@ -42,6 +42,38 @@ describe("shouldAutomaticallyContinueAssistant", () => {
).toBe(true);
});
test("continues an approved mutation when the same step also reports a provider-executed server tool", () => {
expect(
shouldAutomaticallyContinueAssistant({
messages: [
{
id: "assistant-message",
role: "assistant",
parts: [
{ type: "step-start" },
{
type: "dynamic-tool",
toolName: "openrouter_web_search",
toolCallId: "search-1",
input: { query: "PokéAPI" },
state: "input-available",
providerExecuted: true,
},
{
type: "dynamic-tool",
toolName: "customWidget_createFromPreview",
toolCallId: "create-1",
input: { previewSessionId: "preview-1" },
state: "approval-responded",
approval: { id: "approval-1", approved: true },
},
],
},
],
}),
).toBe(true);
});
test("waits while a human tool has no result", () => {
expect(
shouldAutomaticallyContinueAssistant({
@@ -1,6 +1,35 @@
import { lastAssistantMessageIsCompleteWithApprovalResponses, lastAssistantMessageIsCompleteWithToolCalls } from "ai";
import {
isToolUIPart,
lastAssistantMessageIsCompleteWithApprovalResponses,
lastAssistantMessageIsCompleteWithToolCalls,
} from "ai";
import type { UIMessage } from "ai";
type AutoSubmitOptions = Parameters<typeof lastAssistantMessageIsCompleteWithToolCalls>[0];
const lastAssistantMessageHasCompleteLocalApprovalResponses = (messages: UIMessage[]) => {
const message = messages.at(-1);
if (message?.role !== "assistant") return false;
const lastStepStartIndex = message.parts.reduce(
(lastIndex, part, index) => (part.type === "step-start" ? index : lastIndex),
-1,
);
const localToolParts = message.parts
.slice(lastStepStartIndex + 1)
.filter(isToolUIPart)
.filter((part) => !part.providerExecuted);
return (
localToolParts.some((part) => part.state === "approval-responded") &&
localToolParts.every(
(part) =>
part.state === "output-available" || part.state === "output-error" || part.state === "approval-responded",
)
);
};
export const shouldAutomaticallyContinueAssistant = (options: AutoSubmitOptions) =>
lastAssistantMessageIsCompleteWithApprovalResponses(options) || lastAssistantMessageIsCompleteWithToolCalls(options);
lastAssistantMessageIsCompleteWithApprovalResponses(options) ||
lastAssistantMessageHasCompleteLocalApprovalResponses(options.messages) ||
lastAssistantMessageIsCompleteWithToolCalls(options);
@@ -83,7 +83,6 @@ export const AssistantAskUserTool = ({
completed: result !== undefined,
confirm: () => {
if (!affirmativeResult) return;
setSubmitting(true);
addResult(affirmativeResult);
},
});
@@ -299,7 +298,6 @@ export const AssistantConfigureAppTool = ({
completed: result !== undefined,
confirm: () => {
if (!initialValues) return;
setSubmitting(true);
addResult(initialValues);
},
});
@@ -868,7 +868,6 @@ const ToolPart = ({
completed: !awaitingApproval,
confirm: () => {
respondToApproval({ approved: true, reason: "Approved automatically by the user." });
setApprovalResponse("approve");
},
});