test: strengthen custom widget AI evaluation

apps/docs/docs/management/custom-widgets/agent-authoring.mdx

@@ -96,7 +96,10 @@ OPENROUTER_API_KEY=... pnpm --filter @homarr/custom-widgets test:ai
 
 The suite uses DeepSeek V4 Pro to author Pokédex, Portainer, football, Jellyfin, and Home Assistant widgets. Before DeepSeek V4 Flash judges a result, it must pass Homarr's real clipboard parser, canonical schema, and deterministic scenario checks for the documented API routes, authentication, bindings, actions, and essential runtime components. It saves every response, validated widget manifest, and detailed category scores under `packages/custom-widgets/.ai-evaluations/`.
 
-The run has a hard budget of ten generation attempts across all scenarios. Override the models or budget with `OPENROUTER_GENERATOR_MODEL`, `OPENROUTER_JUDGE_MODEL`, and `CUSTOM_WIDGET_AI_MAX_LOOPS`. Generated artifacts are local and ignored by Git; promote a reviewed, validated result into the bundled-widget source deliberately.
+Each scenario stops as soon as it passes and has a hard limit of ten generation attempts. Override the models or
+per-scenario limit with `OPENROUTER_GENERATOR_MODEL`, `OPENROUTER_JUDGE_MODEL`, and
+`CUSTOM_WIDGET_AI_MAX_LOOPS`. Generated artifacts are local and ignored by Git; promote a reviewed, validated result
+into the bundled-widget source deliberately.
 
 To exercise the connected MCP workflow with the Vercel AI SDK agent, start Homarr and create an API key for an administrator:
 
@@ -112,7 +115,8 @@ and validation results, one successful preview call for every scenario query and
 calls. It can repair and retest a widget, but it does not persist by default. Add `--persist` to create a passing widget,
 or `--definition=<id> --persist` to test the edit/debug/update path. Authenticated sources use
 `customWidget_configurationRequestUser`; credential-writing tools are not exposed to the model and plaintext credentials
-never enter its context. The deterministic unit suite separately proves the repair, persistence, and secure
-source-configuration loop without contacting a model or changing a Homarr instance.
+never enter its context. A passing result is rejected until that request reports `completed` and the required preview
+queries and simulated actions succeed afterward. The deterministic unit suite separately proves the repair, persistence,
+and secure source-configuration loop without contacting a model or changing a Homarr instance.
 
 The reference API check uses a real PokeAPI list/detail request and a read-only Portainer container query when `PORTAINER_BASE_URL` and `PORTAINER_API_KEY` are set. Use `pnpm --filter @homarr/custom-widgets test:reference-apis -- --pokeapi-only` for the public PokeAPI check alone. To exercise start, restart, and stop against a disposable container, also set `PORTAINER_TEST_CONTAINER_ID` and the explicit safeguard `PORTAINER_ALLOW_MUTATIONS=1`; the script restores the container's original running state.

packages/custom-widgets/scripts/ai-evaluation-cases.ts

@@ -11,9 +11,9 @@ export interface CustomWidgetAiEvaluationCase {
       kind?: "query" | "action";
       method?: "GET" | "POST" | "PUT" | "DELETE" | "PATCH";
       trigger?: "load" | "manual";
-      optionReference?: boolean;
-      parameterReference?: boolean;
-      invalidates?: boolean;
+      optionReference?: string;
+      parameterReference?: string;
+      invalidatesRequest?: string;
     }>;
     templateComponents: readonly string[];
   };
@@ -36,7 +36,7 @@ export const CUSTOM_WIDGET_AI_EVALUATION_CASES: readonly CustomWidgetAiEvaluatio
           pathIncludes: "/api/v2/pokemon/",
           kind: "query",
           trigger: "manual",
-          parameterReference: true,
+          parameterReference: "name",
         },
       ],
       templateComponents: ["TextInput", "SubFetch", "Image", "Progress"],
@@ -56,31 +56,34 @@ export const CUSTOM_WIDGET_AI_EVALUATION_CASES: readonly CustomWidgetAiEvaluatio
           label: "container list",
           pathIncludes: "/containers/json",
           kind: "query",
-          optionReference: true,
+          optionReference: "endpointId",
         },
         {
           label: "start action",
           pathIncludes: "/start",
           kind: "action",
           method: "POST",
-          parameterReference: true,
-          invalidates: true,
+          optionReference: "endpointId",
+          parameterReference: "id",
+          invalidatesRequest: "container list",
         },
         {
           label: "stop action",
           pathIncludes: "/stop",
           kind: "action",
           method: "POST",
-          parameterReference: true,
-          invalidates: true,
+          optionReference: "endpointId",
+          parameterReference: "id",
+          invalidatesRequest: "container list",
         },
         {
           label: "restart action",
           pathIncludes: "/restart",
           kind: "action",
           method: "POST",
-          parameterReference: true,
-          invalidates: true,
+          optionReference: "endpointId",
+          parameterReference: "id",
+          invalidatesRequest: "container list",
         },
       ],
       templateComponents: ["ActionButton", "Badge", "RefreshButton"],
@@ -100,13 +103,13 @@ export const CUSTOM_WIDGET_AI_EVALUATION_CASES: readonly CustomWidgetAiEvaluatio
           label: "competition standings",
           pathIncludes: "/standings",
           kind: "query",
-          optionReference: true,
+          optionReference: "competition",
         },
         {
           label: "competition fixtures",
           pathIncludes: "/matches",
           kind: "query",
-          optionReference: true,
+          optionReference: "competition",
         },
       ],
       templateComponents: ["Table", "Image", "Badge", "RefreshButton"],
@@ -142,29 +145,29 @@ export const CUSTOM_WIDGET_AI_EVALUATION_CASES: readonly CustomWidgetAiEvaluatio
           label: "sensor state",
           pathIncludes: "/api/states/",
           kind: "query",
-          optionReference: true,
+          optionReference: "sensorEntity",
         },
         {
           label: "light state",
           pathIncludes: "/api/states/",
           kind: "query",
-          optionReference: true,
+          optionReference: "lightEntity",
         },
         {
           label: "light on action",
           pathIncludes: "/api/services/light/turn_on",
           kind: "action",
           method: "POST",
-          optionReference: true,
-          invalidates: true,
+          optionReference: "lightEntity",
+          invalidatesRequest: "light state",
         },
         {
           label: "light off action",
           pathIncludes: "/api/services/light/turn_off",
           kind: "action",
           method: "POST",
-          optionReference: true,
-          invalidates: true,
+          optionReference: "lightEntity",
+          invalidatesRequest: "light state",
         },
       ],
       templateComponents: ["ToggleSwitch", "Badge", "SimpleGrid"],

packages/custom-widgets/scripts/ai-evaluation.ts

@@ -101,7 +101,7 @@ export function buildRepairPrompt(
   const diagnostics = issues
     .map((issue) => `${issue.path?.length ? `${issue.path.join(".")}: ` : ""}${issue.message}`)
     .join("\n");
-  return `${originalPrompt}\n\nYour previous response did not validate. Correct only the generic contract/runtime problems below while preserving a polished design.\n\nDiagnostics:\n${diagnostics}\n\nPrevious response:\n${previousResponse}`;
+  return `${originalPrompt}\n\nYour previous response did not meet the release quality bar. Correct every diagnosed contract, runtime, goal-fulfillment, visual, and UX problem below. Preserve grounded API behavior and the strongest working parts, but redesign weak areas when the feedback requires it.\n\nDiagnostics:\n${diagnostics}\n\nPrevious response:\n${previousResponse}`;
 }
 
 export function buildJudgePrompt(testCase: CustomWidgetAiEvaluationCase, widget: HomarrCustomWidgetV2): string {
@@ -183,6 +183,7 @@ export function getScenarioAcceptanceIssues(
   }
 
   const matchedRequestIds = new Set<string>();
+  const requestIdsByLabel = new Map<string, string>();
   for (const rule of testCase.acceptance.requestRules) {
     const match = Object.entries(widget.requests).find(([requestId, request]) => {
       if (matchedRequestIds.has(requestId)) return false;
@@ -191,18 +192,17 @@ export function getScenarioAcceptanceIssues(
       if (rule.method && request.method !== rule.method) return false;
       if (rule.trigger && request.trigger !== rule.trigger) return false;
       const references = collectCustomWidgetRequestReferences(request);
-      if (rule.optionReference && references.options.size === 0) return false;
-      if (rule.parameterReference && references.params.size === 0) return false;
-      if (
-        rule.invalidates &&
-        !request.invalidates?.some((invalidatedRequestId) => widget.requests[invalidatedRequestId]?.kind === "query")
-      ) {
+      if (rule.optionReference && !references.options.has(rule.optionReference)) return false;
+      if (rule.parameterReference && !references.params.has(rule.parameterReference)) return false;
+      const invalidatedRequestId = rule.invalidatesRequest ? requestIdsByLabel.get(rule.invalidatesRequest) : undefined;
+      if (rule.invalidatesRequest && (!invalidatedRequestId || !request.invalidates?.includes(invalidatedRequestId)))
         return false;
-      }
       return true;
     });
-    if (match) matchedRequestIds.add(match[0]);
-    else {
+    if (match) {
+      matchedRequestIds.add(match[0]);
+      requestIdsByLabel.set(rule.label, match[0]);
+    } else {
       issues.push({
         path: ["requests"],
         message: `Missing grounded ${rule.label} request (${rule.pathIncludes}).`,

packages/custom-widgets/scripts/evaluate-ai-authoring.ts

@@ -10,35 +10,28 @@ if (!apiKey) throw new Error("OPENROUTER_API_KEY is required for the live Custom
 
 const requestedCase = process.argv.find((value) => value.startsWith("--case="))?.slice("--case=".length);
 const configuredLoops = Number(process.env.CUSTOM_WIDGET_AI_MAX_LOOPS ?? 10);
-const totalLoopBudget = Number.isInteger(configuredLoops) && configuredLoops > 0 ? Math.min(configuredLoops, 10) : 10;
+const maxLoopsPerCase = Number.isInteger(configuredLoops) && configuredLoops > 0 ? Math.min(configuredLoops, 10) : 10;
 const selectedCases = requestedCase
   ? CUSTOM_WIDGET_AI_EVALUATION_CASES.filter((testCase) => testCase.id === requestedCase)
   : CUSTOM_WIDGET_AI_EVALUATION_CASES;
 if (selectedCases.length === 0) throw new Error(`Unknown AI evaluation case '${requestedCase}'`);
-if (totalLoopBudget < selectedCases.length) {
-  throw new Error(`CUSTOM_WIDGET_AI_MAX_LOOPS must be at least ${selectedCases.length} for this run`);
-}
 
 const runId = new Date().toISOString().replaceAll(/[:.]/gu, "-");
 const outputRoot = path.resolve(process.cwd(), ".ai-evaluations", runId);
 await mkdir(outputRoot, { recursive: true });
 
 const results: AiEvaluationResult[] = [];
-let remainingLoopBudget = totalLoopBudget;
 for (const testCase of selectedCases) {
-  const remainingCases = selectedCases.length - results.length;
-  const maxLoops = Math.max(1, Math.floor(remainingLoopBudget / remainingCases));
   process.stdout.write(`Evaluating ${testCase.id}...\n`);
   const result = await evaluateCustomWidgetCase({
     testCase,
     apiKey,
     outputRoot,
-    maxLoops,
+    maxLoops: maxLoopsPerCase,
     generatorModel: process.env.OPENROUTER_GENERATOR_MODEL ?? DEFAULT_GENERATOR_MODEL,
     judgeModel: process.env.OPENROUTER_JUDGE_MODEL ?? DEFAULT_JUDGE_MODEL,
   });
   results.push(result);
-  remainingLoopBudget -= result.attempts;
   process.stdout.write(
     result.judge
       ? `  ${result.judge.total}/100 after ${result.attempts} attempt(s)\n`

packages/custom-widgets/scripts/mcp-agent.ts

@@ -44,6 +44,7 @@ export interface CustomWidgetMcpToolExecution {
   tool: string;
   succeeded: boolean;
   requestId?: string;
+  configurationStatus?: "pending" | "completed";
 }
 
 interface AgentToolStep {
@@ -55,7 +56,7 @@ const AGENT_INSTRUCTIONS = `You are the Homarr Custom Widget release evaluator.
 
 Read the live authoring prompt, skill, and schema before authoring. Never invent a tool, API route, validation result, preview result, or credential. Validate every revision. Create a preview, execute every load query that can run, simulate actions instead of causing side effects, and inspect the preview journal. Repair validation, request, runtime, and template problems and repeat the failed checks.
 
-Never put plaintext credentials in a widget, response, summary, or tool argument. For an authenticated source without existing credentials, create a user configuration request with customWidget_configurationRequestUser and report needs-user-configuration. Do not call secret-writing tools.
+Never put plaintext credentials in a widget, response, summary, or tool argument. For an authenticated source without existing credentials, create a user configuration request with customWidget_configurationRequestUser. If the user completes it while the workflow is active, check the same request until it reports completed and then resume the preview checks. Otherwise report needs-user-configuration. Never repeat its URL or request token in the structured result. Do not call secret-writing tools.
 
 Revise unsaved definitions in your working context; do not mutate a stored template while debugging. Persist only when the user prompt explicitly says persistence is allowed and all runnable checks pass. For edits, read the current definition first and update the same definition. Return the required structured result with evidence tied to successful tool calls. A pass must have no remaining issues.`;
 
@@ -105,10 +106,16 @@ export function getCustomWidgetMcpToolExecutions(steps: readonly AgentToolStep[]
   return steps.flatMap(({ toolCalls }) =>
     toolCalls.map((call) => {
       const requestId = asRecord(call.input)?.requestId;
+      const output = results.get(call.toolCallId)?.output;
+      const status = asRecord(getMcpToolPayload(output))?.status;
       return {
         tool: call.toolName,
-        succeeded: isSuccessfulMcpToolOutput(call.toolName, results.get(call.toolCallId)?.output),
+        succeeded: isSuccessfulMcpToolOutput(call.toolName, output),
         ...(typeof requestId === "string" ? { requestId } : {}),
+        ...(call.toolName === "customWidget_configurationRequestUser" &&
+        (status === "pending" || status === "completed")
+          ? { configurationStatus: status }
+          : {}),
       };
     }),
   );
@@ -147,6 +154,9 @@ export function getCustomWidgetMcpWorkflowIssues(args: {
     if (!args.persist && !args.definitionId) required.push("customWidget_previewCreate");
     if (args.persist && !args.definitionId) required.push("customWidget_create");
   }
+  if (hasTool("customWidget_configurationRequestUser") && !required.includes("customWidget_configurationRequestUser")) {
+    required.push("customWidget_configurationRequestUser");
+  }
   for (const tool of required) {
     if (!hasTool(tool)) issues.push(`Required tool was not used: ${tool}`);
     else if (!hasSuccessfulTool(tool)) issues.push(`Required tool did not complete successfully: ${tool}`);
@@ -183,11 +193,22 @@ export function getCustomWidgetMcpWorkflowIssues(args: {
     }
   }
   if (args.output.status === "pass") {
+    const completedConfiguration = args.toolExecutions.findLastIndex(
+      (entry) =>
+        entry.tool === "customWidget_configurationRequestUser" &&
+        entry.succeeded &&
+        entry.configurationStatus === "completed",
+    );
+    if (hasTool("customWidget_configurationRequestUser") && completedConfiguration < 0) {
+      issues.push("A passing result requires the user configuration request to report completed");
+    }
+    const verifiedExecutions =
+      completedConfiguration >= 0 ? args.toolExecutions.slice(completedConfiguration + 1) : args.toolExecutions;
     for (const [tool, requiredCount] of [
       ["customWidget_previewQuery", args.requiredQueryCount ?? 1],
       ["customWidget_previewAction", args.requiredActionCount ?? 0],
     ] as const) {
-      const completed = args.toolExecutions.filter((entry) => entry.tool === tool && entry.succeeded);
+      const completed = verifiedExecutions.filter((entry) => entry.tool === tool && entry.succeeded);
       const distinctRequestIds = new Set(completed.flatMap(({ requestId }) => (requestId ? [requestId] : [])));
       const completedCount = distinctRequestIds.size > 0 ? distinctRequestIds.size : completed.length;
       if (completedCount < requiredCount) {
@@ -246,7 +267,13 @@ export function getCustomWidgetMcpWorkflowIssues(args: {
     } else if (!evidenceTools.has("customWidget_configurationRequestUser")) {
       issues.push("Structured evidence is missing customWidget_configurationRequestUser");
     }
-    const configurationRequest = lastSuccessfulIndex("customWidget_configurationRequestUser");
+    const finalConfiguration = args.toolExecutions.findLast(
+      (entry) => entry.tool === "customWidget_configurationRequestUser" && entry.succeeded,
+    );
+    if (finalConfiguration?.configurationStatus !== "pending") {
+      issues.push("User configuration status requires a pending configuration request");
+    }
+    const configurationRequest = firstSuccessfulIndex("customWidget_configurationRequestUser");
     const targetCreation = args.persist ? lastSuccessfulIndex(persistenceTool) : previewCreate;
     if (!args.definitionId && targetCreation >= 0 && configurationRequest < targetCreation) {
       issues.push("customWidget_configurationRequestUser must run after its definition or preview target exists");
@@ -258,7 +285,7 @@ export function getCustomWidgetMcpWorkflowIssues(args: {
 function isSuccessfulMcpToolOutput(tool: string, output: unknown) {
   const record = asRecord(output);
   if (!record || record.isError === true) return false;
-  const payload = decodeMcpTextPayload(record) ?? record;
+  const payload = getMcpToolPayload(output);
   if (tool === "customWidget_validate") return asRecord(payload)?.valid === true;
   if (tool === "customWidget_previewCreate") return asRecord(payload)?.success === true;
   if (tool === "customWidget_previewQuery" || tool === "customWidget_previewAction") {
@@ -267,6 +294,11 @@ function isSuccessfulMcpToolOutput(tool: string, output: unknown) {
   return output !== undefined;
 }
 
+function getMcpToolPayload(output: unknown) {
+  const record = asRecord(output);
+  return record ? (decodeMcpTextPayload(record) ?? record) : undefined;
+}
+
 function decodeMcpTextPayload(output: Record<string, unknown>) {
   if (!Array.isArray(output.content)) return undefined;
   const text = output.content

packages/custom-widgets/src/test/ai-evaluation.spec.ts

@@ -13,6 +13,7 @@ import {
 } from "../../scripts/ai-evaluation";
 import type { CustomWidgetJudgeResult } from "../../scripts/ai-evaluation";
 import { BUNDLED_CUSTOM_WIDGETS, CUSTOM_WIDGET_STARTER, customWidgetDefinitionSchema } from "../core";
+import { PORTAINER_REFERENCE_WIDGET } from "./fixtures/reference-widgets";
 
 const categoryNames = [
   "schemaAndBindings",
@@ -76,6 +77,8 @@ describe("AI authoring evaluation", () => {
     ]);
     expect(prompt).toContain("requests.list.path: Required");
     expect(prompt).toContain("bad response");
+    expect(prompt).toContain("visual, and UX problem");
+    expect(prompt).toContain("redesign weak areas");
   });
 
   it("requires grounded scenario capabilities before model judging", () => {
@@ -97,6 +100,39 @@ describe("AI authoring evaluation", () => {
     );
   });
 
+  it("requires the exact option bindings and invalidation target for scenario actions", () => {
+    const testCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === "portainer-containers");
+    if (!testCase) throw new Error("Expected the Portainer evaluation case");
+    const portainer = customWidgetDefinitionSchema.parse(PORTAINER_REFERENCE_WIDGET);
+    expect(getScenarioAcceptanceIssues(testCase, portainer)).toEqual([]);
+
+    const wrongBinding = customWidgetDefinitionSchema.parse({
+      ...portainer,
+      requests: {
+        ...portainer.requests,
+        start: {
+          ...portainer.requests.start,
+          path: portainer.requests.start?.path.replace("{option:endpointId}", "{option:showAll}"),
+        },
+      },
+    });
+    expect(getScenarioAcceptanceIssues(testCase, wrongBinding).map(({ message }) => message)).toContain(
+      "Missing grounded start action request (/start).",
+    );
+
+    const wrongInvalidation = customWidgetDefinitionSchema.parse({
+      ...portainer,
+      requests: {
+        ...portainer.requests,
+        health: { path: "/api/status" },
+        start: { ...portainer.requests.start, invalidates: ["health"] },
+      },
+    });
+    expect(getScenarioAcceptanceIssues(testCase, wrongInvalidation).map(({ message }) => message)).toContain(
+      "Missing grounded start action request (/start).",
+    );
+  });
+
   it("uses the requested DeepSeek models and strict structured judge output", () => {
     expect(DEFAULT_GENERATOR_MODEL).toBe("deepseek/deepseek-v4-pro");
     expect(DEFAULT_JUDGE_MODEL).toBe("deepseek/deepseek-v4-flash");

packages/custom-widgets/src/test/mcp-agent.spec.ts

@@ -339,12 +339,143 @@ describe("Custom Widget MCP agent", () => {
           })),
           remainingIssues: [],
         },
-        toolExecutions: successfulExecutions(toolsUsed),
+        toolExecutions: successfulExecutions(toolsUsed).map((entry) =>
+          entry.tool === "customWidget_configurationRequestUser"
+            ? { ...entry, configurationStatus: "pending" as const }
+            : entry,
+        ),
         persist: false,
       }),
     ).toEqual([]);
   });
 
+  it("resumes preview checks after secure user configuration reports completion", async () => {
+    let configurationCalls = 0;
+    const sequence = [
+      "customWidget_getAuthoringPrompt",
+      "customWidget_getSkill",
+      "customWidget_schema",
+      "customWidget_validate",
+      "customWidget_previewCreate",
+      "customWidget_configurationRequestUser",
+      "customWidget_configurationRequestUser",
+      "customWidget_previewQuery",
+      "customWidget_previewJournal",
+    ];
+    const output = {
+      status: "pass" as const,
+      summary: "The user completed secure source configuration and the authenticated preview passed.",
+      definitionId: null,
+      previewUrl: "http://localhost:3000/manage/custom-widgets/preview/session-configured",
+      iterations: 2,
+      evidence: [...new Set(sequence)].map((name) => ({
+        tool: name,
+        outcome: "passed" as const,
+        detail: `${name} completed.`,
+      })),
+      remainingIssues: [],
+    };
+    const responses = [
+      ...sequence.map((toolName, index) => {
+        if (toolName === "customWidget_configurationRequestUser") {
+          return toolCall(
+            toolName,
+            index,
+            configurationCalls++ === 0
+              ? { previewSessionId: "preview-1", sourceId: "default" }
+              : { requestId: "configuration-1" },
+          );
+        }
+        return toolCall(toolName, index, toolName === "customWidget_previewQuery" ? { requestId: "query-1" } : {});
+      }),
+      finalResult(output),
+    ];
+    let responseIndex = 0;
+    let configurationResults = 0;
+    const model = new MockLanguageModelV3({
+      doGenerate: async () => responses[responseIndex++] ?? finalResult(output),
+    });
+    const tools = Object.fromEntries(
+      sequence.map((name) => [
+        name,
+        tool({
+          description: name,
+          inputSchema: z.object({
+            requestId: z.string().optional(),
+            previewSessionId: z.string().optional(),
+            sourceId: z.string().optional(),
+          }),
+          execute: () => {
+            if (name === "customWidget_validate") return mcpOutput({ valid: true });
+            if (name === "customWidget_previewCreate") return mcpOutput({ success: true });
+            if (name === "customWidget_previewQuery") return mcpOutput({ ok: true });
+            if (name === "customWidget_configurationRequestUser") {
+              configurationResults += 1;
+              return mcpOutput({
+                requestId: "configuration-1",
+                status: configurationResults === 1 ? "pending" : "completed",
+              });
+            }
+            return mcpOutput({});
+          },
+        }),
+      ]),
+    );
+
+    const result = await createCustomWidgetMcpAgent({ model, tools }).generate({ prompt: "Build a Jellyfin widget" });
+    const toolExecutions = getCustomWidgetMcpToolExecutions(result.steps);
+    expect(
+      toolExecutions
+        .filter(({ tool: toolName }) => toolName === "customWidget_configurationRequestUser")
+        .map(({ configurationStatus }) => configurationStatus),
+    ).toEqual(["pending", "completed"]);
+    expect(
+      getCustomWidgetMcpWorkflowIssues({
+        output: result.output,
+        toolExecutions,
+        persist: false,
+      }),
+    ).toEqual([]);
+  });
+
+  it("rejects success while secure user configuration is still pending", () => {
+    const toolsUsed = [
+      "customWidget_getAuthoringPrompt",
+      "customWidget_getSkill",
+      "customWidget_schema",
+      "customWidget_validate",
+      "customWidget_previewCreate",
+      "customWidget_configurationRequestUser",
+      "customWidget_previewQuery",
+      "customWidget_previewJournal",
+    ];
+    const toolExecutions = successfulExecutions(toolsUsed).map((entry) =>
+      entry.tool === "customWidget_configurationRequestUser"
+        ? { ...entry, configurationStatus: "pending" as const }
+        : entry,
+    );
+
+    expect(
+      getCustomWidgetMcpWorkflowIssues({
+        output: {
+          status: "pass",
+          summary: "The agent claimed the authenticated preview passed before configuration completed.",
+          definitionId: null,
+          previewUrl: "http://localhost:3000/manage/custom-widgets/preview/session-pending",
+          iterations: 1,
+          evidence: toolsUsed.map((name) => ({
+            tool: name,
+            outcome: "passed" as const,
+            detail: `${name} completed.`,
+          })),
+          remainingIssues: [],
+        },
+        toolExecutions,
+        persist: false,
+      }),
+    ).toContain("A passing result requires the user configuration request to report completed");
+  });
+
   it("never exposes a credential-writing tool to the model", async () => {
     const forbiddenTool = tool({
       description: "Write a plaintext secret",