test: strengthen custom widget AI evaluation

packages/custom-widgets/scripts/ai-evaluation-cases.ts

@@ -11,9 +11,9 @@ export interface CustomWidgetAiEvaluationCase {
       kind?: "query" | "action";
       method?: "GET" | "POST" | "PUT" | "DELETE" | "PATCH";
       trigger?: "load" | "manual";
-      optionReference?: boolean;
-      parameterReference?: boolean;
-      invalidates?: boolean;
+      optionReference?: string;
+      parameterReference?: string;
+      invalidatesRequest?: string;
     }>;
     templateComponents: readonly string[];
   };
@@ -36,7 +36,7 @@ export const CUSTOM_WIDGET_AI_EVALUATION_CASES: readonly CustomWidgetAiEvaluatio
           pathIncludes: "/api/v2/pokemon/",
           kind: "query",
           trigger: "manual",
-          parameterReference: true,
+          parameterReference: "name",
         },
       ],
       templateComponents: ["TextInput", "SubFetch", "Image", "Progress"],
@@ -56,31 +56,34 @@ export const CUSTOM_WIDGET_AI_EVALUATION_CASES: readonly CustomWidgetAiEvaluatio
           label: "container list",
           pathIncludes: "/containers/json",
           kind: "query",
-          optionReference: true,
+          optionReference: "endpointId",
         },
         {
           label: "start action",
           pathIncludes: "/start",
           kind: "action",
           method: "POST",
-          parameterReference: true,
-          invalidates: true,
+          optionReference: "endpointId",
+          parameterReference: "id",
+          invalidatesRequest: "container list",
         },
         {
           label: "stop action",
           pathIncludes: "/stop",
           kind: "action",
           method: "POST",
-          parameterReference: true,
-          invalidates: true,
+          optionReference: "endpointId",
+          parameterReference: "id",
+          invalidatesRequest: "container list",
         },
         {
           label: "restart action",
           pathIncludes: "/restart",
           kind: "action",
           method: "POST",
-          parameterReference: true,
-          invalidates: true,
+          optionReference: "endpointId",
+          parameterReference: "id",
+          invalidatesRequest: "container list",
         },
       ],
       templateComponents: ["ActionButton", "Badge", "RefreshButton"],
@@ -100,13 +103,13 @@ export const CUSTOM_WIDGET_AI_EVALUATION_CASES: readonly CustomWidgetAiEvaluatio
           label: "competition standings",
           pathIncludes: "/standings",
           kind: "query",
-          optionReference: true,
+          optionReference: "competition",
         },
         {
           label: "competition fixtures",
           pathIncludes: "/matches",
           kind: "query",
-          optionReference: true,
+          optionReference: "competition",
         },
       ],
       templateComponents: ["Table", "Image", "Badge", "RefreshButton"],
@@ -142,29 +145,29 @@ export const CUSTOM_WIDGET_AI_EVALUATION_CASES: readonly CustomWidgetAiEvaluatio
           label: "sensor state",
           pathIncludes: "/api/states/",
           kind: "query",
-          optionReference: true,
+          optionReference: "sensorEntity",
         },
         {
           label: "light state",
           pathIncludes: "/api/states/",
           kind: "query",
-          optionReference: true,
+          optionReference: "lightEntity",
         },
         {
           label: "light on action",
           pathIncludes: "/api/services/light/turn_on",
           kind: "action",
           method: "POST",
-          optionReference: true,
-          invalidates: true,
+          optionReference: "lightEntity",
+          invalidatesRequest: "light state",
         },
         {
           label: "light off action",
           pathIncludes: "/api/services/light/turn_off",
           kind: "action",
           method: "POST",
-          optionReference: true,
-          invalidates: true,
+          optionReference: "lightEntity",
+          invalidatesRequest: "light state",
         },
       ],
       templateComponents: ["ToggleSwitch", "Badge", "SimpleGrid"],

packages/custom-widgets/scripts/ai-evaluation.ts

@@ -101,7 +101,7 @@ export function buildRepairPrompt(
   const diagnostics = issues
     .map((issue) => `${issue.path?.length ? `${issue.path.join(".")}: ` : ""}${issue.message}`)
     .join("\n");
-  return `${originalPrompt}\n\nYour previous response did not validate. Correct only the generic contract/runtime problems below while preserving a polished design.\n\nDiagnostics:\n${diagnostics}\n\nPrevious response:\n${previousResponse}`;
+  return `${originalPrompt}\n\nYour previous response did not meet the release quality bar. Correct every diagnosed contract, runtime, goal-fulfillment, visual, and UX problem below. Preserve grounded API behavior and the strongest working parts, but redesign weak areas when the feedback requires it.\n\nDiagnostics:\n${diagnostics}\n\nPrevious response:\n${previousResponse}`;
 }
 
 export function buildJudgePrompt(testCase: CustomWidgetAiEvaluationCase, widget: HomarrCustomWidgetV2): string {
@@ -183,6 +183,7 @@ export function getScenarioAcceptanceIssues(
   }
 
   const matchedRequestIds = new Set<string>();
+  const requestIdsByLabel = new Map<string, string>();
   for (const rule of testCase.acceptance.requestRules) {
     const match = Object.entries(widget.requests).find(([requestId, request]) => {
       if (matchedRequestIds.has(requestId)) return false;
@@ -191,18 +192,17 @@ export function getScenarioAcceptanceIssues(
       if (rule.method && request.method !== rule.method) return false;
       if (rule.trigger && request.trigger !== rule.trigger) return false;
       const references = collectCustomWidgetRequestReferences(request);
-      if (rule.optionReference && references.options.size === 0) return false;
-      if (rule.parameterReference && references.params.size === 0) return false;
-      if (
-        rule.invalidates &&
-        !request.invalidates?.some((invalidatedRequestId) => widget.requests[invalidatedRequestId]?.kind === "query")
-      ) {
+      if (rule.optionReference && !references.options.has(rule.optionReference)) return false;
+      if (rule.parameterReference && !references.params.has(rule.parameterReference)) return false;
+      const invalidatedRequestId = rule.invalidatesRequest ? requestIdsByLabel.get(rule.invalidatesRequest) : undefined;
+      if (rule.invalidatesRequest && (!invalidatedRequestId || !request.invalidates?.includes(invalidatedRequestId)))
         return false;
-      }
       return true;
     });
-    if (match) matchedRequestIds.add(match[0]);
-    else {
+    if (match) {
+      matchedRequestIds.add(match[0]);
+      requestIdsByLabel.set(rule.label, match[0]);
+    } else {
       issues.push({
         path: ["requests"],
         message: `Missing grounded ${rule.label} request (${rule.pathIncludes}).`,

packages/custom-widgets/scripts/evaluate-ai-authoring.ts

@@ -10,35 +10,28 @@ if (!apiKey) throw new Error("OPENROUTER_API_KEY is required for the live Custom
 
 const requestedCase = process.argv.find((value) => value.startsWith("--case="))?.slice("--case=".length);
 const configuredLoops = Number(process.env.CUSTOM_WIDGET_AI_MAX_LOOPS ?? 10);
-const totalLoopBudget = Number.isInteger(configuredLoops) && configuredLoops > 0 ? Math.min(configuredLoops, 10) : 10;
+const maxLoopsPerCase = Number.isInteger(configuredLoops) && configuredLoops > 0 ? Math.min(configuredLoops, 10) : 10;
 const selectedCases = requestedCase
   ? CUSTOM_WIDGET_AI_EVALUATION_CASES.filter((testCase) => testCase.id === requestedCase)
   : CUSTOM_WIDGET_AI_EVALUATION_CASES;
 if (selectedCases.length === 0) throw new Error(`Unknown AI evaluation case '${requestedCase}'`);
-if (totalLoopBudget < selectedCases.length) {
-  throw new Error(`CUSTOM_WIDGET_AI_MAX_LOOPS must be at least ${selectedCases.length} for this run`);
-}
 
 const runId = new Date().toISOString().replaceAll(/[:.]/gu, "-");
 const outputRoot = path.resolve(process.cwd(), ".ai-evaluations", runId);
 await mkdir(outputRoot, { recursive: true });
 
 const results: AiEvaluationResult[] = [];
-let remainingLoopBudget = totalLoopBudget;
 for (const testCase of selectedCases) {
-  const remainingCases = selectedCases.length - results.length;
-  const maxLoops = Math.max(1, Math.floor(remainingLoopBudget / remainingCases));
   process.stdout.write(`Evaluating ${testCase.id}...\n`);
   const result = await evaluateCustomWidgetCase({
     testCase,
     apiKey,
     outputRoot,
-    maxLoops,
+    maxLoops: maxLoopsPerCase,
     generatorModel: process.env.OPENROUTER_GENERATOR_MODEL ?? DEFAULT_GENERATOR_MODEL,
     judgeModel: process.env.OPENROUTER_JUDGE_MODEL ?? DEFAULT_JUDGE_MODEL,
   });
   results.push(result);
-  remainingLoopBudget -= result.attempts;
   process.stdout.write(
     result.judge
       ? `  ${result.judge.total}/100 after ${result.attempts} attempt(s)\n`

packages/custom-widgets/scripts/mcp-agent.ts

@@ -44,6 +44,7 @@ export interface CustomWidgetMcpToolExecution {
   tool: string;
   succeeded: boolean;
   requestId?: string;
+  configurationStatus?: "pending" | "completed";
 }
 
 interface AgentToolStep {
@@ -55,7 +56,7 @@ const AGENT_INSTRUCTIONS = `You are the Homarr Custom Widget release evaluator.
 
 Read the live authoring prompt, skill, and schema before authoring. Never invent a tool, API route, validation result, preview result, or credential. Validate every revision. Create a preview, execute every load query that can run, simulate actions instead of causing side effects, and inspect the preview journal. Repair validation, request, runtime, and template problems and repeat the failed checks.
 
-Never put plaintext credentials in a widget, response, summary, or tool argument. For an authenticated source without existing credentials, create a user configuration request with customWidget_configurationRequestUser and report needs-user-configuration. Do not call secret-writing tools.
+Never put plaintext credentials in a widget, response, summary, or tool argument. For an authenticated source without existing credentials, create a user configuration request with customWidget_configurationRequestUser. If the user completes it while the workflow is active, check the same request until it reports completed and then resume the preview checks. Otherwise report needs-user-configuration. Never repeat its URL or request token in the structured result. Do not call secret-writing tools.
 
 Revise unsaved definitions in your working context; do not mutate a stored template while debugging. Persist only when the user prompt explicitly says persistence is allowed and all runnable checks pass. For edits, read the current definition first and update the same definition. Return the required structured result with evidence tied to successful tool calls. A pass must have no remaining issues.`;
 
@@ -105,10 +106,16 @@ export function getCustomWidgetMcpToolExecutions(steps: readonly AgentToolStep[]
   return steps.flatMap(({ toolCalls }) =>
     toolCalls.map((call) => {
       const requestId = asRecord(call.input)?.requestId;
+      const output = results.get(call.toolCallId)?.output;
+      const status = asRecord(getMcpToolPayload(output))?.status;
       return {
         tool: call.toolName,
-        succeeded: isSuccessfulMcpToolOutput(call.toolName, results.get(call.toolCallId)?.output),
+        succeeded: isSuccessfulMcpToolOutput(call.toolName, output),
         ...(typeof requestId === "string" ? { requestId } : {}),
+        ...(call.toolName === "customWidget_configurationRequestUser" &&
+        (status === "pending" || status === "completed")
+          ? { configurationStatus: status }
+          : {}),
       };
     }),
   );
@@ -147,6 +154,9 @@ export function getCustomWidgetMcpWorkflowIssues(args: {
     if (!args.persist && !args.definitionId) required.push("customWidget_previewCreate");
     if (args.persist && !args.definitionId) required.push("customWidget_create");
   }
+  if (hasTool("customWidget_configurationRequestUser") && !required.includes("customWidget_configurationRequestUser")) {
+    required.push("customWidget_configurationRequestUser");
+  }
   for (const tool of required) {
     if (!hasTool(tool)) issues.push(`Required tool was not used: ${tool}`);
     else if (!hasSuccessfulTool(tool)) issues.push(`Required tool did not complete successfully: ${tool}`);
@@ -183,11 +193,22 @@ export function getCustomWidgetMcpWorkflowIssues(args: {
     }
   }
   if (args.output.status === "pass") {
+    const completedConfiguration = args.toolExecutions.findLastIndex(
+      (entry) =>
+        entry.tool === "customWidget_configurationRequestUser" &&
+        entry.succeeded &&
+        entry.configurationStatus === "completed",
+    );
+    if (hasTool("customWidget_configurationRequestUser") && completedConfiguration < 0) {
+      issues.push("A passing result requires the user configuration request to report completed");
+    }
+    const verifiedExecutions =
+      completedConfiguration >= 0 ? args.toolExecutions.slice(completedConfiguration + 1) : args.toolExecutions;
     for (const [tool, requiredCount] of [
       ["customWidget_previewQuery", args.requiredQueryCount ?? 1],
       ["customWidget_previewAction", args.requiredActionCount ?? 0],
     ] as const) {
-      const completed = args.toolExecutions.filter((entry) => entry.tool === tool && entry.succeeded);
+      const completed = verifiedExecutions.filter((entry) => entry.tool === tool && entry.succeeded);
       const distinctRequestIds = new Set(completed.flatMap(({ requestId }) => (requestId ? [requestId] : [])));
       const completedCount = distinctRequestIds.size > 0 ? distinctRequestIds.size : completed.length;
       if (completedCount < requiredCount) {
@@ -246,7 +267,13 @@ export function getCustomWidgetMcpWorkflowIssues(args: {
     } else if (!evidenceTools.has("customWidget_configurationRequestUser")) {
       issues.push("Structured evidence is missing customWidget_configurationRequestUser");
     }
-    const configurationRequest = lastSuccessfulIndex("customWidget_configurationRequestUser");
+    const finalConfiguration = args.toolExecutions.findLast(
+      (entry) => entry.tool === "customWidget_configurationRequestUser" && entry.succeeded,
+    );
+    if (finalConfiguration?.configurationStatus !== "pending") {
+      issues.push("User configuration status requires a pending configuration request");
+    }
+    const configurationRequest = firstSuccessfulIndex("customWidget_configurationRequestUser");
     const targetCreation = args.persist ? lastSuccessfulIndex(persistenceTool) : previewCreate;
     if (!args.definitionId && targetCreation >= 0 && configurationRequest < targetCreation) {
       issues.push("customWidget_configurationRequestUser must run after its definition or preview target exists");
@@ -258,7 +285,7 @@ export function getCustomWidgetMcpWorkflowIssues(args: {
 function isSuccessfulMcpToolOutput(tool: string, output: unknown) {
   const record = asRecord(output);
   if (!record || record.isError === true) return false;
-  const payload = decodeMcpTextPayload(record) ?? record;
+  const payload = getMcpToolPayload(output);
   if (tool === "customWidget_validate") return asRecord(payload)?.valid === true;
   if (tool === "customWidget_previewCreate") return asRecord(payload)?.success === true;
   if (tool === "customWidget_previewQuery" || tool === "customWidget_previewAction") {
@@ -267,6 +294,11 @@ function isSuccessfulMcpToolOutput(tool: string, output: unknown) {
   return output !== undefined;
 }
 
+function getMcpToolPayload(output: unknown) {
+  const record = asRecord(output);
+  return record ? (decodeMcpTextPayload(record) ?? record) : undefined;
+}
+
 function decodeMcpTextPayload(output: Record<string, unknown>) {
   if (!Array.isArray(output.content)) return undefined;
   const text = output.content