test custom widget assistant authoring

packages/custom-widgets/scripts/ai-evaluation.ts

@@ -4,13 +4,16 @@ import path from "node:path";
 import { z } from "zod/v4";
 
 import { buildCustomWidgetAiPrompt } from "../src/core/ai-prompt";
+import { getCustomWidgetSkillContent } from "../src/core/authoring-resources";
 import { customWidgetDefinitionSchema } from "../src/core/custom-jsx-schema";
 import type { HomarrCustomWidgetV2 } from "../src/core/custom-jsx-schema";
 import { formatCustomWidgetImportIssues, parseCustomWidgetAiResponse } from "../src/core/import";
 import type { CustomWidgetAiEvaluationCase } from "./ai-evaluation-cases";
 
-export const DEFAULT_GENERATOR_MODEL = "deepseek/deepseek-v4-pro";
-export const DEFAULT_JUDGE_MODEL = "deepseek/deepseek-v4-flash";
+// OpenRouter exposes the rolling "latest" route with a leading tilde. The non-tilde
+// deepseek/deepseek-v4-flash-latest alias is rejected by the chat-completions API.
+export const DEFAULT_GENERATOR_MODEL = "~deepseek/deepseek-v4-flash-latest";
+export const DEFAULT_JUDGE_MODEL = "~deepseek/deepseek-v4-flash-latest";
 export const MAX_AI_EVALUATION_LOOPS = 10;
 
 const scoreSchema = z.number().int().min(0).max(100);
@@ -85,13 +88,101 @@ interface OpenRouterResponse {
 export function buildEvaluationPrompt(testCase: CustomWidgetAiEvaluationCase): string {
   return buildCustomWidgetAiPrompt(
     undefined,
-    null,
+    getEvaluationResponseFixtureText(testCase),
     null,
     `${testCase.request}\n\nVerified API notes:\n${testCase.apiNotes}`,
     testCase.documentationUrl,
   );
 }
 
+export function getEvaluationResponseFixtureText(testCase: CustomWidgetAiEvaluationCase): string | null {
+  if (testCase.previewResponses?.length) {
+    return testCase.previewResponses
+      .map(({ pathIncludes, response }) => `${pathIncludes}:\n${JSON.stringify(response, null, 2)}`)
+      .join("\n\n");
+  }
+  return testCase.sampleResponse === undefined ? null : JSON.stringify(testCase.sampleResponse, null, 2);
+}
+
+const getExpectedBindingValue = (value: unknown): string => {
+  if (typeof value === "string") return value;
+  if (typeof value !== "object" || value === null || Array.isArray(value)) return JSON.stringify(value);
+  if ("$param" in value && typeof value.$param === "string") return `$param:${value.$param}`;
+  if ("$option" in value && typeof value.$option === "string") return `$option:${value.$option}`;
+  return JSON.stringify(value);
+};
+
+export function getDeterministicEvaluationIssues(
+  testCase: CustomWidgetAiEvaluationCase,
+  widget: HomarrCustomWidgetV2,
+): Array<{ path?: Array<string | number>; message: string }> {
+  const expectations = testCase.expectations;
+  if (!expectations) return [];
+  const issues: Array<{ path?: Array<string | number>; message: string }> = [];
+  const source = widget.sources.default;
+  if (!source || source.baseUrl !== expectations.sourceBaseUrl) {
+    issues.push({
+      path: ["sources", "default", "baseUrl"],
+      message: `Use the verified source URL ${expectations.sourceBaseUrl}.`,
+    });
+  }
+  const authType = typeof source?.auth === "string" ? source.auth : source?.auth.type;
+  if (authType !== expectations.sourceAuth) {
+    issues.push({
+      path: ["sources", "default", "auth"],
+      message: `Use the verified ${expectations.sourceAuth} authentication mode.`,
+    });
+  }
+  if (
+    expectations.sourceAuth === "apiKeyQuery" &&
+    typeof source?.auth === "object" &&
+    source.auth.name !== expectations.sourceAuthName
+  ) {
+    issues.push({
+      path: ["sources", "default", "auth", "name"],
+      message: `Use '${expectations.sourceAuthName}' as the verified API-key query parameter.`,
+    });
+  }
+  const matchedRequestIds = new Set<string>();
+  for (const expected of expectations.requests) {
+    const match = Object.entries(widget.requests).find(([requestId, request]) => {
+      if (matchedRequestIds.has(requestId)) return false;
+      if (
+        request.kind !== expected.kind ||
+        request.method !== expected.method ||
+        !request.path.includes(expected.pathIncludes) ||
+        (expected.trigger !== undefined && request.trigger !== expected.trigger)
+      ) {
+        return false;
+      }
+      return Object.entries(expected.queryIncludes ?? {}).every(
+        ([key, value]) => getExpectedBindingValue(request.query?.[key]) === value,
+      );
+    });
+    if (!match) {
+      issues.push({
+        path: ["requests"],
+        message:
+          `Define the verified ${expected.trigger ?? ""} ${expected.kind} request ${expected.method} ${expected.pathIncludes} with its documented query bindings.`.replace(
+            /\s+/gu,
+            " ",
+          ),
+      });
+    } else {
+      matchedRequestIds.add(match[0]);
+    }
+  }
+  for (const requiredText of expectations.templateIncludes ?? []) {
+    if (!widget.template.includes(requiredText)) {
+      issues.push({
+        path: ["template"],
+        message: `Render or use the required '${requiredText}' capability from the verified response and request.`,
+      });
+    }
+  }
+  return issues;
+}
+
 export function buildRepairPrompt(
   originalPrompt: string,
   previousResponse: string,
@@ -106,12 +197,20 @@ export function buildRepairPrompt(
 export function buildJudgePrompt(testCase: CustomWidgetAiEvaluationCase, widget: HomarrCustomWidgetV2): string {
   return `You are a hostile-but-fair product review panel evaluating a safe dashboard widget. Most competent drafts should score 55-75, not 90. Judge only evidence present in the manifest and JSX. Never reward unsupported capabilities, invented API routes, aspirational claims, or code that merely validates.
 
+The installed Homarr skill and runtime references below are authoritative. In particular, request state is exposed as status.<requestId> with loading/ok/status/error fields while successful payloads are exposed as data.<requestId>. RefreshButton is an installed runtime helper that manually refreshes load queries. Every component in this already-validated template exists in the installed release. Do not penalize those documented facts. The widget has already passed Homarr's real schema and JSX analyzer, which proves syntax and component compatibility but does not prove API correctness, visual quality, usefulness, or accessibility.
+
+Installed Homarr Custom Widget authoring contract:
+${getCustomWidgetSkillContent()}
+
 Request:
 ${testCase.request}
 
 Verified API notes:
 ${testCase.apiNotes}
 
+Representative API response used by the preview test:
+${getEvaluationResponseFixtureText(testCase) ?? "Not supplied."}
+
 Validated widget:
 ${JSON.stringify(widget, null, 2)}
 
@@ -128,8 +227,10 @@ Required review behavior:
 - Judge visual quality, not component count: hierarchy, density, whitespace, typography, restrained color, scanability, and avoidance of repetitive nested cards.
 - Judge daily usefulness: information priority, interaction cost, refresh behavior, narrow-tile usability, and whether the widget is pleasant rather than demo-like.
 - Judge complexity discipline: penalize duplicate requests/options, unnecessary controls, excessive JSX, cleverness, and UI chrome that does not help the goal. Complexity must earn its place.
+- Recommend only interactions supported by the installed authoring contract. Do not suggest portals, modals, arbitrary event handlers, or other blocked capabilities. Prefer a responsive in-widget detail area when separation is useful.
 - A visually generic but valid widget should normally score below 75 for visualQuality. A widget that is attractive but inconvenient should score below 75 for dailyUsefulness.
 - Give a concrete evidence sentence for every category. List all score-capping issues under fatalProblems.
+- Return an empty fatalProblems array when there are no fatal problems; never put "none" or an explanation of their absence in that array.
 
 Homarr computes the weighted total and final verdict itself. Your total and verdict are advisory, but must be internally honest. A pass requires a weighted total of at least 90, every category at least 75, goalFulfillment/visualQuality/dailyUsefulness at least 85, complexityDiscipline at least 80, no fatal problem, and dailyUseDecision="would-use-daily". Return only the requested structured object.`;
 }
@@ -158,6 +259,30 @@ export function getJudgeResponseFormat() {
   };
 }
 
+export async function requestCustomWidgetJudge(args: {
+  testCase: CustomWidgetAiEvaluationCase;
+  widget: HomarrCustomWidgetV2;
+  apiKey: string;
+  judgeModel?: string;
+}) {
+  return callOpenRouter({
+    apiKey: args.apiKey,
+    model: args.judgeModel ?? DEFAULT_JUDGE_MODEL,
+    prompt: buildJudgePrompt(args.testCase, args.widget),
+    purpose: "judge",
+  });
+}
+
+export async function judgeCustomWidgetCase(args: {
+  testCase: CustomWidgetAiEvaluationCase;
+  widget: HomarrCustomWidgetV2;
+  apiKey: string;
+  judgeModel?: string;
+}) {
+  const raw = await requestCustomWidgetJudge(args);
+  return { raw, result: parseJudgeResult(raw) };
+}
+
 export async function evaluateCustomWidgetCase(args: {
   testCase: CustomWidgetAiEvaluationCase;
   apiKey: string;
@@ -200,17 +325,25 @@ export async function evaluateCustomWidgetCase(args: {
     }
 
     const canonical = customWidgetDefinitionSchema.parse(parsed.widget);
+    const deterministicIssues = getDeterministicEvaluationIssues(args.testCase, canonical);
+    if (deterministicIssues.length > 0) {
+      errors.push(
+        `Attempt ${attempt}: deterministic scenario checks failed — ${deterministicIssues.map((issue) => issue.message).join("; ")}`,
+      );
+      prompt = buildRepairPrompt(originalPrompt, response, deterministicIssues);
+      continue;
+    }
     await writeWidgetFiles(caseDirectory, canonical, `attempt-${attempt}`);
     let judge: CustomWidgetJudgeResult;
     try {
-      const judgeRaw = await callOpenRouter({
+      const { raw: judgeRaw, result } = await judgeCustomWidgetCase({
+        testCase: args.testCase,
+        widget: canonical,
         apiKey: args.apiKey,
-        model: args.judgeModel ?? DEFAULT_JUDGE_MODEL,
-        prompt: buildJudgePrompt(args.testCase, canonical),
-        purpose: "judge",
+        judgeModel: args.judgeModel,
       });
       await writeFile(path.join(caseDirectory, `judge-${attempt}.json`), judgeRaw, "utf8");
-      judge = parseJudgeResult(judgeRaw);
+      judge = result;
     } catch (error) {
       errors.push(
         `Attempt ${attempt}: judge response invalid — ${error instanceof Error ? error.message : "Unknown error"}`,
@@ -253,14 +386,22 @@ export async function evaluateCustomWidgetCase(args: {
 }
 
 export function parseJudgeResult(raw: string): CustomWidgetJudgeResult {
-  const parsed: unknown = JSON.parse(raw);
+  const normalizedRaw = raw
+    .trim()
+    .replace(/^```(?:json)?\s*/iu, "")
+    .replace(/\s*```$/u, "");
+  const parsed: unknown = JSON.parse(normalizedRaw);
   const result = judgeResultSchema.parse(parsed);
+  const fatalProblems = result.fatalProblems.filter(
+    (problem) => !/^(?:none|no fatal problems?)(?:\b|\s*[:.-])/iu.test(problem.trim()),
+  );
   const weightedTotal = Object.entries(categoryWeights).reduce(
     (sum, [category, weight]) => sum + result.categories[category as keyof typeof categoryWeights] * weight,
     0,
   );
   const normalized = {
     ...result,
+    fatalProblems,
     total: Math.round(weightedTotal / 100),
   };
   return { ...normalized, verdict: judgePasses(normalized) ? "pass" : "fail" };
@@ -290,7 +431,7 @@ async function callOpenRouter(args: {
       messages: [{ role: "user", content: args.prompt }],
       temperature: isJudge ? 0 : 0.2,
       max_tokens: isJudge ? 8_000 : 20_000,
-      reasoning: { effort: "xhigh", exclude: true },
+      reasoning: isJudge ? { enabled: false, exclude: true } : { effort: "high", exclude: true },
       ...(isJudge ? { response_format: getJudgeResponseFormat() } : {}),
     }),
     signal: AbortSignal.timeout(180_000),