Harden AI widget evaluation

packages/custom-widgets/package.json

@@ -28,8 +28,8 @@
     "generate:catalog": "tsx ./scripts/generate-component-metadata.ts",
     "generate:components": "pnpm generate:catalog",
     "generate": "pnpm generate:catalog",
-    "evaluate:ai": "tsx ./scripts/evaluate-ai-authoring.ts",
-    "test:ai": "tsx ./scripts/evaluate-ai-authoring.ts --ci",
+    "evaluate:ai": "bun ./scripts/evaluate-ai-authoring.ts",
+    "test:ai": "bun ./scripts/evaluate-ai-authoring.ts --ci",
     "test:reference-apis": "tsx ./scripts/verify-reference-apis.ts",
     "format": "oxfmt .",
     "lint": "oxlint",

packages/custom-widgets/scripts/ai-evaluation.ts

@@ -14,25 +14,58 @@ export const DEFAULT_JUDGE_MODEL = "deepseek/deepseek-v4-flash";
 export const MAX_AI_EVALUATION_LOOPS = 10;
 
 const scoreSchema = z.number().int().min(0).max(100);
+const judgeCategoriesSchema = z.strictObject({
+  schemaAndBindings: scoreSchema,
+  apiAndRequestDesign: scoreSchema,
+  runtimeCompatibility: scoreSchema,
+  goalFulfillment: scoreSchema,
+  visualQuality: scoreSchema,
+  responsiveAndTheme: scoreSchema,
+  loadingEmptyErrorSuccess: scoreSchema,
+  dailyUsefulness: scoreSchema,
+  complexityDiscipline: scoreSchema,
+  accessibility: scoreSchema,
+  actionSafety: scoreSchema,
+});
+const categoryReasonSchema = z.string().min(1).max(600);
 const judgeResultSchema = z.strictObject({
   total: scoreSchema,
   verdict: z.enum(["pass", "fail"]),
-  categories: z.strictObject({
-    schemaAndBindings: scoreSchema,
-    apiAndRequestDesign: scoreSchema,
-    runtimeCompatibility: scoreSchema,
-    visualHierarchy: scoreSchema,
-    responsiveAndTheme: scoreSchema,
-    loadingEmptyErrorSuccess: scoreSchema,
-    accessibility: scoreSchema,
-    actionSafety: scoreSchema,
-    simplicity: scoreSchema,
+  dailyUseDecision: z.enum(["would-use-daily", "promising-but-not-daily", "not-practical"]),
+  categories: judgeCategoriesSchema,
+  categoryReasons: z.strictObject({
+    schemaAndBindings: categoryReasonSchema,
+    apiAndRequestDesign: categoryReasonSchema,
+    runtimeCompatibility: categoryReasonSchema,
+    goalFulfillment: categoryReasonSchema,
+    visualQuality: categoryReasonSchema,
+    responsiveAndTheme: categoryReasonSchema,
+    loadingEmptyErrorSuccess: categoryReasonSchema,
+    dailyUsefulness: categoryReasonSchema,
+    complexityDiscipline: categoryReasonSchema,
+    accessibility: categoryReasonSchema,
+    actionSafety: categoryReasonSchema,
   }),
-  strengths: z.array(z.string()).max(8),
-  problems: z.array(z.string()).max(12),
-  highestImpactFixes: z.array(z.string()).max(8),
+  strengths: z.array(z.string().min(1).max(400)).max(6),
+  problems: z.array(z.string().min(1).max(500)).max(12),
+  fatalProblems: z.array(z.string().min(1).max(500)).max(8),
+  highestImpactFixes: z.array(z.string().min(1).max(500)).max(6),
 });
 
+const categoryWeights = {
+  schemaAndBindings: 8,
+  apiAndRequestDesign: 10,
+  runtimeCompatibility: 8,
+  goalFulfillment: 15,
+  visualQuality: 15,
+  responsiveAndTheme: 8,
+  loadingEmptyErrorSuccess: 8,
+  dailyUsefulness: 12,
+  complexityDiscipline: 8,
+  accessibility: 4,
+  actionSafety: 4,
+} as const satisfies Record<keyof z.infer<typeof judgeCategoriesSchema>, number>;
+
 export type CustomWidgetJudgeResult = z.infer<typeof judgeResultSchema>;
 
 export interface AiEvaluationResult {
@@ -71,7 +104,7 @@ export function buildRepairPrompt(
 }
 
 export function buildJudgePrompt(testCase: CustomWidgetAiEvaluationCase, widget: HomarrCustomWidgetV2): string {
-  return `You are a strict evaluator for a safe dashboard-widget language. Judge the widget against the request and API notes. Do not reward unsupported capabilities, invented API routes, or visual claims not present in the JSX.
+  return `You are a hostile-but-fair product review panel evaluating a safe dashboard widget. Most competent drafts should score 55-75, not 90. Judge only evidence present in the manifest and JSX. Never reward unsupported capabilities, invented API routes, aspirational claims, or code that merely validates.
 
 Request:
 ${testCase.request}
@@ -82,12 +115,47 @@ ${testCase.apiNotes}
 Validated widget:
 ${JSON.stringify(widget, null, 2)}
 
-Score every category from 0 to 100. The total must reflect the whole result. A pass requires total >= 80 and no category below 65. Return JSON only with this exact shape:
-{"total":0,"verdict":"fail","categories":{"schemaAndBindings":0,"apiAndRequestDesign":0,"runtimeCompatibility":0,"visualHierarchy":0,"responsiveAndTheme":0,"loadingEmptyErrorSuccess":0,"accessibility":0,"actionSafety":0,"simplicity":0},"strengths":[],"problems":[],"highestImpactFixes":[]}`;
+Scoring calibration:
+- 95-100: exceptional, purpose-built quality; complete, beautiful, restrained, and something a demanding user would choose every day. Almost never award this.
+- 85-94: excellent with only small, specific defects. It must fully achieve the request and feel deliberately designed.
+- 70-84: good prototype or useful widget with visible compromises, generic design, missing polish, or avoidable complexity.
+- 50-69: functional but incomplete, awkward, visually ordinary, overbuilt, or impractical for repeated use.
+- 0-49: broken, misleading, unsafe, substantially incomplete, or incompatible.
+
+Required review behavior:
+- Compare every requested capability with concrete manifest/JSX evidence. A missing or invented core capability is fatal and caps total at 79.
+- Judge whether the API design can actually reach the stated goal, including response paths, bindings, invalidation, and action safety.
+- Judge visual quality, not component count: hierarchy, density, whitespace, typography, restrained color, scanability, and avoidance of repetitive nested cards.
+- Judge daily usefulness: information priority, interaction cost, refresh behavior, narrow-tile usability, and whether the widget is pleasant rather than demo-like.
+- Judge complexity discipline: penalize duplicate requests/options, unnecessary controls, excessive JSX, cleverness, and UI chrome that does not help the goal. Complexity must earn its place.
+- A visually generic but valid widget should normally score below 75 for visualQuality. A widget that is attractive but inconvenient should score below 75 for dailyUsefulness.
+- Give a concrete evidence sentence for every category. List all score-capping issues under fatalProblems.
+
+Homarr computes the weighted total and final verdict itself. Your total and verdict are advisory, but must be internally honest. A pass requires a weighted total of at least 90, every category at least 75, goalFulfillment/visualQuality/dailyUsefulness at least 85, complexityDiscipline at least 80, no fatal problem, and dailyUseDecision="would-use-daily". Return only the requested structured object.`;
 }
 
 export function judgePasses(result: CustomWidgetJudgeResult): boolean {
-  return result.total >= 80 && Object.values(result.categories).every((score) => score >= 65);
+  return (
+    result.total >= 90 &&
+    Object.values(result.categories).every((score) => score >= 75) &&
+    result.categories.goalFulfillment >= 85 &&
+    result.categories.visualQuality >= 85 &&
+    result.categories.dailyUsefulness >= 85 &&
+    result.categories.complexityDiscipline >= 80 &&
+    result.dailyUseDecision === "would-use-daily" &&
+    result.fatalProblems.length === 0
+  );
+}
+
+export function getJudgeResponseFormat() {
+  return {
+    type: "json_schema" as const,
+    json_schema: {
+      name: "homarr_custom_widget_review",
+      strict: true,
+      schema: z.toJSONSchema(judgeResultSchema, { io: "output" }),
+    },
+  };
 }
 
 export async function evaluateCustomWidgetCase(args: {
@@ -114,7 +182,7 @@ export async function evaluateCustomWidgetCase(args: {
         apiKey: args.apiKey,
         model: args.generatorModel ?? DEFAULT_GENERATOR_MODEL,
         prompt,
-        json: false,
+        purpose: "generation",
       });
     } catch (error) {
       errors.push(
@@ -139,7 +207,7 @@ export async function evaluateCustomWidgetCase(args: {
         apiKey: args.apiKey,
         model: args.judgeModel ?? DEFAULT_JUDGE_MODEL,
         prompt: buildJudgePrompt(args.testCase, canonical),
-        json: true,
+        purpose: "judge",
       });
       await writeFile(path.join(caseDirectory, `judge-${attempt}.json`), judgeRaw, "utf8");
       judge = parseJudgeResult(judgeRaw);
@@ -185,12 +253,15 @@ export async function evaluateCustomWidgetCase(args: {
 }
 
 export function parseJudgeResult(raw: string): CustomWidgetJudgeResult {
-  const fenced = /```(?:json)?\s*([\s\S]*?)```/iu.exec(raw)?.[1];
-  const parsed: unknown = JSON.parse(fenced ?? raw);
+  const parsed: unknown = JSON.parse(raw);
   const result = judgeResultSchema.parse(parsed);
+  const weightedTotal = Object.entries(categoryWeights).reduce(
+    (sum, [category, weight]) => sum + result.categories[category as keyof typeof categoryWeights] * weight,
+    0,
+  );
   const normalized = {
     ...result,
-    total: Math.round(Object.values(result.categories).reduce((sum, score) => sum + score, 0) / 9),
+    total: Math.round(weightedTotal / 100),
   };
   return { ...normalized, verdict: judgePasses(normalized) ? "pass" : "fail" };
 }
@@ -207,7 +278,13 @@ async function writeWidgetFiles(directory: string, widget: HomarrCustomWidgetV2,
   ]);
 }
 
-async function callOpenRouter(args: { apiKey: string; model: string; prompt: string; json: boolean }): Promise<string> {
+async function callOpenRouter(args: {
+  apiKey: string;
+  model: string;
+  prompt: string;
+  purpose: "generation" | "judge";
+}): Promise<string> {
+  const isJudge = args.purpose === "judge";
   const response = await fetch("https://openrouter.ai/api/v1/chat/completions", {
     method: "POST",
     headers: {
@@ -219,10 +296,10 @@ async function callOpenRouter(args: { apiKey: string; model: string; prompt: str
     body: JSON.stringify({
       model: args.model,
       messages: [{ role: "user", content: args.prompt }],
-      temperature: args.json ? 0 : 0.25,
-      max_tokens: args.json ? 8_000 : 20_000,
-      ...(args.json ? { reasoning: { effort: "low", exclude: true } } : {}),
-      ...(args.json ? { response_format: { type: "json_object" } } : {}),
+      temperature: isJudge ? 0 : 0.2,
+      max_tokens: isJudge ? 8_000 : 20_000,
+      reasoning: { effort: "xhigh", exclude: true },
+      ...(isJudge ? { response_format: getJudgeResponseFormat() } : {}),
     }),
     signal: AbortSignal.timeout(180_000),
   });

packages/custom-widgets/src/test/ai-evaluation.spec.ts

@@ -1,7 +1,49 @@
 import { describe, expect, it } from "vitest";
 
 import { CUSTOM_WIDGET_AI_EVALUATION_CASES } from "../../scripts/ai-evaluation-cases";
-import { buildEvaluationPrompt, buildRepairPrompt, judgePasses, parseJudgeResult } from "../../scripts/ai-evaluation";
+import {
+  buildEvaluationPrompt,
+  buildRepairPrompt,
+  DEFAULT_GENERATOR_MODEL,
+  DEFAULT_JUDGE_MODEL,
+  getJudgeResponseFormat,
+  judgePasses,
+  parseJudgeResult,
+} from "../../scripts/ai-evaluation";
+import type { CustomWidgetJudgeResult } from "../../scripts/ai-evaluation";
+
+const categoryNames = [
+  "schemaAndBindings",
+  "apiAndRequestDesign",
+  "runtimeCompatibility",
+  "goalFulfillment",
+  "visualQuality",
+  "responsiveAndTheme",
+  "loadingEmptyErrorSuccess",
+  "dailyUsefulness",
+  "complexityDiscipline",
+  "accessibility",
+  "actionSafety",
+] as const;
+
+const makeCategories = (score: number) =>
+  Object.fromEntries(categoryNames.map((name) => [name, score])) as CustomWidgetJudgeResult["categories"];
+const makeReasons = () =>
+  Object.fromEntries(
+    categoryNames.map((name) => [name, `Concrete evidence for ${name}.`]),
+  ) as CustomWidgetJudgeResult["categoryReasons"];
+
+const makeJudgeResult = (score: number) => ({
+  total: score,
+  verdict: "pass" as const,
+  dailyUseDecision: "would-use-daily" as const,
+  categories: makeCategories(score),
+  categoryReasons: makeReasons(),
+  strengths: [],
+  problems: [],
+  fatalProblems: [],
+  highestImpactFixes: [],
+});
 
 describe("AI authoring evaluation", () => {
   it("defines five distinct complex scenarios", () => {
@@ -27,54 +69,40 @@ describe("AI authoring evaluation", () => {
     expect(prompt).toContain("bad response");
   });
 
-  it("requires a strong overall and category score", () => {
-    const categories = {
-      schemaAndBindings: 90,
-      apiAndRequestDesign: 90,
-      runtimeCompatibility: 90,
-      visualHierarchy: 90,
-      responsiveAndTheme: 90,
-      loadingEmptyErrorSuccess: 90,
-      accessibility: 90,
-      actionSafety: 90,
-      simplicity: 90,
-    };
-    expect(
-      judgePasses({ total: 90, verdict: "pass", categories, strengths: [], problems: [], highestImpactFixes: [] }),
-    ).toBe(true);
-    expect(
-      judgePasses({
-        total: 90,
-        verdict: "pass",
-        categories: { ...categories, accessibility: 64 },
-        strengths: [],
-        problems: [],
-        highestImpactFixes: [],
-      }),
-    ).toBe(false);
+  it("uses the requested DeepSeek models and strict structured judge output", () => {
+    expect(DEFAULT_GENERATOR_MODEL).toBe("deepseek/deepseek-v4-pro");
+    expect(DEFAULT_JUDGE_MODEL).toBe("deepseek/deepseek-v4-flash");
+    const format = getJudgeResponseFormat();
+    expect(format.type).toBe("json_schema");
+    expect(format.json_schema.strict).toBe(true);
+    expect(format.json_schema.schema).toMatchObject({ type: "object", additionalProperties: false });
   });
 
-  it("normalizes a judge verdict from numeric scores", () => {
+  it("requires exceptional goal, design, practicality, and complexity scores", () => {
+    expect(judgePasses(makeJudgeResult(90))).toBe(true);
+    expect(
+      judgePasses({
+        ...makeJudgeResult(92),
+        categories: { ...makeCategories(92), dailyUsefulness: 84 },
+      }),
+    ).toBe(false);
+    expect(judgePasses({ ...makeJudgeResult(95), fatalProblems: ["A requested core action is missing."] })).toBe(false);
+  });
+
+  it("computes the weighted score and refuses inflated advisory verdicts", () => {
     const parsed = parseJudgeResult(
       JSON.stringify({
-        total: 84,
-        verdict: "fail",
-        categories: {
-          schemaAndBindings: 85,
-          apiAndRequestDesign: 82,
-          runtimeCompatibility: 84,
-          visualHierarchy: 86,
-          responsiveAndTheme: 83,
-          loadingEmptyErrorSuccess: 82,
-          accessibility: 80,
-          actionSafety: 87,
-          simplicity: 85,
-        },
+        ...makeJudgeResult(95),
+        total: 99,
+        categories: { ...makeCategories(95), visualQuality: 70 },
+        categoryReasons: makeReasons(),
         strengths: [],
         problems: [],
+        fatalProblems: [],
         highestImpactFixes: [],
       }),
     );
-    expect(parsed.verdict).toBe("pass");
+    expect(parsed.total).toBe(91);
+    expect(parsed.verdict).toBe("fail");
   });
 });

packages/custom-widgets/src/test/authoring-resources.spec.ts

@@ -1,4 +1,5 @@
 import { readFileSync } from "node:fs";
+import path from "node:path";
 import { describe, expect, it } from "vitest";
 
 import {
@@ -19,7 +20,7 @@ describe("authoring resources", () => {
     expect(skill).not.toHaveProperty("componentCatalog");
     expect(getCustomWidgetSkillContent()).toBe(skill.skillMd);
     expect(skill.skillMd.trim()).toBe(
-      readFileSync(new URL("../../../../.agents/skills/homarr-custom-widget/SKILL.md", import.meta.url), "utf8").trim(),
+      readFileSync(path.join(process.cwd(), ".agents/skills/homarr-custom-widget/SKILL.md"), "utf8").trim(),
     );
   });