diff --git a/packages/custom-widgets/package.json b/packages/custom-widgets/package.json index 65dfc2f93..86133e38d 100644 --- a/packages/custom-widgets/package.json +++ b/packages/custom-widgets/package.json @@ -28,8 +28,8 @@ "generate:catalog": "tsx ./scripts/generate-component-metadata.ts", "generate:components": "pnpm generate:catalog", "generate": "pnpm generate:catalog", - "evaluate:ai": "tsx ./scripts/evaluate-ai-authoring.ts", - "test:ai": "tsx ./scripts/evaluate-ai-authoring.ts --ci", + "evaluate:ai": "bun ./scripts/evaluate-ai-authoring.ts", + "test:ai": "bun ./scripts/evaluate-ai-authoring.ts --ci", "test:reference-apis": "tsx ./scripts/verify-reference-apis.ts", "format": "oxfmt .", "lint": "oxlint", diff --git a/packages/custom-widgets/scripts/ai-evaluation.ts b/packages/custom-widgets/scripts/ai-evaluation.ts index e42a586e4..4c7c3d316 100644 --- a/packages/custom-widgets/scripts/ai-evaluation.ts +++ b/packages/custom-widgets/scripts/ai-evaluation.ts @@ -14,25 +14,58 @@ export const DEFAULT_JUDGE_MODEL = "deepseek/deepseek-v4-flash"; export const MAX_AI_EVALUATION_LOOPS = 10; const scoreSchema = z.number().int().min(0).max(100); +const judgeCategoriesSchema = z.strictObject({ + schemaAndBindings: scoreSchema, + apiAndRequestDesign: scoreSchema, + runtimeCompatibility: scoreSchema, + goalFulfillment: scoreSchema, + visualQuality: scoreSchema, + responsiveAndTheme: scoreSchema, + loadingEmptyErrorSuccess: scoreSchema, + dailyUsefulness: scoreSchema, + complexityDiscipline: scoreSchema, + accessibility: scoreSchema, + actionSafety: scoreSchema, +}); +const categoryReasonSchema = z.string().min(1).max(600); const judgeResultSchema = z.strictObject({ total: scoreSchema, verdict: z.enum(["pass", "fail"]), - categories: z.strictObject({ - schemaAndBindings: scoreSchema, - apiAndRequestDesign: scoreSchema, - runtimeCompatibility: scoreSchema, - visualHierarchy: scoreSchema, - responsiveAndTheme: scoreSchema, - loadingEmptyErrorSuccess: scoreSchema, - accessibility: scoreSchema, - actionSafety: scoreSchema, - simplicity: scoreSchema, + dailyUseDecision: z.enum(["would-use-daily", "promising-but-not-daily", "not-practical"]), + categories: judgeCategoriesSchema, + categoryReasons: z.strictObject({ + schemaAndBindings: categoryReasonSchema, + apiAndRequestDesign: categoryReasonSchema, + runtimeCompatibility: categoryReasonSchema, + goalFulfillment: categoryReasonSchema, + visualQuality: categoryReasonSchema, + responsiveAndTheme: categoryReasonSchema, + loadingEmptyErrorSuccess: categoryReasonSchema, + dailyUsefulness: categoryReasonSchema, + complexityDiscipline: categoryReasonSchema, + accessibility: categoryReasonSchema, + actionSafety: categoryReasonSchema, }), - strengths: z.array(z.string()).max(8), - problems: z.array(z.string()).max(12), - highestImpactFixes: z.array(z.string()).max(8), + strengths: z.array(z.string().min(1).max(400)).max(6), + problems: z.array(z.string().min(1).max(500)).max(12), + fatalProblems: z.array(z.string().min(1).max(500)).max(8), + highestImpactFixes: z.array(z.string().min(1).max(500)).max(6), }); +const categoryWeights = { + schemaAndBindings: 8, + apiAndRequestDesign: 10, + runtimeCompatibility: 8, + goalFulfillment: 15, + visualQuality: 15, + responsiveAndTheme: 8, + loadingEmptyErrorSuccess: 8, + dailyUsefulness: 12, + complexityDiscipline: 8, + accessibility: 4, + actionSafety: 4, +} as const satisfies Record, number>; + export type CustomWidgetJudgeResult = z.infer; export interface AiEvaluationResult { @@ -71,7 +104,7 @@ export function buildRepairPrompt( } export function buildJudgePrompt(testCase: CustomWidgetAiEvaluationCase, widget: HomarrCustomWidgetV2): string { - return `You are a strict evaluator for a safe dashboard-widget language. Judge the widget against the request and API notes. Do not reward unsupported capabilities, invented API routes, or visual claims not present in the JSX. + return `You are a hostile-but-fair product review panel evaluating a safe dashboard widget. Most competent drafts should score 55-75, not 90. Judge only evidence present in the manifest and JSX. Never reward unsupported capabilities, invented API routes, aspirational claims, or code that merely validates. Request: ${testCase.request} @@ -82,12 +115,47 @@ ${testCase.apiNotes} Validated widget: ${JSON.stringify(widget, null, 2)} -Score every category from 0 to 100. The total must reflect the whole result. A pass requires total >= 80 and no category below 65. Return JSON only with this exact shape: -{"total":0,"verdict":"fail","categories":{"schemaAndBindings":0,"apiAndRequestDesign":0,"runtimeCompatibility":0,"visualHierarchy":0,"responsiveAndTheme":0,"loadingEmptyErrorSuccess":0,"accessibility":0,"actionSafety":0,"simplicity":0},"strengths":[],"problems":[],"highestImpactFixes":[]}`; +Scoring calibration: +- 95-100: exceptional, purpose-built quality; complete, beautiful, restrained, and something a demanding user would choose every day. Almost never award this. +- 85-94: excellent with only small, specific defects. It must fully achieve the request and feel deliberately designed. +- 70-84: good prototype or useful widget with visible compromises, generic design, missing polish, or avoidable complexity. +- 50-69: functional but incomplete, awkward, visually ordinary, overbuilt, or impractical for repeated use. +- 0-49: broken, misleading, unsafe, substantially incomplete, or incompatible. + +Required review behavior: +- Compare every requested capability with concrete manifest/JSX evidence. A missing or invented core capability is fatal and caps total at 79. +- Judge whether the API design can actually reach the stated goal, including response paths, bindings, invalidation, and action safety. +- Judge visual quality, not component count: hierarchy, density, whitespace, typography, restrained color, scanability, and avoidance of repetitive nested cards. +- Judge daily usefulness: information priority, interaction cost, refresh behavior, narrow-tile usability, and whether the widget is pleasant rather than demo-like. +- Judge complexity discipline: penalize duplicate requests/options, unnecessary controls, excessive JSX, cleverness, and UI chrome that does not help the goal. Complexity must earn its place. +- A visually generic but valid widget should normally score below 75 for visualQuality. A widget that is attractive but inconvenient should score below 75 for dailyUsefulness. +- Give a concrete evidence sentence for every category. List all score-capping issues under fatalProblems. + +Homarr computes the weighted total and final verdict itself. Your total and verdict are advisory, but must be internally honest. A pass requires a weighted total of at least 90, every category at least 75, goalFulfillment/visualQuality/dailyUsefulness at least 85, complexityDiscipline at least 80, no fatal problem, and dailyUseDecision="would-use-daily". Return only the requested structured object.`; } export function judgePasses(result: CustomWidgetJudgeResult): boolean { - return result.total >= 80 && Object.values(result.categories).every((score) => score >= 65); + return ( + result.total >= 90 && + Object.values(result.categories).every((score) => score >= 75) && + result.categories.goalFulfillment >= 85 && + result.categories.visualQuality >= 85 && + result.categories.dailyUsefulness >= 85 && + result.categories.complexityDiscipline >= 80 && + result.dailyUseDecision === "would-use-daily" && + result.fatalProblems.length === 0 + ); +} + +export function getJudgeResponseFormat() { + return { + type: "json_schema" as const, + json_schema: { + name: "homarr_custom_widget_review", + strict: true, + schema: z.toJSONSchema(judgeResultSchema, { io: "output" }), + }, + }; } export async function evaluateCustomWidgetCase(args: { @@ -114,7 +182,7 @@ export async function evaluateCustomWidgetCase(args: { apiKey: args.apiKey, model: args.generatorModel ?? DEFAULT_GENERATOR_MODEL, prompt, - json: false, + purpose: "generation", }); } catch (error) { errors.push( @@ -139,7 +207,7 @@ export async function evaluateCustomWidgetCase(args: { apiKey: args.apiKey, model: args.judgeModel ?? DEFAULT_JUDGE_MODEL, prompt: buildJudgePrompt(args.testCase, canonical), - json: true, + purpose: "judge", }); await writeFile(path.join(caseDirectory, `judge-${attempt}.json`), judgeRaw, "utf8"); judge = parseJudgeResult(judgeRaw); @@ -185,12 +253,15 @@ export async function evaluateCustomWidgetCase(args: { } export function parseJudgeResult(raw: string): CustomWidgetJudgeResult { - const fenced = /```(?:json)?\s*([\s\S]*?)```/iu.exec(raw)?.[1]; - const parsed: unknown = JSON.parse(fenced ?? raw); + const parsed: unknown = JSON.parse(raw); const result = judgeResultSchema.parse(parsed); + const weightedTotal = Object.entries(categoryWeights).reduce( + (sum, [category, weight]) => sum + result.categories[category as keyof typeof categoryWeights] * weight, + 0, + ); const normalized = { ...result, - total: Math.round(Object.values(result.categories).reduce((sum, score) => sum + score, 0) / 9), + total: Math.round(weightedTotal / 100), }; return { ...normalized, verdict: judgePasses(normalized) ? "pass" : "fail" }; } @@ -207,7 +278,13 @@ async function writeWidgetFiles(directory: string, widget: HomarrCustomWidgetV2, ]); } -async function callOpenRouter(args: { apiKey: string; model: string; prompt: string; json: boolean }): Promise { +async function callOpenRouter(args: { + apiKey: string; + model: string; + prompt: string; + purpose: "generation" | "judge"; +}): Promise { + const isJudge = args.purpose === "judge"; const response = await fetch("https://openrouter.ai/api/v1/chat/completions", { method: "POST", headers: { @@ -219,10 +296,10 @@ async function callOpenRouter(args: { apiKey: string; model: string; prompt: str body: JSON.stringify({ model: args.model, messages: [{ role: "user", content: args.prompt }], - temperature: args.json ? 0 : 0.25, - max_tokens: args.json ? 8_000 : 20_000, - ...(args.json ? { reasoning: { effort: "low", exclude: true } } : {}), - ...(args.json ? { response_format: { type: "json_object" } } : {}), + temperature: isJudge ? 0 : 0.2, + max_tokens: isJudge ? 8_000 : 20_000, + reasoning: { effort: "xhigh", exclude: true }, + ...(isJudge ? { response_format: getJudgeResponseFormat() } : {}), }), signal: AbortSignal.timeout(180_000), }); diff --git a/packages/custom-widgets/src/test/ai-evaluation.spec.ts b/packages/custom-widgets/src/test/ai-evaluation.spec.ts index 48c6616a1..4fc48fda9 100644 --- a/packages/custom-widgets/src/test/ai-evaluation.spec.ts +++ b/packages/custom-widgets/src/test/ai-evaluation.spec.ts @@ -1,7 +1,49 @@ import { describe, expect, it } from "vitest"; import { CUSTOM_WIDGET_AI_EVALUATION_CASES } from "../../scripts/ai-evaluation-cases"; -import { buildEvaluationPrompt, buildRepairPrompt, judgePasses, parseJudgeResult } from "../../scripts/ai-evaluation"; +import { + buildEvaluationPrompt, + buildRepairPrompt, + DEFAULT_GENERATOR_MODEL, + DEFAULT_JUDGE_MODEL, + getJudgeResponseFormat, + judgePasses, + parseJudgeResult, +} from "../../scripts/ai-evaluation"; +import type { CustomWidgetJudgeResult } from "../../scripts/ai-evaluation"; + +const categoryNames = [ + "schemaAndBindings", + "apiAndRequestDesign", + "runtimeCompatibility", + "goalFulfillment", + "visualQuality", + "responsiveAndTheme", + "loadingEmptyErrorSuccess", + "dailyUsefulness", + "complexityDiscipline", + "accessibility", + "actionSafety", +] as const; + +const makeCategories = (score: number) => + Object.fromEntries(categoryNames.map((name) => [name, score])) as CustomWidgetJudgeResult["categories"]; +const makeReasons = () => + Object.fromEntries( + categoryNames.map((name) => [name, `Concrete evidence for ${name}.`]), + ) as CustomWidgetJudgeResult["categoryReasons"]; + +const makeJudgeResult = (score: number) => ({ + total: score, + verdict: "pass" as const, + dailyUseDecision: "would-use-daily" as const, + categories: makeCategories(score), + categoryReasons: makeReasons(), + strengths: [], + problems: [], + fatalProblems: [], + highestImpactFixes: [], +}); describe("AI authoring evaluation", () => { it("defines five distinct complex scenarios", () => { @@ -27,54 +69,40 @@ describe("AI authoring evaluation", () => { expect(prompt).toContain("bad response"); }); - it("requires a strong overall and category score", () => { - const categories = { - schemaAndBindings: 90, - apiAndRequestDesign: 90, - runtimeCompatibility: 90, - visualHierarchy: 90, - responsiveAndTheme: 90, - loadingEmptyErrorSuccess: 90, - accessibility: 90, - actionSafety: 90, - simplicity: 90, - }; - expect( - judgePasses({ total: 90, verdict: "pass", categories, strengths: [], problems: [], highestImpactFixes: [] }), - ).toBe(true); - expect( - judgePasses({ - total: 90, - verdict: "pass", - categories: { ...categories, accessibility: 64 }, - strengths: [], - problems: [], - highestImpactFixes: [], - }), - ).toBe(false); + it("uses the requested DeepSeek models and strict structured judge output", () => { + expect(DEFAULT_GENERATOR_MODEL).toBe("deepseek/deepseek-v4-pro"); + expect(DEFAULT_JUDGE_MODEL).toBe("deepseek/deepseek-v4-flash"); + const format = getJudgeResponseFormat(); + expect(format.type).toBe("json_schema"); + expect(format.json_schema.strict).toBe(true); + expect(format.json_schema.schema).toMatchObject({ type: "object", additionalProperties: false }); }); - it("normalizes a judge verdict from numeric scores", () => { + it("requires exceptional goal, design, practicality, and complexity scores", () => { + expect(judgePasses(makeJudgeResult(90))).toBe(true); + expect( + judgePasses({ + ...makeJudgeResult(92), + categories: { ...makeCategories(92), dailyUsefulness: 84 }, + }), + ).toBe(false); + expect(judgePasses({ ...makeJudgeResult(95), fatalProblems: ["A requested core action is missing."] })).toBe(false); + }); + + it("computes the weighted score and refuses inflated advisory verdicts", () => { const parsed = parseJudgeResult( JSON.stringify({ - total: 84, - verdict: "fail", - categories: { - schemaAndBindings: 85, - apiAndRequestDesign: 82, - runtimeCompatibility: 84, - visualHierarchy: 86, - responsiveAndTheme: 83, - loadingEmptyErrorSuccess: 82, - accessibility: 80, - actionSafety: 87, - simplicity: 85, - }, + ...makeJudgeResult(95), + total: 99, + categories: { ...makeCategories(95), visualQuality: 70 }, + categoryReasons: makeReasons(), strengths: [], problems: [], + fatalProblems: [], highestImpactFixes: [], }), ); - expect(parsed.verdict).toBe("pass"); + expect(parsed.total).toBe(91); + expect(parsed.verdict).toBe("fail"); }); }); diff --git a/packages/custom-widgets/src/test/authoring-resources.spec.ts b/packages/custom-widgets/src/test/authoring-resources.spec.ts index 0bfac4b7d..dc9cf1245 100644 --- a/packages/custom-widgets/src/test/authoring-resources.spec.ts +++ b/packages/custom-widgets/src/test/authoring-resources.spec.ts @@ -1,4 +1,5 @@ import { readFileSync } from "node:fs"; +import path from "node:path"; import { describe, expect, it } from "vitest"; import { @@ -19,7 +20,7 @@ describe("authoring resources", () => { expect(skill).not.toHaveProperty("componentCatalog"); expect(getCustomWidgetSkillContent()).toBe(skill.skillMd); expect(skill.skillMd.trim()).toBe( - readFileSync(new URL("../../../../.agents/skills/homarr-custom-widget/SKILL.md", import.meta.url), "utf8").trim(), + readFileSync(path.join(process.cwd(), ".agents/skills/homarr-custom-widget/SKILL.md"), "utf8").trim(), ); });