Harden AI widget evaluation

This commit is contained in:
Thomas Camlong committed 2026-07-21 08:39:50 +02:00
1 parent 51644985eb
commit 49193c37e0
4 files changed
+177 -71

No files matched your search

@@ -1,7 +1,49 @@
import { describe, expect, it } from "vitest";
import { CUSTOM_WIDGET_AI_EVALUATION_CASES } from "../../scripts/ai-evaluation-cases";
import { buildEvaluationPrompt, buildRepairPrompt, judgePasses, parseJudgeResult } from "../../scripts/ai-evaluation";
import {
buildEvaluationPrompt,
buildRepairPrompt,
DEFAULT_GENERATOR_MODEL,
DEFAULT_JUDGE_MODEL,
getJudgeResponseFormat,
judgePasses,
parseJudgeResult,
} from "../../scripts/ai-evaluation";
import type { CustomWidgetJudgeResult } from "../../scripts/ai-evaluation";
const categoryNames = [
"schemaAndBindings",
"apiAndRequestDesign",
"runtimeCompatibility",
"goalFulfillment",
"visualQuality",
"responsiveAndTheme",
"loadingEmptyErrorSuccess",
"dailyUsefulness",
"complexityDiscipline",
"accessibility",
"actionSafety",
] as const;
const makeCategories = (score: number) =>
Object.fromEntries(categoryNames.map((name) => [name, score])) as CustomWidgetJudgeResult["categories"];
const makeReasons = () =>
Object.fromEntries(
categoryNames.map((name) => [name, `Concrete evidence for ${name}.`]),
) as CustomWidgetJudgeResult["categoryReasons"];
const makeJudgeResult = (score: number) => ({
total: score,
verdict: "pass" as const,
dailyUseDecision: "would-use-daily" as const,
categories: makeCategories(score),
categoryReasons: makeReasons(),
strengths: [],
problems: [],
fatalProblems: [],
highestImpactFixes: [],
});
describe("AI authoring evaluation", () => {
it("defines five distinct complex scenarios", () => {
@@ -27,54 +69,40 @@ describe("AI authoring evaluation", () => {
expect(prompt).toContain("bad response");
});
it("requires a strong overall and category score", () => {
const categories = {
schemaAndBindings: 90,
apiAndRequestDesign: 90,
runtimeCompatibility: 90,
visualHierarchy: 90,
responsiveAndTheme: 90,
loadingEmptyErrorSuccess: 90,
accessibility: 90,
actionSafety: 90,
simplicity: 90,
};
expect(
judgePasses({ total: 90, verdict: "pass", categories, strengths: [], problems: [], highestImpactFixes: [] }),
).toBe(true);
expect(
judgePasses({
total: 90,
verdict: "pass",
categories: { ...categories, accessibility: 64 },
strengths: [],
problems: [],
highestImpactFixes: [],
}),
).toBe(false);
it("uses the requested DeepSeek models and strict structured judge output", () => {
expect(DEFAULT_GENERATOR_MODEL).toBe("deepseek/deepseek-v4-pro");
expect(DEFAULT_JUDGE_MODEL).toBe("deepseek/deepseek-v4-flash");
const format = getJudgeResponseFormat();
expect(format.type).toBe("json_schema");
expect(format.json_schema.strict).toBe(true);
expect(format.json_schema.schema).toMatchObject({ type: "object", additionalProperties: false });
});
it("normalizes a judge verdict from numeric scores", () => {
it("requires exceptional goal, design, practicality, and complexity scores", () => {
expect(judgePasses(makeJudgeResult(90))).toBe(true);
expect(
judgePasses({
...makeJudgeResult(92),
categories: { ...makeCategories(92), dailyUsefulness: 84 },
}),
).toBe(false);
expect(judgePasses({ ...makeJudgeResult(95), fatalProblems: ["A requested core action is missing."] })).toBe(false);
});
it("computes the weighted score and refuses inflated advisory verdicts", () => {
const parsed = parseJudgeResult(
JSON.stringify({
total: 84,
verdict: "fail",
categories: {
schemaAndBindings: 85,
apiAndRequestDesign: 82,
runtimeCompatibility: 84,
visualHierarchy: 86,
responsiveAndTheme: 83,
loadingEmptyErrorSuccess: 82,
accessibility: 80,
actionSafety: 87,
simplicity: 85,
},
...makeJudgeResult(95),
total: 99,
categories: { ...makeCategories(95), visualQuality: 70 },
categoryReasons: makeReasons(),
strengths: [],
problems: [],
fatalProblems: [],
highestImpactFixes: [],
}),
);
expect(parsed.verdict).toBe("pass");
expect(parsed.total).toBe(91);
expect(parsed.verdict).toBe("fail");
});
});
@@ -1,4 +1,5 @@
import { readFileSync } from "node:fs";
import path from "node:path";
import { describe, expect, it } from "vitest";
import {
@@ -19,7 +20,7 @@ describe("authoring resources", () => {
expect(skill).not.toHaveProperty("componentCatalog");
expect(getCustomWidgetSkillContent()).toBe(skill.skillMd);
expect(skill.skillMd.trim()).toBe(
readFileSync(new URL("../../../../.agents/skills/homarr-custom-widget/SKILL.md", import.meta.url), "utf8").trim(),
readFileSync(path.join(process.cwd(), ".agents/skills/homarr-custom-widget/SKILL.md"), "utf8").trim(),
);
});