import { describe, expect, it } from "vitest"; import { CUSTOM_WIDGET_AI_EVALUATION_CASES } from "../../scripts/ai-evaluation-cases"; import { buildEvaluationPrompt, buildJudgePrompt, buildRepairPrompt, DEFAULT_AI_PROVIDER_BASE_URL, DEFAULT_GENERATOR_MODEL, DEFAULT_JUDGE_MODEL, getAiProviderChatCompletionsUrl, getAiEvaluationProviderPreferences, getAiEvaluationReasoningOptions, getDeterministicEvaluationIssues, getEvaluationResponseFixtureText, getExpectedWidgetCase, getAiEvaluationMaxOutputTokens, getJudgeResponseFormat, judgePasses, parseJudgeResult, resolveAiEvaluationProviderConfig, summarizeCustomWidgetJudgePanel, } from "../../scripts/ai-evaluation"; import type { CustomWidgetJudgePanelCall, CustomWidgetJudgeResult } from "../../scripts/ai-evaluation"; import { getCustomWidgetExample } from "../core/authoring-resources"; import { customWidgetDefinitionSchema } from "../core/custom-jsx-schema"; const categoryNames = [ "schemaAndBindings", "apiAndRequestDesign", "runtimeCompatibility", "goalFulfillment", "visualQuality", "responsiveAndTheme", "loadingEmptyErrorSuccess", "dailyUsefulness", "complexityDiscipline", "accessibility", "actionSafety", ] as const; const makeCategories = (score: number) => Object.fromEntries(categoryNames.map((name) => [name, score])) as CustomWidgetJudgeResult["categories"]; const makeReasons = () => Object.fromEntries( categoryNames.map((name) => [name, `Concrete evidence for ${name}.`]), ) as CustomWidgetJudgeResult["categoryReasons"]; const makeJudgeResult = (score: number) => ({ total: score, verdict: "pass" as const, dailyUseDecision: "would-use-daily" as const, categories: makeCategories(score), categoryReasons: makeReasons(), strengths: [], problems: [], fatalProblems: [], highestImpactFixes: [], }); const judgePanelWidget = customWidgetDefinitionSchema.parse({ $schema: "homarr-custom-widget-v2", name: "Panel fixture", sources: { default: { baseUrl: "https://status.example.test", networkScope: "public", auth: "none", }, }, requests: {}, options: {}, template: "Panel fixture", }); const makePanelCalls = (scores: readonly number[]): CustomWidgetJudgePanelCall[] => scores.map((score, index) => ({ call: index + 1, raw: JSON.stringify(makeJudgeResult(score)), result: makeJudgeResult(score), error: null, })); describe("AI authoring evaluation", () => { it("allows bounded output reservations for low-credit live judges without changing defaults", () => { expect(getAiEvaluationMaxOutputTokens("judge", undefined)).toBe(32_768); expect(getAiEvaluationMaxOutputTokens("judge", "3000")).toBe(3_000); expect(getAiEvaluationMaxOutputTokens("generation", "1000")).toBe(4_096); expect(getAiEvaluationMaxOutputTokens("generation", "invalid")).toBe(32_768); }); it("defines distinct complex and public-API scenarios", () => { expect(CUSTOM_WIDGET_AI_EVALUATION_CASES).toHaveLength(18); expect(new Set(CUSTOM_WIDGET_AI_EVALUATION_CASES.map((entry) => entry.id)).size).toBe(18); expect(CUSTOM_WIDGET_AI_EVALUATION_CASES.map(({ id }) => id)).toEqual( expect.arrayContaining([ "pokedex", "fake-service-health", "coinmarketcap-keyless", "bored-activity", "agify-name", "seerr-media-workflows", "seed-dispatcharr-channels", "seed-karakeep-bookmarks", "seed-mealie-today", "seed-romm-library", "seed-tubearchivist-queue", "seed-frigate-alerts", "seed-frigate-system", "seed-frigate-live-streams", ]), ); const advancedCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === "seerr-media-workflows"); expect(advancedCase?.expectedWidgets?.every(({ expectations }) => !("maximumRequests" in expectations))).toBe(true); expect(advancedCase?.research).toMatchObject({ requiredReferences: ["schema", "runtime", "security"], allowedReferences: ["schema", "runtime", "security"], }); expect(advancedCase?.request).toContain("voteAverage"); expect(advancedCase?.request).toContain("functional pagination"); expect(advancedCase?.request).toContain("responsive SimpleGrid"); expect(advancedCase?.request).toContain("createdAt"); expect(advancedCase?.request).toContain("prefers backdropPath"); expect(advancedCase?.request).toContain("approve and decline"); expect(advancedCase?.request).toContain("TMDB 603 · Request #91"); expect(advancedCase?.request).toContain("Rating label only when voteAverage is present"); expect(advancedCase?.request).toContain("without redundant outer or per-item chrome"); expect(advancedCase?.request).toContain("relies on native loading/error/retry"); expect(advancedCase?.request).toContain("query/page/total-results summary"); expect(advancedCase?.request).toContain("Not provided"); expect(advancedCase?.request).toContain("Media profile"); expect(advancedCase?.request).toContain("when only its sibling request fails"); expect(advancedCase?.request).toContain("compact thumbnail beside content at base"); expect(advancedCase?.request).toContain("Avatar imageProps.alt"); expect(advancedCase?.request).toContain("Pagination only when totalPages is greater than 1"); expect(advancedCase?.apiNotes).toContain("render the array at response.results"); expect(advancedCase?.apiNotes).toContain("/request/{param:requestId}/approve"); const operationsExpectations = advancedCase?.expectedWidgets?.find( ({ id }) => id === "request-operations", )?.expectations; expect(operationsExpectations?.requests).toEqual( expect.arrayContaining([ expect.objectContaining({ kind: "query", requiresStatusBinding: true }), expect.objectContaining({ kind: "action", pathIncludes: "/request/{param:requestId}/approve" }), expect.objectContaining({ kind: "action", pathIncludes: "/request/{param:requestId}/decline" }), ]), ); expect(operationsExpectations?.templateIncludes).toEqual( expect.arrayContaining([ "processing", "declined", "tmdbId", "status4k", "Failed", "Completed", "Not provided", "imageProps", ]), ); expect(operationsExpectations?.templateIncludesAny).toEqual( expect.arrayContaining([ ["TMDB", "Tmdb"], ["Request #", "Request ID"], ["UTC", "utc"], ]), ); const mediaExpectations = advancedCase?.expectedWidgets?.find(({ id }) => id === "media-research")?.expectations; expect(mediaExpectations?.templateIncludes).toEqual( expect.arrayContaining([ ".results", "backdropPath", "https://image.tmdb.org/t/p/", "w780", "inputs.page ?? 1", "status4k", 'trigger="manual"', "lineClamp", ]), ); expect(mediaExpectations?.templateIncludesAny).not.toContainEqual(["SimpleGrid", "Grid"]); expect(mediaExpectations?.templateIncludesAny).toContainEqual(["Rating", "★", "/10"]); expect(mediaExpectations?.templateIncludesAny).toContainEqual(["Partially Available", "Partially available"]); }); it("accepts every shipped preset against its benchmark contract", () => { for (const testCase of CUSTOM_WIDGET_AI_EVALUATION_CASES.filter((entry) => entry.preferredExampleId)) { const example = getCustomWidgetExample(testCase.preferredExampleId ?? ""); if (!example) throw new Error(`Bundled example '${testCase.preferredExampleId}' was not found`); expect( getDeterministicEvaluationIssues(testCase, customWidgetDefinitionSchema.parse(example.widget)), testCase.id, ).toEqual([]); } }); it("scopes multi-widget judging to each requested capability and its exact fixtures", () => { const testCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === "seerr-media-workflows"); const expectedWidgets = testCase?.expectedWidgets; if (!testCase || !expectedWidgets) throw new Error("Advanced Seerr expectations are missing"); const expectedOperations = expectedWidgets.find(({ id }) => id === "request-operations"); const expectedResearch = expectedWidgets.find(({ id }) => id === "media-research"); if (!expectedOperations || !expectedResearch) throw new Error("Scoped Seerr expectations are missing"); const operations = getExpectedWidgetCase(testCase, expectedOperations); const research = getExpectedWidgetCase(testCase, expectedResearch); expect(operations.apiNotes).not.toContain("GET /search"); expect(operations.previewResponses?.map(({ pathIncludes }) => pathIncludes)).toEqual([ "/request/count", "/request", "/request/{param:requestId}/approve", "/request/{param:requestId}/decline", ]); expect(research.apiNotes).not.toContain("GET /request/count"); expect(research.previewResponses?.map(({ pathIncludes }) => pathIncludes)).toEqual(["/search", "/request"]); }); it("grounds fixture-backed scenarios in verified routes and authentication", () => { const testCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === "agify-name"); if (!testCase) throw new Error("Agify evaluation case is missing"); const validWidget = { $schema: "homarr-custom-widget-v2" as const, name: "Agify", sources: { default: { baseUrl: "https://api.agify.io", networkScope: "public" as const, auth: { type: "apiKeyQuery" as const, name: "apikey" }, }, }, requests: { prediction: { source: "default", kind: "query" as const, method: "GET" as const, path: "/", trigger: "manual" as const, query: { name: { $param: "name" }, country_id: { $param: "country" } }, auth: "inherit" as const, permission: "view" as const, }, }, options: {}, template: '{(result) => {result.age} from {result.count}}', }; expect(getDeterministicEvaluationIssues(testCase, validWidget)).toEqual([]); expect( getDeterministicEvaluationIssues(testCase, { ...validWidget, sources: { default: { ...validWidget.sources.default, auth: "none" as const } }, }), ).toContainEqual(expect.objectContaining({ path: ["sources", "default", "auth"] })); }); it("checks the public API response contract instead of only the endpoint", () => { const testCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === "coinmarketcap-keyless"); if (!testCase) throw new Error("CoinMarketCap evaluation case is missing"); const widget = { $schema: "homarr-custom-widget-v2" as const, name: "Crypto", sources: { default: { baseUrl: "https://pro-api.coinmarketcap.com", networkScope: "public" as const, auth: "none" as const, }, }, requests: { prices: { source: "default", kind: "query" as const, method: "GET" as const, path: "/public-api/v3/cryptocurrency/quotes/latest", trigger: "load" as const, query: { id: "1,1027,5426", convert: "USD" }, auth: "inherit" as const, permission: "view" as const, }, }, options: {}, template: "{data.prices?.data?.map(item => {item.quote?.[0]?.percent_change_24h} {item.quote?.[0]?.market_cap} {item.quote?.[0]?.volume_24h} {item.last_updated})}", }; expect(getDeterministicEvaluationIssues(testCase, widget)).toEqual([]); expect( getDeterministicEvaluationIssues(testCase, { ...widget, requests: { prices: { ...widget.requests.prices, query: { id: "1,1027", convert: "USD" } } }, }), ).toContainEqual(expect.objectContaining({ path: ["requests"] })); expect( getDeterministicEvaluationIssues(testCase, { ...widget, requests: { prices: { ...widget.requests.prices, query: { id: { $option: "symbols" }, convert: "USD" }, }, }, options: { symbols: { label: "Symbols", control: "text", default: "1,1027,5426" }, }, }), ).toEqual([]); }); it("accepts equivalent empty-state wording from an allowed semantic group", () => { const baseCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === "fake-service-health"); if (!baseCase?.expectations) throw new Error("Fake service-health expectations are missing"); const testCase = { ...baseCase, expectations: { ...baseCase.expectations, templateIncludesAny: [["No matching services", "No services found", "Nothing to show"]], }, }; const widget = { $schema: "homarr-custom-widget-v2" as const, name: "Health", sources: { default: { baseUrl: "https://status.example.test", networkScope: "public" as const, auth: "none" as const }, }, requests: { health: { source: "default", kind: "query" as const, method: "GET" as const, path: "/v1/health", trigger: "load" as const, auth: "inherit" as const, permission: "view" as const, }, }, options: {}, template: "status openIncidents latencyMs checkedAt servicesNo services found", }; expect(getDeterministicEvaluationIssues(testCase, widget)).toEqual([]); expect( getDeterministicEvaluationIssues(testCase, { ...widget, template: widget.template.replace("No services found", "All services loaded"), }), ).toContainEqual(expect.objectContaining({ path: ["template"] })); }); it("rejects a wrapped response rendered as an array and unwired pagination", () => { const testCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === "seerr-media-workflows"); if (!testCase?.expectedWidgets) throw new Error("Advanced Seerr expectations are missing"); const mediaCase = { ...testCase, expectations: testCase.expectedWidgets.find(({ id }) => id === "media-research")?.expectations, expectedWidgets: undefined, }; const operationsCase = { ...testCase, expectations: testCase.expectedWidgets.find(({ id }) => id === "request-operations")?.expectations, expectedWidgets: undefined, }; const validMedia = { $schema: "homarr-custom-widget-v2" as const, name: "Seerr research", sources: { default: { baseUrl: "http://seerr.local:5055/api/v1", networkScope: "private" as const, auth: { type: "apiKeyHeader" as const, name: "X-Api-Key" }, }, }, requests: { search: { source: "default", kind: "query" as const, method: "GET" as const, path: "/search", trigger: "manual" as const, query: { query: { $param: "query" }, page: { $param: "page" } }, auth: "inherit" as const, permission: "view" as const, }, requestMovie: { source: "default", kind: "action" as const, method: "POST" as const, path: "/request", trigger: "manual" as const, body: { mediaType: "movie", mediaId: { $param: "id" } }, auth: "inherit" as const, permission: "modify" as const, confirmation: "Request movie?", invalidates: ["search"], }, requestSeries: { source: "default", kind: "action" as const, method: "POST" as const, path: "/request", trigger: "manual" as const, body: { mediaType: "tv", mediaId: { $param: "id" }, seasons: "all" }, auth: "inherit" as const, permission: "modify" as const, confirmation: "Request series?", invalidates: ["search"], }, }, options: {}, template: '{(response) => {response.totalResults}{response.results.map(item => {item.title}{item.overview})}}overview voteAverage ★ mediaInfo status4k Unknown Pending Processing Partially available Available Blocklisted Deleted No results'.repeat( 4, ), }; const brokenMedia = { ...validMedia, template: validMedia.template.replaceAll("response.results", "response"), }; expect(getDeterministicEvaluationIssues(mediaCase, validMedia)).toEqual([]); const missingSearchInvalidation = { ...validMedia, requests: { ...validMedia.requests, requestMovie: { ...validMedia.requests.requestMovie, invalidates: [] }, }, }; expect(getDeterministicEvaluationIssues(mediaCase, missingSearchInvalidation)).toContainEqual( expect.objectContaining({ path: ["requests"], message: expect.stringMatching(/invalidates.*\/search/u), }), ); const actionIssue = getDeterministicEvaluationIssues(mediaCase, missingSearchInvalidation).find( (issue) => issue.path?.[0] === "requests", ); expect(actionIssue?.message).toContain('mediaType="movie" or any $param binding'); expect(actionIssue?.message).not.toContain('["movie","$param:*"]'); const boundActionValues = { ...validMedia, requests: { ...validMedia.requests, requestMovie: { ...validMedia.requests.requestMovie, body: { mediaType: { $param: "mediaType" }, mediaId: { $param: "id" } }, }, requestSeries: { ...validMedia.requests.requestSeries, body: { mediaType: { $param: "mediaType" }, mediaId: { $param: "id" }, seasons: { $param: "seasons" }, }, }, }, }; expect(getDeterministicEvaluationIssues(mediaCase, boundActionValues)).toEqual([]); expect( getDeterministicEvaluationIssues(mediaCase, { ...boundActionValues, requests: { ...boundActionValues.requests, requestSeries: { ...boundActionValues.requests.requestSeries, body: { mediaType: "tv" } }, }, }), ).toContainEqual(expect.objectContaining({ path: ["requests"] })); expect(getDeterministicEvaluationIssues(mediaCase, brokenMedia)).toContainEqual( expect.objectContaining({ message: expect.stringContaining("'.results'") }), ); const operationsWidget = { $schema: "homarr-custom-widget-v2" as const, name: "Seerr operations", sources: validMedia.sources, requests: { counts: { source: "default", kind: "query" as const, method: "GET" as const, path: "/request/count", trigger: "load" as const, auth: "inherit" as const, permission: "view" as const, }, recent: { source: "default", kind: "query" as const, method: "GET" as const, path: "/request", trigger: "load" as const, query: { take: 10, skip: 0, sort: "added", sortDirection: "desc" }, auth: "inherit" as const, permission: "view" as const, }, approvePending: { source: "default", kind: "action" as const, method: "POST" as const, path: "/request/{param:requestId}/approve", trigger: "manual" as const, auth: "inherit" as const, permission: "modify" as const, confirmation: "Approve request?", invalidates: ["counts", "recent"], }, declinePending: { source: "default", kind: "action" as const, method: "POST" as const, path: "/request/{param:requestId}/decline", trigger: "manual" as const, auth: "inherit" as const, permission: "modify" as const, confirmation: "Decline request?", invalidates: ["counts", "recent"], }, }, options: {}, template: 'pending approved available processing declined status.{status.counts.loading} {status.recent.error} {data.recent.results} {data.recent.pageInfo} {data.recent.createdAt} UTC TMDB {data.recent.tmdbId} {data.recent.mediaType} {data.recent.status4k} Request #'.repeat( 8, ), }; const operationIssues = getDeterministicEvaluationIssues(operationsCase, operationsWidget); expect(operationIssues.filter((issue) => issue.path?.[0] === "requests")).toEqual([]); expect(operationIssues).toContainEqual(expect.objectContaining({ message: expect.stringContaining("Pagination") })); expect( getDeterministicEvaluationIssues(operationsCase, { ...operationsWidget, template: operationsWidget.template.replaceAll("status.counts", "status").replaceAll("status.recent", "status"), }), ).toContainEqual(expect.objectContaining({ message: expect.stringContaining("no global status.loading") })); }); it("includes every path-specific Pokédex fixture in generator and judge context", () => { const testCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === "pokedex"); if (!testCase) throw new Error("Pokédex evaluation case is missing"); const fixtureText = getEvaluationResponseFixtureText(testCase); expect(fixtureText).toContain("/api/v2/pokemon/{param:name}"); expect(fixtureText).toContain('"official-artwork"'); expect(fixtureText).toContain('"results"'); expect(testCase.minimumPreviewCycles).toBe(3); }); it("uses the exact clipboard prompt and grounded API notes", () => { const testCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.at(0); if (!testCase) throw new Error("Expected at least one AI evaluation case"); const prompt = buildEvaluationPrompt(testCase); expect(prompt.length).toBeLessThanOrEqual(12_000); expect(prompt).toContain("Verified API notes"); expect(prompt).not.toContain("customWidget_validate"); expect(prompt).not.toContain("homarr://"); }); it("keeps validation failures concrete during repair", () => { const prompt = buildRepairPrompt("original", "bad response", [ { path: ["requests", "list", "path"], message: "Required" }, ]); expect(prompt).toContain("requests.list.path: Required"); expect(prompt).toContain("bad response"); }); it("uses the requested DeepSeek models and strict structured judge output", () => { expect(DEFAULT_GENERATOR_MODEL).toBe("deepseek/deepseek-v4.1-flash"); expect(DEFAULT_JUDGE_MODEL).toBe("openai/gpt-5.6-luna"); expect(getAiEvaluationReasoningOptions(DEFAULT_GENERATOR_MODEL)).toEqual({ effort: "xhigh", exclude: true }); expect(getAiEvaluationReasoningOptions(DEFAULT_JUDGE_MODEL)).toEqual({ effort: "high", exclude: true }); expect(getAiEvaluationProviderPreferences(DEFAULT_GENERATOR_MODEL)).toEqual({ order: ["deepinfra/fp8"], quantizations: ["fp8"], allow_fallbacks: true, }); expect(getAiEvaluationProviderPreferences(DEFAULT_JUDGE_MODEL)).toBeUndefined(); const format = getJudgeResponseFormat(); expect(format.type).toBe("json_schema"); expect(format.json_schema.strict).toBe(true); expect(format.json_schema.schema).toMatchObject({ type: "object", additionalProperties: false }); }); it("can run the same evaluation against the Homarr provider endpoint", () => { expect(getAiProviderChatCompletionsUrl("https://homarr.dev/api/ai/v1/")).toBe( "https://homarr.dev/api/ai/v1/chat/completions", ); expect(getAiProviderChatCompletionsUrl()).toBe("https://openrouter.ai/api/v1/chat/completions"); expect( resolveAiEvaluationProviderConfig({ AI_PROVIDER_BASE_URL: "https://homarr.dev/api/ai/v1/", AI_PROVIDER_API_KEY: "workshop-token", }), ).toEqual({ apiKey: "workshop-token", baseUrl: "https://homarr.dev/api/ai/v1", generatorModel: "homarr/model", judgeModel: "homarr/model", judgeBaseUrl: "https://homarr.dev/api/ai/v1", judgeApiKey: "workshop-token", }); expect( resolveAiEvaluationProviderConfig({ AI_PROVIDER_BASE_URL: " ", OPENROUTER_API_KEY: "legacy-key", OPENROUTER_GENERATOR_MODEL: "legacy-generator", OPENROUTER_JUDGE_MODEL: "legacy-judge", }), ).toEqual({ apiKey: "legacy-key", baseUrl: DEFAULT_AI_PROVIDER_BASE_URL, generatorModel: "legacy-generator", judgeModel: "legacy-judge", judgeBaseUrl: DEFAULT_AI_PROVIDER_BASE_URL, judgeApiKey: "legacy-key", }); expect( resolveAiEvaluationProviderConfig({ AI_PROVIDER_BASE_URL: "https://homarr.dev/api/ai/v1", AI_PROVIDER_API_KEY: "workshop-token", OPENROUTER_API_KEY: "judge-key", OPENROUTER_GENERATOR_MODEL: "must-not-apply", OPENROUTER_JUDGE_MODEL: "independent-judge", }), ).toEqual({ apiKey: "workshop-token", baseUrl: "https://homarr.dev/api/ai/v1", generatorModel: "homarr/model", judgeModel: "independent-judge", judgeBaseUrl: DEFAULT_AI_PROVIDER_BASE_URL, judgeApiKey: "judge-key", }); }); it("requires an excellent result without weak categories", () => { expect( judgePasses({ ...makeJudgeResult(85), categories: { ...makeCategories(85), visualQuality: 76, dailyUsefulness: 76, complexityDiscipline: 80, }, }), ).toBe(true); expect(judgePasses(makeJudgeResult(84))).toBe(false); expect( judgePasses({ ...makeJudgeResult(90), categories: { ...makeCategories(90), accessibility: 74 }, }), ).toBe(false); expect( judgePasses({ ...makeJudgeResult(92), categories: { ...makeCategories(92), goalFulfillment: 84 }, }), ).toBe(false); expect( judgePasses({ ...makeJudgeResult(92), categories: { ...makeCategories(92), complexityDiscipline: 79 }, }), ).toBe(false); expect(judgePasses({ ...makeJudgeResult(92), dailyUseDecision: "promising-but-not-daily" })).toBe(false); expect(judgePasses({ ...makeJudgeResult(95), fatalProblems: ["A requested core action is missing."] })).toBe(false); }); it("uses a unanimous three-vote panel and escalates split votes to five", () => { const unanimous = summarizeCustomWidgetJudgePanel(judgePanelWidget, makePanelCalls([87, 89, 86])); expect(unanimous).toMatchObject({ status: "pass", passVotes: 3, failVotes: 0, medianTotal: 87 }); expect(unanimous.scoreRange).toEqual({ min: 86, max: 89 }); expect(unanimous.artifactHash).toMatch(/^[a-f0-9]{64}$/u); const split = summarizeCustomWidgetJudgePanel(judgePanelWidget, makePanelCalls([87, 83, 86])); expect(split.status).toBe("inconclusive"); const resolvedPass = summarizeCustomWidgetJudgePanel(judgePanelWidget, makePanelCalls([87, 83, 86, 88, 82])); expect(resolvedPass).toMatchObject({ status: "pass", passVotes: 3, failVotes: 2, medianTotal: 86 }); const resolvedFail = summarizeCustomWidgetJudgePanel(judgePanelWidget, makePanelCalls([87, 83, 82, 88, 81])); expect(resolvedFail).toMatchObject({ status: "fail", passVotes: 2, failVotes: 3, medianTotal: 83 }); if (!resolvedFail.representative) throw new Error("Expected a representative judge result"); expect(judgePasses(resolvedFail.representative)).toBe(false); }); it("marks an incomplete judge panel inconclusive", () => { const calls = makePanelCalls([90, 89]); calls.push({ call: 3, raw: null, result: null, error: "empty provider response" }); expect(summarizeCustomWidgetJudgePanel(judgePanelWidget, calls)).toMatchObject({ status: "inconclusive", passVotes: 2, failVotes: 0, medianTotal: 90, }); }); it("gives the harsh judge the authoritative request-state runtime contract", () => { const testCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === "fake-service-health"); if (!testCase) throw new Error("Fake service-health case is missing"); const widget = { $schema: "homarr-custom-widget-v2" as const, name: "Health", sources: { default: { baseUrl: "https://status.example.test", networkScope: "public" as const, auth: "none" as const }, }, requests: {}, options: {}, template: "Health", }; const prompt = buildJudgePrompt(testCase, widget); expect(prompt).toContain("status. with loading/ok/status/error fields"); expect(prompt).toContain("RefreshButton is an installed runtime helper"); expect(prompt).toContain("cannot display stale results under edited inputs"); expect(prompt).toContain("do not penalize a repeated short literal label array"); expect(prompt).toContain(JSON.stringify(testCase.sampleResponse, null, 2)); expect(prompt).toContain("# Runtime"); expect(prompt).not.toContain("# Bundled file:"); expect(prompt).toContain("does not prove API correctness, visual quality, usefulness, or accessibility"); expect(prompt).toContain( "verified API notes and representative response fixtures are authoritative for endpoint paths, authentication requirements, and response shapes", ); expect(prompt).toContain("Do not invent external endpoint or authentication objections from outside assumptions"); expect(prompt).toContain("an endpoint mentioned in broader API notes is available, not automatically required"); expect(prompt).toContain( "Decorative icons paired with equivalent adjacent visible status text need no separate aria-label", ); expect(prompt).toContain("A Badge containing explicit visible status text is not color-only"); expect(prompt).toContain('Date.toLocaleString(value, "en-US", "UTC") is an installed safe static helper'); expect(prompt).toContain( 'A pass requires a weighted total of at least 85, every category at least 75, goalFulfillment at least 85, complexityDiscipline at least 80, no fatal problem, and dailyUseDecision="would-use-daily"', ); }); it("computes the weighted score and refuses inflated advisory verdicts", () => { const parsed = parseJudgeResult( JSON.stringify({ ...makeJudgeResult(95), total: 99, categories: { ...makeCategories(95), visualQuality: 70 }, categoryReasons: makeReasons(), strengths: [], problems: [], fatalProblems: [], highestImpactFixes: [], }), ); expect(parsed.total).toBe(91); expect(parsed.verdict).toBe("fail"); }); it("accepts a provider response that wraps structured review JSON in a code fence", () => { const parsed = parseJudgeResult(`\`\`\`json\n${JSON.stringify(makeJudgeResult(90))}\n\`\`\``); expect(parsed.total).toBe(90); expect(parsed.verdict).toBe("pass"); }); it("normalizes a prose 'none' entry out of an otherwise empty fatal-problem list", () => { const parsed = parseJudgeResult( JSON.stringify({ ...makeJudgeResult(90), fatalProblems: ["None: all requested capabilities are present.", "No fatal issues."], }), ); expect(parsed.fatalProblems).toEqual([]); expect(parsed.verdict).toBe("pass"); }); });