Files
homarr/packages/custom-widgets/src/test/ai-evaluation.spec.ts
T
+12 fcd7c979df feat!: release v2 (#6545)
Co-authored-by: Artur <145776561+arturict@users.noreply.github.com>
Co-authored-by: arturict <arturict@users.noreply.github.com>
Co-authored-by: Thomas <thomas@ajnart.fr>
Co-authored-by: Afonso Jorge Ramos <afonsojorgeramos@gmail.com>
Co-authored-by: Danura30082 <danura30082+github@gmail.com>
Co-authored-by: Kushie <122986913+kushiemoon-dev@users.noreply.github.com>
Co-authored-by: ajnart <thomas@ajnart.fr>
Co-authored-by: ajnart <thomas@ajnart.dev>
Co-authored-by: chenjun <chenjun@foxmail.com>
Co-authored-by: Wikibear <wikibear@users.noreply.github.com>
Co-authored-by: Bergasha <bergasha@gmail.com>
Co-authored-by: leafyelin <43306196+leafyelin@users.noreply.github.com>
Co-authored-by: Lucas Araujo <lucas.lra@gmail.com>
Co-authored-by: GreenMonito <86085570+GreenMonito@users.noreply.github.com>
Co-authored-by: Nerdy-Technician <roffo@nerdy-technician.social>
Co-authored-by: Jun <34545896+jcstrong@users.noreply.github.com>
Co-authored-by: Edmond Abraham <54119952+gitedmond@users.noreply.github.com>
Co-authored-by: thrawnx <tight.bulb9384@fastmail.com>
Co-authored-by: Chris <58918656+nordada@users.noreply.github.com>
Co-authored-by: yi111 <153097222+Yi-111-a@users.noreply.github.com>
Co-authored-by: Yanis <naamaneyanis@gmail.com>
Co-authored-by: Wikibear <me@sej.de>
Co-authored-by: alessiottavo <93441886+alessiottavo@users.noreply.github.com>
2026-10-02 21:00:14 +02:00

767 lines
32 KiB
TypeScript

import { describe, expect, it } from "vitest";
import { CUSTOM_WIDGET_AI_EVALUATION_CASES } from "../../scripts/ai-evaluation-cases";
import {
buildEvaluationPrompt,
buildJudgePrompt,
buildRepairPrompt,
DEFAULT_AI_PROVIDER_BASE_URL,
DEFAULT_GENERATOR_MODEL,
DEFAULT_JUDGE_MODEL,
getAiProviderChatCompletionsUrl,
getAiEvaluationProviderPreferences,
getAiEvaluationReasoningOptions,
getDeterministicEvaluationIssues,
getEvaluationResponseFixtureText,
getExpectedWidgetCase,
getAiEvaluationMaxOutputTokens,
getJudgeResponseFormat,
judgePasses,
parseJudgeResult,
resolveAiEvaluationProviderConfig,
summarizeCustomWidgetJudgePanel,
} from "../../scripts/ai-evaluation";
import type { CustomWidgetJudgePanelCall, CustomWidgetJudgeResult } from "../../scripts/ai-evaluation";
import { getCustomWidgetExample } from "../core/authoring-resources";
import { customWidgetDefinitionSchema } from "../core/custom-jsx-schema";
const categoryNames = [
"schemaAndBindings",
"apiAndRequestDesign",
"runtimeCompatibility",
"goalFulfillment",
"visualQuality",
"responsiveAndTheme",
"loadingEmptyErrorSuccess",
"dailyUsefulness",
"complexityDiscipline",
"accessibility",
"actionSafety",
] as const;
const makeCategories = (score: number) =>
Object.fromEntries(categoryNames.map((name) => [name, score])) as CustomWidgetJudgeResult["categories"];
const makeReasons = () =>
Object.fromEntries(
categoryNames.map((name) => [name, `Concrete evidence for ${name}.`]),
) as CustomWidgetJudgeResult["categoryReasons"];
const makeJudgeResult = (score: number) => ({
total: score,
verdict: "pass" as const,
dailyUseDecision: "would-use-daily" as const,
categories: makeCategories(score),
categoryReasons: makeReasons(),
strengths: [],
problems: [],
fatalProblems: [],
highestImpactFixes: [],
});
const judgePanelWidget = customWidgetDefinitionSchema.parse({
$schema: "homarr-custom-widget-v2",
name: "Panel fixture",
sources: {
default: {
baseUrl: "https://status.example.test",
networkScope: "public",
auth: "none",
},
},
requests: {},
options: {},
template: "<Text>Panel fixture</Text>",
});
const makePanelCalls = (scores: readonly number[]): CustomWidgetJudgePanelCall[] =>
scores.map((score, index) => ({
call: index + 1,
raw: JSON.stringify(makeJudgeResult(score)),
result: makeJudgeResult(score),
error: null,
}));
describe("AI authoring evaluation", () => {
it("allows bounded output reservations for low-credit live judges without changing defaults", () => {
expect(getAiEvaluationMaxOutputTokens("judge", undefined)).toBe(32_768);
expect(getAiEvaluationMaxOutputTokens("judge", "3000")).toBe(3_000);
expect(getAiEvaluationMaxOutputTokens("generation", "1000")).toBe(4_096);
expect(getAiEvaluationMaxOutputTokens("generation", "invalid")).toBe(32_768);
});
it("defines distinct complex and public-API scenarios", () => {
expect(CUSTOM_WIDGET_AI_EVALUATION_CASES).toHaveLength(18);
expect(new Set(CUSTOM_WIDGET_AI_EVALUATION_CASES.map((entry) => entry.id)).size).toBe(18);
expect(CUSTOM_WIDGET_AI_EVALUATION_CASES.map(({ id }) => id)).toEqual(
expect.arrayContaining([
"pokedex",
"fake-service-health",
"coinmarketcap-keyless",
"bored-activity",
"agify-name",
"seerr-media-workflows",
"seed-dispatcharr-channels",
"seed-karakeep-bookmarks",
"seed-mealie-today",
"seed-romm-library",
"seed-tubearchivist-queue",
"seed-frigate-alerts",
"seed-frigate-system",
"seed-frigate-live-streams",
]),
);
const advancedCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === "seerr-media-workflows");
expect(advancedCase?.expectedWidgets?.every(({ expectations }) => !("maximumRequests" in expectations))).toBe(true);
expect(advancedCase?.research).toMatchObject({
requiredReferences: ["schema", "runtime", "security"],
allowedReferences: ["schema", "runtime", "security"],
});
expect(advancedCase?.request).toContain("voteAverage");
expect(advancedCase?.request).toContain("functional pagination");
expect(advancedCase?.request).toContain("responsive SimpleGrid");
expect(advancedCase?.request).toContain("createdAt");
expect(advancedCase?.request).toContain("prefers backdropPath");
expect(advancedCase?.request).toContain("approve and decline");
expect(advancedCase?.request).toContain("TMDB 603 · Request #91");
expect(advancedCase?.request).toContain("Rating label only when voteAverage is present");
expect(advancedCase?.request).toContain("without redundant outer or per-item chrome");
expect(advancedCase?.request).toContain("relies on native loading/error/retry");
expect(advancedCase?.request).toContain("query/page/total-results summary");
expect(advancedCase?.request).toContain("Not provided");
expect(advancedCase?.request).toContain("Media profile");
expect(advancedCase?.request).toContain("when only its sibling request fails");
expect(advancedCase?.request).toContain("compact thumbnail beside content at base");
expect(advancedCase?.request).toContain("Avatar imageProps.alt");
expect(advancedCase?.request).toContain("Pagination only when totalPages is greater than 1");
expect(advancedCase?.apiNotes).toContain("render the array at response.results");
expect(advancedCase?.apiNotes).toContain("/request/{param:requestId}/approve");
const operationsExpectations = advancedCase?.expectedWidgets?.find(
({ id }) => id === "request-operations",
)?.expectations;
expect(operationsExpectations?.requests).toEqual(
expect.arrayContaining([
expect.objectContaining({ kind: "query", requiresStatusBinding: true }),
expect.objectContaining({ kind: "action", pathIncludes: "/request/{param:requestId}/approve" }),
expect.objectContaining({ kind: "action", pathIncludes: "/request/{param:requestId}/decline" }),
]),
);
expect(operationsExpectations?.templateIncludes).toEqual(
expect.arrayContaining([
"processing",
"declined",
"tmdbId",
"status4k",
"Failed",
"Completed",
"Not provided",
"imageProps",
]),
);
expect(operationsExpectations?.templateIncludesAny).toEqual(
expect.arrayContaining([
["TMDB", "Tmdb"],
["Request #", "Request ID"],
["UTC", "utc"],
]),
);
const mediaExpectations = advancedCase?.expectedWidgets?.find(({ id }) => id === "media-research")?.expectations;
expect(mediaExpectations?.templateIncludes).toEqual(
expect.arrayContaining([
".results",
"backdropPath",
"https://image.tmdb.org/t/p/",
"w780",
"inputs.page ?? 1",
"status4k",
'trigger="manual"',
"lineClamp",
]),
);
expect(mediaExpectations?.templateIncludesAny).not.toContainEqual(["SimpleGrid", "Grid"]);
expect(mediaExpectations?.templateIncludesAny).toContainEqual(["Rating", "★", "/10"]);
expect(mediaExpectations?.templateIncludesAny).toContainEqual(["Partially Available", "Partially available"]);
});
it("accepts every shipped preset against its benchmark contract", () => {
for (const testCase of CUSTOM_WIDGET_AI_EVALUATION_CASES.filter((entry) => entry.preferredExampleId)) {
const example = getCustomWidgetExample(testCase.preferredExampleId ?? "");
if (!example) throw new Error(`Bundled example '${testCase.preferredExampleId}' was not found`);
expect(
getDeterministicEvaluationIssues(testCase, customWidgetDefinitionSchema.parse(example.widget)),
testCase.id,
).toEqual([]);
}
});
it("scopes multi-widget judging to each requested capability and its exact fixtures", () => {
const testCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === "seerr-media-workflows");
const expectedWidgets = testCase?.expectedWidgets;
if (!testCase || !expectedWidgets) throw new Error("Advanced Seerr expectations are missing");
const expectedOperations = expectedWidgets.find(({ id }) => id === "request-operations");
const expectedResearch = expectedWidgets.find(({ id }) => id === "media-research");
if (!expectedOperations || !expectedResearch) throw new Error("Scoped Seerr expectations are missing");
const operations = getExpectedWidgetCase(testCase, expectedOperations);
const research = getExpectedWidgetCase(testCase, expectedResearch);
expect(operations.apiNotes).not.toContain("GET /search");
expect(operations.previewResponses?.map(({ pathIncludes }) => pathIncludes)).toEqual([
"/request/count",
"/request",
"/request/{param:requestId}/approve",
"/request/{param:requestId}/decline",
]);
expect(research.apiNotes).not.toContain("GET /request/count");
expect(research.previewResponses?.map(({ pathIncludes }) => pathIncludes)).toEqual(["/search", "/request"]);
});
it("grounds fixture-backed scenarios in verified routes and authentication", () => {
const testCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === "agify-name");
if (!testCase) throw new Error("Agify evaluation case is missing");
const validWidget = {
$schema: "homarr-custom-widget-v2" as const,
name: "Agify",
sources: {
default: {
baseUrl: "https://api.agify.io",
networkScope: "public" as const,
auth: { type: "apiKeyQuery" as const, name: "apikey" },
},
},
requests: {
prediction: {
source: "default",
kind: "query" as const,
method: "GET" as const,
path: "/",
trigger: "manual" as const,
query: { name: { $param: "name" }, country_id: { $param: "country" } },
auth: "inherit" as const,
permission: "view" as const,
},
},
options: {},
template:
'<Stack><TextInput bind="name" /><SubFetch requestId="prediction" params={{ name: inputs.name, country: "US" }}>{(result) => <Text>{result.age} from {result.count}</Text>}</SubFetch></Stack>',
};
expect(getDeterministicEvaluationIssues(testCase, validWidget)).toEqual([]);
expect(
getDeterministicEvaluationIssues(testCase, {
...validWidget,
sources: { default: { ...validWidget.sources.default, auth: "none" as const } },
}),
).toContainEqual(expect.objectContaining({ path: ["sources", "default", "auth"] }));
});
it("checks the public API response contract instead of only the endpoint", () => {
const testCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === "coinmarketcap-keyless");
if (!testCase) throw new Error("CoinMarketCap evaluation case is missing");
const widget = {
$schema: "homarr-custom-widget-v2" as const,
name: "Crypto",
sources: {
default: {
baseUrl: "https://pro-api.coinmarketcap.com",
networkScope: "public" as const,
auth: "none" as const,
},
},
requests: {
prices: {
source: "default",
kind: "query" as const,
method: "GET" as const,
path: "/public-api/v3/cryptocurrency/quotes/latest",
trigger: "load" as const,
query: { id: "1,1027,5426", convert: "USD" },
auth: "inherit" as const,
permission: "view" as const,
},
},
options: {},
template:
"<Stack><RefreshButton />{data.prices?.data?.map(item => <Text key={item.symbol}>{item.quote?.[0]?.percent_change_24h} {item.quote?.[0]?.market_cap} {item.quote?.[0]?.volume_24h} {item.last_updated}</Text>)}</Stack>",
};
expect(getDeterministicEvaluationIssues(testCase, widget)).toEqual([]);
expect(
getDeterministicEvaluationIssues(testCase, {
...widget,
requests: { prices: { ...widget.requests.prices, query: { id: "1,1027", convert: "USD" } } },
}),
).toContainEqual(expect.objectContaining({ path: ["requests"] }));
expect(
getDeterministicEvaluationIssues(testCase, {
...widget,
requests: {
prices: {
...widget.requests.prices,
query: { id: { $option: "symbols" }, convert: "USD" },
},
},
options: {
symbols: { label: "Symbols", control: "text", default: "1,1027,5426" },
},
}),
).toEqual([]);
});
it("accepts equivalent empty-state wording from an allowed semantic group", () => {
const baseCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === "fake-service-health");
if (!baseCase?.expectations) throw new Error("Fake service-health expectations are missing");
const testCase = {
...baseCase,
expectations: {
...baseCase.expectations,
templateIncludesAny: [["No matching services", "No services found", "Nothing to show"]],
},
};
const widget = {
$schema: "homarr-custom-widget-v2" as const,
name: "Health",
sources: {
default: { baseUrl: "https://status.example.test", networkScope: "public" as const, auth: "none" as const },
},
requests: {
health: {
source: "default",
kind: "query" as const,
method: "GET" as const,
path: "/v1/health",
trigger: "load" as const,
auth: "inherit" as const,
permission: "view" as const,
},
},
options: {},
template:
"<Stack><RefreshButton /><Text>status openIncidents latencyMs checkedAt services</Text><Text>No services found</Text></Stack>",
};
expect(getDeterministicEvaluationIssues(testCase, widget)).toEqual([]);
expect(
getDeterministicEvaluationIssues(testCase, {
...widget,
template: widget.template.replace("No services found", "All services loaded"),
}),
).toContainEqual(expect.objectContaining({ path: ["template"] }));
});
it("rejects a wrapped response rendered as an array and unwired pagination", () => {
const testCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === "seerr-media-workflows");
if (!testCase?.expectedWidgets) throw new Error("Advanced Seerr expectations are missing");
const mediaCase = {
...testCase,
expectations: testCase.expectedWidgets.find(({ id }) => id === "media-research")?.expectations,
expectedWidgets: undefined,
};
const operationsCase = {
...testCase,
expectations: testCase.expectedWidgets.find(({ id }) => id === "request-operations")?.expectations,
expectedWidgets: undefined,
};
const validMedia = {
$schema: "homarr-custom-widget-v2" as const,
name: "Seerr research",
sources: {
default: {
baseUrl: "http://seerr.local:5055/api/v1",
networkScope: "private" as const,
auth: { type: "apiKeyHeader" as const, name: "X-Api-Key" },
},
},
requests: {
search: {
source: "default",
kind: "query" as const,
method: "GET" as const,
path: "/search",
trigger: "manual" as const,
query: { query: { $param: "query" }, page: { $param: "page" } },
auth: "inherit" as const,
permission: "view" as const,
},
requestMovie: {
source: "default",
kind: "action" as const,
method: "POST" as const,
path: "/request",
trigger: "manual" as const,
body: { mediaType: "movie", mediaId: { $param: "id" } },
auth: "inherit" as const,
permission: "modify" as const,
confirmation: "Request movie?",
invalidates: ["search"],
},
requestSeries: {
source: "default",
kind: "action" as const,
method: "POST" as const,
path: "/request",
trigger: "manual" as const,
body: { mediaType: "tv", mediaId: { $param: "id" }, seasons: "all" },
auth: "inherit" as const,
permission: "modify" as const,
confirmation: "Request series?",
invalidates: ["search"],
},
},
options: {},
template:
'<Stack><TextInput bind="query" /><SubFetch requestId="search" trigger="manual" params={{ query: inputs.query, page: inputs.page ?? 1 }}>{(response) => <Stack><Group><Text>{response.totalResults}</Text><RefreshButton requestId="search" label="Run again" /></Group><SimpleGrid>{response.results.map(item => <Stack key={item.id}><Image src={`https://image.tmdb.org/t/p/w780${item.backdropPath ?? item.posterPath}`} alt={item.title} /><Text lineClamp={2}>{item.overview}</Text></Stack>)}</SimpleGrid><Pagination bind="page" defaultValue={1} resetKey={inputs.query} total={response.totalPages} /></Stack>}</SubFetch><Text>overview voteAverage ★ mediaInfo status4k Unknown Pending Processing Partially available Available Blocklisted Deleted No results</Text><ActionButton requestId="requestMovie" /><ActionButton requestId="requestSeries" /></Stack>'.repeat(
4,
),
};
const brokenMedia = {
...validMedia,
template: validMedia.template.replaceAll("response.results", "response"),
};
expect(getDeterministicEvaluationIssues(mediaCase, validMedia)).toEqual([]);
const missingSearchInvalidation = {
...validMedia,
requests: {
...validMedia.requests,
requestMovie: { ...validMedia.requests.requestMovie, invalidates: [] },
},
};
expect(getDeterministicEvaluationIssues(mediaCase, missingSearchInvalidation)).toContainEqual(
expect.objectContaining({
path: ["requests"],
message: expect.stringMatching(/invalidates.*\/search/u),
}),
);
const actionIssue = getDeterministicEvaluationIssues(mediaCase, missingSearchInvalidation).find(
(issue) => issue.path?.[0] === "requests",
);
expect(actionIssue?.message).toContain('mediaType="movie" or any $param binding');
expect(actionIssue?.message).not.toContain('["movie","$param:*"]');
const boundActionValues = {
...validMedia,
requests: {
...validMedia.requests,
requestMovie: {
...validMedia.requests.requestMovie,
body: { mediaType: { $param: "mediaType" }, mediaId: { $param: "id" } },
},
requestSeries: {
...validMedia.requests.requestSeries,
body: {
mediaType: { $param: "mediaType" },
mediaId: { $param: "id" },
seasons: { $param: "seasons" },
},
},
},
};
expect(getDeterministicEvaluationIssues(mediaCase, boundActionValues)).toEqual([]);
expect(
getDeterministicEvaluationIssues(mediaCase, {
...boundActionValues,
requests: {
...boundActionValues.requests,
requestSeries: { ...boundActionValues.requests.requestSeries, body: { mediaType: "tv" } },
},
}),
).toContainEqual(expect.objectContaining({ path: ["requests"] }));
expect(getDeterministicEvaluationIssues(mediaCase, brokenMedia)).toContainEqual(
expect.objectContaining({ message: expect.stringContaining("'.results'") }),
);
const operationsWidget = {
$schema: "homarr-custom-widget-v2" as const,
name: "Seerr operations",
sources: validMedia.sources,
requests: {
counts: {
source: "default",
kind: "query" as const,
method: "GET" as const,
path: "/request/count",
trigger: "load" as const,
auth: "inherit" as const,
permission: "view" as const,
},
recent: {
source: "default",
kind: "query" as const,
method: "GET" as const,
path: "/request",
trigger: "load" as const,
query: { take: 10, skip: 0, sort: "added", sortDirection: "desc" },
auth: "inherit" as const,
permission: "view" as const,
},
approvePending: {
source: "default",
kind: "action" as const,
method: "POST" as const,
path: "/request/{param:requestId}/approve",
trigger: "manual" as const,
auth: "inherit" as const,
permission: "modify" as const,
confirmation: "Approve request?",
invalidates: ["counts", "recent"],
},
declinePending: {
source: "default",
kind: "action" as const,
method: "POST" as const,
path: "/request/{param:requestId}/decline",
trigger: "manual" as const,
auth: "inherit" as const,
permission: "modify" as const,
confirmation: "Decline request?",
invalidates: ["counts", "recent"],
},
},
options: {},
template:
'<Stack><RefreshButton /><SimpleGrid><Text>pending approved available processing declined status.</Text></SimpleGrid><Text>{status.counts.loading} {status.recent.error} {data.recent.results} {data.recent.pageInfo} {data.recent.createdAt} UTC TMDB {data.recent.tmdbId} {data.recent.mediaType} {data.recent.status4k} Request #</Text><ActionButton requestId="approvePending" /><ActionButton requestId="declinePending" /><Pagination total={5} /></Stack>'.repeat(
8,
),
};
const operationIssues = getDeterministicEvaluationIssues(operationsCase, operationsWidget);
expect(operationIssues.filter((issue) => issue.path?.[0] === "requests")).toEqual([]);
expect(operationIssues).toContainEqual(expect.objectContaining({ message: expect.stringContaining("Pagination") }));
expect(
getDeterministicEvaluationIssues(operationsCase, {
...operationsWidget,
template: operationsWidget.template.replaceAll("status.counts", "status").replaceAll("status.recent", "status"),
}),
).toContainEqual(expect.objectContaining({ message: expect.stringContaining("no global status.loading") }));
});
it("includes every path-specific Pokédex fixture in generator and judge context", () => {
const testCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === "pokedex");
if (!testCase) throw new Error("Pokédex evaluation case is missing");
const fixtureText = getEvaluationResponseFixtureText(testCase);
expect(fixtureText).toContain("/api/v2/pokemon/{param:name}");
expect(fixtureText).toContain('"official-artwork"');
expect(fixtureText).toContain('"results"');
expect(testCase.minimumPreviewCycles).toBe(3);
});
it("uses the exact clipboard prompt and grounded API notes", () => {
const testCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.at(0);
if (!testCase) throw new Error("Expected at least one AI evaluation case");
const prompt = buildEvaluationPrompt(testCase);
expect(prompt.length).toBeLessThanOrEqual(12_000);
expect(prompt).toContain("Verified API notes");
expect(prompt).not.toContain("customWidget_validate");
expect(prompt).not.toContain("homarr://");
});
it("keeps validation failures concrete during repair", () => {
const prompt = buildRepairPrompt("original", "bad response", [
{ path: ["requests", "list", "path"], message: "Required" },
]);
expect(prompt).toContain("requests.list.path: Required");
expect(prompt).toContain("bad response");
});
it("uses the requested DeepSeek models and strict structured judge output", () => {
expect(DEFAULT_GENERATOR_MODEL).toBe("deepseek/deepseek-v4.1-flash");
expect(DEFAULT_JUDGE_MODEL).toBe("openai/gpt-5.6-luna");
expect(getAiEvaluationReasoningOptions(DEFAULT_GENERATOR_MODEL)).toEqual({ effort: "xhigh", exclude: true });
expect(getAiEvaluationReasoningOptions(DEFAULT_JUDGE_MODEL)).toEqual({ effort: "high", exclude: true });
expect(getAiEvaluationProviderPreferences(DEFAULT_GENERATOR_MODEL)).toEqual({
order: ["deepinfra/fp8"],
quantizations: ["fp8"],
allow_fallbacks: true,
});
expect(getAiEvaluationProviderPreferences(DEFAULT_JUDGE_MODEL)).toBeUndefined();
const format = getJudgeResponseFormat();
expect(format.type).toBe("json_schema");
expect(format.json_schema.strict).toBe(true);
expect(format.json_schema.schema).toMatchObject({ type: "object", additionalProperties: false });
});
it("can run the same evaluation against the Homarr provider endpoint", () => {
expect(getAiProviderChatCompletionsUrl("https://homarr.dev/api/ai/v1/")).toBe(
"https://homarr.dev/api/ai/v1/chat/completions",
);
expect(getAiProviderChatCompletionsUrl()).toBe("https://openrouter.ai/api/v1/chat/completions");
expect(
resolveAiEvaluationProviderConfig({
AI_PROVIDER_BASE_URL: "https://homarr.dev/api/ai/v1/",
AI_PROVIDER_API_KEY: "workshop-token",
}),
).toEqual({
apiKey: "workshop-token",
baseUrl: "https://homarr.dev/api/ai/v1",
generatorModel: "homarr/model",
judgeModel: "homarr/model",
judgeBaseUrl: "https://homarr.dev/api/ai/v1",
judgeApiKey: "workshop-token",
});
expect(
resolveAiEvaluationProviderConfig({
AI_PROVIDER_BASE_URL: " ",
OPENROUTER_API_KEY: "legacy-key",
OPENROUTER_GENERATOR_MODEL: "legacy-generator",
OPENROUTER_JUDGE_MODEL: "legacy-judge",
}),
).toEqual({
apiKey: "legacy-key",
baseUrl: DEFAULT_AI_PROVIDER_BASE_URL,
generatorModel: "legacy-generator",
judgeModel: "legacy-judge",
judgeBaseUrl: DEFAULT_AI_PROVIDER_BASE_URL,
judgeApiKey: "legacy-key",
});
expect(
resolveAiEvaluationProviderConfig({
AI_PROVIDER_BASE_URL: "https://homarr.dev/api/ai/v1",
AI_PROVIDER_API_KEY: "workshop-token",
OPENROUTER_API_KEY: "judge-key",
OPENROUTER_GENERATOR_MODEL: "must-not-apply",
OPENROUTER_JUDGE_MODEL: "independent-judge",
}),
).toEqual({
apiKey: "workshop-token",
baseUrl: "https://homarr.dev/api/ai/v1",
generatorModel: "homarr/model",
judgeModel: "independent-judge",
judgeBaseUrl: DEFAULT_AI_PROVIDER_BASE_URL,
judgeApiKey: "judge-key",
});
});
it("requires an excellent result without weak categories", () => {
expect(
judgePasses({
...makeJudgeResult(85),
categories: {
...makeCategories(85),
visualQuality: 76,
dailyUsefulness: 76,
complexityDiscipline: 80,
},
}),
).toBe(true);
expect(judgePasses(makeJudgeResult(84))).toBe(false);
expect(
judgePasses({
...makeJudgeResult(90),
categories: { ...makeCategories(90), accessibility: 74 },
}),
).toBe(false);
expect(
judgePasses({
...makeJudgeResult(92),
categories: { ...makeCategories(92), goalFulfillment: 84 },
}),
).toBe(false);
expect(
judgePasses({
...makeJudgeResult(92),
categories: { ...makeCategories(92), complexityDiscipline: 79 },
}),
).toBe(false);
expect(judgePasses({ ...makeJudgeResult(92), dailyUseDecision: "promising-but-not-daily" })).toBe(false);
expect(judgePasses({ ...makeJudgeResult(95), fatalProblems: ["A requested core action is missing."] })).toBe(false);
});
it("uses a unanimous three-vote panel and escalates split votes to five", () => {
const unanimous = summarizeCustomWidgetJudgePanel(judgePanelWidget, makePanelCalls([87, 89, 86]));
expect(unanimous).toMatchObject({ status: "pass", passVotes: 3, failVotes: 0, medianTotal: 87 });
expect(unanimous.scoreRange).toEqual({ min: 86, max: 89 });
expect(unanimous.artifactHash).toMatch(/^[a-f0-9]{64}$/u);
const split = summarizeCustomWidgetJudgePanel(judgePanelWidget, makePanelCalls([87, 83, 86]));
expect(split.status).toBe("inconclusive");
const resolvedPass = summarizeCustomWidgetJudgePanel(judgePanelWidget, makePanelCalls([87, 83, 86, 88, 82]));
expect(resolvedPass).toMatchObject({ status: "pass", passVotes: 3, failVotes: 2, medianTotal: 86 });
const resolvedFail = summarizeCustomWidgetJudgePanel(judgePanelWidget, makePanelCalls([87, 83, 82, 88, 81]));
expect(resolvedFail).toMatchObject({ status: "fail", passVotes: 2, failVotes: 3, medianTotal: 83 });
if (!resolvedFail.representative) throw new Error("Expected a representative judge result");
expect(judgePasses(resolvedFail.representative)).toBe(false);
});
it("marks an incomplete judge panel inconclusive", () => {
const calls = makePanelCalls([90, 89]);
calls.push({ call: 3, raw: null, result: null, error: "empty provider response" });
expect(summarizeCustomWidgetJudgePanel(judgePanelWidget, calls)).toMatchObject({
status: "inconclusive",
passVotes: 2,
failVotes: 0,
medianTotal: 90,
});
});
it("gives the harsh judge the authoritative request-state runtime contract", () => {
const testCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === "fake-service-health");
if (!testCase) throw new Error("Fake service-health case is missing");
const widget = {
$schema: "homarr-custom-widget-v2" as const,
name: "Health",
sources: {
default: { baseUrl: "https://status.example.test", networkScope: "public" as const, auth: "none" as const },
},
requests: {},
options: {},
template: "<Text>Health</Text>",
};
const prompt = buildJudgePrompt(testCase, widget);
expect(prompt).toContain("status.<requestId> with loading/ok/status/error fields");
expect(prompt).toContain("RefreshButton is an installed runtime helper");
expect(prompt).toContain("cannot display stale results under edited inputs");
expect(prompt).toContain("do not penalize a repeated short literal label array");
expect(prompt).toContain(JSON.stringify(testCase.sampleResponse, null, 2));
expect(prompt).toContain("# Runtime");
expect(prompt).not.toContain("# Bundled file:");
expect(prompt).toContain("does not prove API correctness, visual quality, usefulness, or accessibility");
expect(prompt).toContain(
"verified API notes and representative response fixtures are authoritative for endpoint paths, authentication requirements, and response shapes",
);
expect(prompt).toContain("Do not invent external endpoint or authentication objections from outside assumptions");
expect(prompt).toContain("an endpoint mentioned in broader API notes is available, not automatically required");
expect(prompt).toContain(
"Decorative icons paired with equivalent adjacent visible status text need no separate aria-label",
);
expect(prompt).toContain("A Badge containing explicit visible status text is not color-only");
expect(prompt).toContain('Date.toLocaleString(value, "en-US", "UTC") is an installed safe static helper');
expect(prompt).toContain(
'A pass requires a weighted total of at least 85, every category at least 75, goalFulfillment at least 85, complexityDiscipline at least 80, no fatal problem, and dailyUseDecision="would-use-daily"',
);
});
it("computes the weighted score and refuses inflated advisory verdicts", () => {
const parsed = parseJudgeResult(
JSON.stringify({
...makeJudgeResult(95),
total: 99,
categories: { ...makeCategories(95), visualQuality: 70 },
categoryReasons: makeReasons(),
strengths: [],
problems: [],
fatalProblems: [],
highestImpactFixes: [],
}),
);
expect(parsed.total).toBe(91);
expect(parsed.verdict).toBe("fail");
});
it("accepts a provider response that wraps structured review JSON in a code fence", () => {
const parsed = parseJudgeResult(`\`\`\`json\n${JSON.stringify(makeJudgeResult(90))}\n\`\`\``);
expect(parsed.total).toBe(90);
expect(parsed.verdict).toBe("pass");
});
it("normalizes a prose 'none' entry out of an otherwise empty fatal-problem list", () => {
const parsed = parseJudgeResult(
JSON.stringify({
...makeJudgeResult(90),
fatalProblems: ["None: all requested capabilities are present.", "No fatal issues."],
}),
);
expect(parsed.fatalProblems).toEqual([]);
expect(parsed.verdict).toBe("pass");
});
});