refactor: trim v2 release hardening
This commit is contained in:
1 parent
3cf1dc8341
commit
657704340c
79 files changed
+405
-3378
No files matched your search
@@ -3,20 +3,6 @@ export interface CustomWidgetAiEvaluationCase {
|
||||
request: string;
|
||||
documentationUrl: string;
|
||||
apiNotes: string;
|
||||
acceptance: {
|
||||
sourceAuth?: { type: "none" | "bearer" | "apiKeyHeader" | "apiKeyQuery"; name?: string };
|
||||
requestRules: ReadonlyArray<{
|
||||
label: string;
|
||||
pathIncludes: string;
|
||||
kind?: "query" | "action";
|
||||
method?: "GET" | "POST" | "PUT" | "DELETE" | "PATCH";
|
||||
trigger?: "load" | "manual";
|
||||
optionReference?: string;
|
||||
parameterReference?: string;
|
||||
invalidatesRequest?: string;
|
||||
}>;
|
||||
templateComponents: readonly string[];
|
||||
};
|
||||
}
|
||||
|
||||
export const CUSTOM_WIDGET_AI_EVALUATION_CASES: readonly CustomWidgetAiEvaluationCase[] = [
|
||||
@@ -27,20 +13,6 @@ export const CUSTOM_WIDGET_AI_EVALUATION_CASES: readonly CustomWidgetAiEvaluatio
|
||||
"Create a polished Pokédex browser with searchable Pokémon, a responsive result grid, manual detail loading, sprites, types, abilities, base-stat progress bars, and clear loading, empty, and failure states.",
|
||||
apiNotes:
|
||||
"Use GET /api/v2/pokemon?limit=<option>&offset=<option> for the list and GET /api/v2/pokemon/{param:name} as a manual detail query. PokeAPI needs no authentication.",
|
||||
acceptance: {
|
||||
sourceAuth: { type: "none" },
|
||||
requestRules: [
|
||||
{ label: "Pokémon list", pathIncludes: "/api/v2/pokemon", kind: "query", trigger: "load" },
|
||||
{
|
||||
label: "manual Pokémon detail",
|
||||
pathIncludes: "/api/v2/pokemon/",
|
||||
kind: "query",
|
||||
trigger: "manual",
|
||||
parameterReference: "name",
|
||||
},
|
||||
],
|
||||
templateComponents: ["TextInput", "SubFetch", "Image", "Progress"],
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "portainer-containers",
|
||||
@@ -49,87 +21,22 @@ export const CUSTOM_WIDGET_AI_EVALUATION_CASES: readonly CustomWidgetAiEvaluatio
|
||||
"Create an excellent Portainer container dashboard: environment option, running/stopped summary, responsive container list, health/status badges, and explicit start, stop, and restart actions with confirmation and refresh after success.",
|
||||
apiNotes:
|
||||
"Use X-API-Key auth. List containers with GET /api/endpoints/{option:endpointId}/docker/containers/json?all=true. Actions are POST /api/endpoints/{option:endpointId}/docker/containers/{param:id}/start, /stop, and /restart and invalidate the list query.",
|
||||
acceptance: {
|
||||
sourceAuth: { type: "apiKeyHeader", name: "X-API-Key" },
|
||||
requestRules: [
|
||||
{
|
||||
label: "container list",
|
||||
pathIncludes: "/containers/json",
|
||||
kind: "query",
|
||||
optionReference: "endpointId",
|
||||
},
|
||||
{
|
||||
label: "start action",
|
||||
pathIncludes: "/start",
|
||||
kind: "action",
|
||||
method: "POST",
|
||||
optionReference: "endpointId",
|
||||
parameterReference: "id",
|
||||
invalidatesRequest: "container list",
|
||||
},
|
||||
{
|
||||
label: "stop action",
|
||||
pathIncludes: "/stop",
|
||||
kind: "action",
|
||||
method: "POST",
|
||||
optionReference: "endpointId",
|
||||
parameterReference: "id",
|
||||
invalidatesRequest: "container list",
|
||||
},
|
||||
{
|
||||
label: "restart action",
|
||||
pathIncludes: "/restart",
|
||||
kind: "action",
|
||||
method: "POST",
|
||||
optionReference: "endpointId",
|
||||
parameterReference: "id",
|
||||
invalidatesRequest: "container list",
|
||||
},
|
||||
],
|
||||
templateComponents: ["ActionButton", "Badge", "RefreshButton"],
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "football-dashboard",
|
||||
documentationUrl: "https://docs.football-data.org/general/v4/competition.html",
|
||||
id: "tautulli-activity",
|
||||
documentationUrl: "https://github.com/Tautulli/Tautulli/wiki/Tautulli-API-Reference",
|
||||
request:
|
||||
"Create a polished football dashboard with a competition selector, league table, upcoming fixtures, club crests, match status and kickoff time. It must be highly scannable in narrow and wide tiles, refreshable, and include complete loading, empty, and error states.",
|
||||
"Create a beautiful Tautulli activity widget with active stream cards, user/player details, progress, transcode/direct-play badges, bandwidth summary, and a refresh control that works in narrow and wide tiles.",
|
||||
apiNotes:
|
||||
"Use https://api.football-data.org/v4 with X-Auth-Token authentication. GET /competitions/{option:competition}/standings returns standings[].table with position, team, playedGames, won, draw, lost, points and goalDifference. GET /competitions/{option:competition}/matches with query status=SCHEDULED returns matches with utcDate, status, homeTeam, awayTeam and score. Use fixed choices such as PL, CL, PD, BL1, SA and FL1 for the competition option.",
|
||||
acceptance: {
|
||||
sourceAuth: { type: "apiKeyHeader", name: "X-Auth-Token" },
|
||||
requestRules: [
|
||||
{
|
||||
label: "competition standings",
|
||||
pathIncludes: "/standings",
|
||||
kind: "query",
|
||||
optionReference: "competition",
|
||||
},
|
||||
{
|
||||
label: "competition fixtures",
|
||||
pathIncludes: "/matches",
|
||||
kind: "query",
|
||||
optionReference: "competition",
|
||||
},
|
||||
],
|
||||
templateComponents: ["Table", "Image", "Badge", "RefreshButton"],
|
||||
},
|
||||
"Use an apiKeyQuery source with parameter name apikey. Query GET /api/v2 with query cmd=get_activity. The response payload is under response.data and sessions is an array.",
|
||||
},
|
||||
{
|
||||
id: "jellyfin-activity",
|
||||
documentationUrl: "https://api.jellyfin.org/",
|
||||
id: "bambubuddy-printer",
|
||||
documentationUrl: "https://wiki.bambuddy.cool/reference/api/",
|
||||
request:
|
||||
"Create a premium Jellyfin overview with movie, series, episode and song totals plus active playback sessions. Show user, title, client, play method, progress, paused/playing state and a calm empty state when nothing is playing. It must remain useful in narrow and wide tiles.",
|
||||
"Create a premium BambuBuddy printer status widget with printer selector, current job, progress, remaining time, nozzle and bed temperatures, connection state, and safe pause/resume/stop controls when supported.",
|
||||
apiNotes:
|
||||
"Use the user's Jellyfin server URL with private network scope and X-Emby-Token authentication. GET /Items/Counts returns MovieCount, SeriesCount, EpisodeCount and SongCount. GET /Sessions with query activeWithinSeconds=300 returns sessions; active playback has NowPlayingItem, UserName, Client, PlayState.PositionTicks, PlayState.IsPaused and TranscodingInfo or NowPlayingItem.RunTimeTicks. Do not invent playback-control actions.",
|
||||
acceptance: {
|
||||
sourceAuth: { type: "apiKeyHeader", name: "X-Emby-Token" },
|
||||
requestRules: [
|
||||
{ label: "library counts", pathIncludes: "/Items/Counts", kind: "query" },
|
||||
{ label: "active sessions", pathIncludes: "/Sessions", kind: "query" },
|
||||
],
|
||||
templateComponents: ["Progress", "Badge", "RefreshButton"],
|
||||
},
|
||||
"Use X-API-Key auth and base path /api/v1. GET /printers lists printers and GET /printers/{id}/status returns state, progress, remaining_time, temperatures.nozzle, temperatures.bed, and hms_status. The official reference does not document pause, resume, or stop endpoints, so omit those actions. Configure the selected printer with choicesFrom on a widget option and use that option in the status path.",
|
||||
},
|
||||
{
|
||||
id: "home-assistant-control",
|
||||
@@ -138,39 +45,5 @@ export const CUSTOM_WIDGET_AI_EVALUATION_CASES: readonly CustomWidgetAiEvaluatio
|
||||
"Create a refined Home Assistant room widget with temperature and humidity readings, light status, a room/entity option, and an actionable light toggle. Use calm hierarchy and responsive controls, not a pile of nested cards.",
|
||||
apiNotes:
|
||||
"Use bearer auth. GET /api/states/{option:sensorEntity} and /api/states/{option:lightEntity} load entity state. POST /api/services/light/turn_on and /turn_off accept a body with entity_id from an option and should invalidate the light query.",
|
||||
acceptance: {
|
||||
sourceAuth: { type: "bearer" },
|
||||
requestRules: [
|
||||
{
|
||||
label: "sensor state",
|
||||
pathIncludes: "/api/states/",
|
||||
kind: "query",
|
||||
optionReference: "sensorEntity",
|
||||
},
|
||||
{
|
||||
label: "light state",
|
||||
pathIncludes: "/api/states/",
|
||||
kind: "query",
|
||||
optionReference: "lightEntity",
|
||||
},
|
||||
{
|
||||
label: "light on action",
|
||||
pathIncludes: "/api/services/light/turn_on",
|
||||
kind: "action",
|
||||
method: "POST",
|
||||
optionReference: "lightEntity",
|
||||
invalidatesRequest: "light state",
|
||||
},
|
||||
{
|
||||
label: "light off action",
|
||||
pathIncludes: "/api/services/light/turn_off",
|
||||
kind: "action",
|
||||
method: "POST",
|
||||
optionReference: "lightEntity",
|
||||
invalidatesRequest: "light state",
|
||||
},
|
||||
],
|
||||
templateComponents: ["ToggleSwitch", "Badge", "SimpleGrid"],
|
||||
},
|
||||
},
|
||||
] as const;
|
||||
@@ -7,7 +7,6 @@ import { buildCustomWidgetAiPrompt } from "../src/core/ai-prompt";
|
||||
import { customWidgetDefinitionSchema } from "../src/core/custom-jsx-schema";
|
||||
import type { HomarrCustomWidgetV2 } from "../src/core/custom-jsx-schema";
|
||||
import { formatCustomWidgetImportIssues, parseCustomWidgetAiResponse } from "../src/core/import";
|
||||
import { collectCustomWidgetRequestReferences } from "../src/core/request-schema";
|
||||
import type { CustomWidgetAiEvaluationCase } from "./ai-evaluation-cases";
|
||||
|
||||
export const DEFAULT_GENERATOR_MODEL = "deepseek/deepseek-v4-pro";
|
||||
@@ -101,7 +100,7 @@ export function buildRepairPrompt(
|
||||
const diagnostics = issues
|
||||
.map((issue) => `${issue.path?.length ? `${issue.path.join(".")}: ` : ""}${issue.message}`)
|
||||
.join("\n");
|
||||
return `${originalPrompt}\n\nYour previous response did not meet the release quality bar. Correct every diagnosed contract, runtime, goal-fulfillment, visual, and UX problem below. Preserve grounded API behavior and the strongest working parts, but redesign weak areas when the feedback requires it.\n\nDiagnostics:\n${diagnostics}\n\nPrevious response:\n${previousResponse}`;
|
||||
return `${originalPrompt}\n\nYour previous response did not validate. Correct only the generic contract/runtime problems below while preserving a polished design.\n\nDiagnostics:\n${diagnostics}\n\nPrevious response:\n${previousResponse}`;
|
||||
}
|
||||
|
||||
export function buildJudgePrompt(testCase: CustomWidgetAiEvaluationCase, widget: HomarrCustomWidgetV2): string {
|
||||
@@ -159,85 +158,6 @@ export function getJudgeResponseFormat() {
|
||||
};
|
||||
}
|
||||
|
||||
function matchesScenarioRequestRule(
|
||||
request: HomarrCustomWidgetV2["requests"][string],
|
||||
rule: CustomWidgetAiEvaluationCase["acceptance"]["requestRules"][number],
|
||||
) {
|
||||
if (!request.path.includes(rule.pathIncludes)) return false;
|
||||
if (rule.kind && request.kind !== rule.kind) return false;
|
||||
if (rule.method && request.method !== rule.method) return false;
|
||||
if (rule.trigger && request.trigger !== rule.trigger) return false;
|
||||
const references = collectCustomWidgetRequestReferences(request);
|
||||
if (rule.optionReference && !references.options.has(rule.optionReference)) return false;
|
||||
if (rule.parameterReference && !references.params.has(rule.parameterReference)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
export function getScenarioAcceptanceIssues(
|
||||
testCase: CustomWidgetAiEvaluationCase,
|
||||
widget: HomarrCustomWidgetV2,
|
||||
): Array<{ path?: Array<string | number>; message: string }> {
|
||||
const issues: Array<{ path?: Array<string | number>; message: string }> = [];
|
||||
const expectedAuth = testCase.acceptance.sourceAuth;
|
||||
if (expectedAuth) {
|
||||
const matches = Object.values(widget.sources).some((source) => {
|
||||
const actualType = typeof source.auth === "string" ? source.auth : source.auth.type;
|
||||
if (actualType !== expectedAuth.type) return false;
|
||||
return (
|
||||
expectedAuth.name === undefined ||
|
||||
(typeof source.auth !== "string" && source.auth.name.toLowerCase() === expectedAuth.name.toLowerCase())
|
||||
);
|
||||
});
|
||||
if (!matches) {
|
||||
issues.push({
|
||||
path: ["sources"],
|
||||
message: `The scenario requires ${expectedAuth.type}${expectedAuth.name ? ` '${expectedAuth.name}'` : ""} authentication.`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const requestEntries = Object.entries(widget.requests);
|
||||
const candidateMatches = new Map<string, [string, HomarrCustomWidgetV2["requests"][string]]>();
|
||||
const candidateRequestIds = new Set<string>();
|
||||
for (const rule of testCase.acceptance.requestRules) {
|
||||
const match = requestEntries.find(
|
||||
([requestId, request]) => !candidateRequestIds.has(requestId) && matchesScenarioRequestRule(request, rule),
|
||||
);
|
||||
if (match) {
|
||||
candidateMatches.set(rule.label, match);
|
||||
candidateRequestIds.add(match[0]);
|
||||
}
|
||||
}
|
||||
|
||||
const requestIdsByLabel = new Map([...candidateMatches].map(([label, [requestId]]) => [label, requestId] as const));
|
||||
for (const rule of testCase.acceptance.requestRules) {
|
||||
const match = candidateMatches.get(rule.label);
|
||||
if (!match) {
|
||||
issues.push({
|
||||
path: ["requests"],
|
||||
message: `Missing grounded ${rule.label} request (${rule.pathIncludes}).`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (rule.invalidatesRequest) {
|
||||
const invalidatedRequestId = requestIdsByLabel.get(rule.invalidatesRequest);
|
||||
if (!invalidatedRequestId || !match[1].invalidates?.includes(invalidatedRequestId)) {
|
||||
issues.push({
|
||||
path: ["requests"],
|
||||
message: `The ${rule.label} request must invalidate the ${rule.invalidatesRequest} request.`,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const component of testCase.acceptance.templateComponents) {
|
||||
if (!widget.template.includes(`<${component}`)) {
|
||||
issues.push({ path: ["template"], message: `The ${testCase.id} scenario requires ${component}.` });
|
||||
}
|
||||
}
|
||||
return issues;
|
||||
}
|
||||
|
||||
export async function evaluateCustomWidgetCase(args: {
|
||||
testCase: CustomWidgetAiEvaluationCase;
|
||||
apiKey: string;
|
||||
@@ -281,14 +201,6 @@ export async function evaluateCustomWidgetCase(args: {
|
||||
|
||||
const canonical = customWidgetDefinitionSchema.parse(parsed.widget);
|
||||
await writeWidgetFiles(caseDirectory, canonical, `attempt-${attempt}`);
|
||||
const acceptanceIssues = getScenarioAcceptanceIssues(args.testCase, canonical);
|
||||
if (acceptanceIssues.length > 0) {
|
||||
errors.push(
|
||||
`Attempt ${attempt}: deterministic acceptance failed — ${acceptanceIssues.map(({ message }) => message).join("; ")}`,
|
||||
);
|
||||
prompt = buildRepairPrompt(originalPrompt, response, acceptanceIssues);
|
||||
continue;
|
||||
}
|
||||
let judge: CustomWidgetJudgeResult;
|
||||
try {
|
||||
const judgeRaw = await callOpenRouter({
|
||||
|
||||
@@ -10,28 +10,35 @@ if (!apiKey) throw new Error("OPENROUTER_API_KEY is required for the live Custom
|
||||
|
||||
const requestedCase = process.argv.find((value) => value.startsWith("--case="))?.slice("--case=".length);
|
||||
const configuredLoops = Number(process.env.CUSTOM_WIDGET_AI_MAX_LOOPS ?? 10);
|
||||
const maxLoopsPerCase = Number.isInteger(configuredLoops) && configuredLoops > 0 ? Math.min(configuredLoops, 10) : 10;
|
||||
const totalLoopBudget = Number.isInteger(configuredLoops) && configuredLoops > 0 ? Math.min(configuredLoops, 10) : 10;
|
||||
const selectedCases = requestedCase
|
||||
? CUSTOM_WIDGET_AI_EVALUATION_CASES.filter((testCase) => testCase.id === requestedCase)
|
||||
: CUSTOM_WIDGET_AI_EVALUATION_CASES;
|
||||
if (selectedCases.length === 0) throw new Error(`Unknown AI evaluation case '${requestedCase}'`);
|
||||
if (totalLoopBudget < selectedCases.length) {
|
||||
throw new Error(`CUSTOM_WIDGET_AI_MAX_LOOPS must be at least ${selectedCases.length} for this run`);
|
||||
}
|
||||
|
||||
const runId = new Date().toISOString().replaceAll(/[:.]/gu, "-");
|
||||
const outputRoot = path.resolve(process.cwd(), ".ai-evaluations", runId);
|
||||
await mkdir(outputRoot, { recursive: true });
|
||||
|
||||
const results: AiEvaluationResult[] = [];
|
||||
let remainingLoopBudget = totalLoopBudget;
|
||||
for (const testCase of selectedCases) {
|
||||
const remainingCases = selectedCases.length - results.length;
|
||||
const maxLoops = Math.max(1, Math.floor(remainingLoopBudget / remainingCases));
|
||||
process.stdout.write(`Evaluating ${testCase.id}...\n`);
|
||||
const result = await evaluateCustomWidgetCase({
|
||||
testCase,
|
||||
apiKey,
|
||||
outputRoot,
|
||||
maxLoops: maxLoopsPerCase,
|
||||
maxLoops,
|
||||
generatorModel: process.env.OPENROUTER_GENERATOR_MODEL ?? DEFAULT_GENERATOR_MODEL,
|
||||
judgeModel: process.env.OPENROUTER_JUDGE_MODEL ?? DEFAULT_JUDGE_MODEL,
|
||||
});
|
||||
results.push(result);
|
||||
remainingLoopBudget -= result.attempts;
|
||||
process.stdout.write(
|
||||
result.judge
|
||||
? ` ${result.judge.total}/100 after ${result.attempts} attempt(s)\n`
|
||||
|
||||
@@ -47,7 +47,6 @@ const program = ts.createProgram({ rootNames: [...config.fileNames, import.meta.
|
||||
const checker = program.getTypeChecker();
|
||||
const source = program.getSourceFile(import.meta.filename);
|
||||
if (!source) throw new Error("Unable to inspect the component catalog generator");
|
||||
const sourceNode: ts.Node = source;
|
||||
|
||||
const moduleSymbols = new Map<string, ts.Symbol>();
|
||||
for (const declaration of source.statements) {
|
||||
@@ -164,7 +163,7 @@ function inspectMantineComponent(component: CatalogSourceComponent): InspectedCo
|
||||
const rootName = component.name.split(".")[0];
|
||||
let symbol =
|
||||
moduleSymbol && checker.getExportsOfModule(moduleSymbol).find((candidate) => candidate.name === rootName);
|
||||
let location: ts.Node = symbol?.valueDeclaration ?? symbol?.declarations?.[0] ?? sourceNode;
|
||||
let location = symbol?.valueDeclaration ?? symbol?.declarations?.[0] ?? source;
|
||||
let type = symbol && checker.getTypeOfSymbolAtLocation(symbol, location);
|
||||
for (const segment of component.name.split(".").slice(1)) {
|
||||
symbol = type?.getProperty(segment);
|
||||
@@ -312,7 +311,7 @@ function mergeGlobalProp(name: string, candidates: UninternedPropDescriptor[]):
|
||||
const fallback = createFallbackProp(name, "global");
|
||||
const type = mostFrequent(candidates.map((candidate) => candidate.type)) ?? fallback.type;
|
||||
const descriptions = candidates.flatMap(({ description }) => (description ? [description] : []));
|
||||
const literalValues = uniqueLiterals(candidates.flatMap((candidate) => candidate.literalValues ?? []));
|
||||
const literalValues = uniqueLiterals(candidates.flatMap(({ literalValues }) => literalValues ?? []));
|
||||
return {
|
||||
name,
|
||||
type,
|
||||
|
||||
Reference in new issue
Block a user