import { mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; import { z } from "zod/v4"; import { buildCustomWidgetAiPrompt } from "../src/core/ai-prompt"; import { customWidgetDefinitionSchema } from "../src/core/custom-jsx-schema"; import type { HomarrCustomWidgetV2 } from "../src/core/custom-jsx-schema"; import { formatCustomWidgetImportIssues, parseCustomWidgetAiResponse } from "../src/core/import"; import type { CustomWidgetAiEvaluationCase } from "./ai-evaluation-cases"; export const DEFAULT_GENERATOR_MODEL = "deepseek/deepseek-v4-pro"; export const DEFAULT_JUDGE_MODEL = "deepseek/deepseek-v4-flash"; export const MAX_AI_EVALUATION_LOOPS = 10; const scoreSchema = z.number().int().min(0).max(100); const judgeResultSchema = z.strictObject({ total: scoreSchema, verdict: z.enum(["pass", "fail"]), categories: z.strictObject({ schemaAndBindings: scoreSchema, apiAndRequestDesign: scoreSchema, runtimeCompatibility: scoreSchema, visualHierarchy: scoreSchema, responsiveAndTheme: scoreSchema, loadingEmptyErrorSuccess: scoreSchema, accessibility: scoreSchema, actionSafety: scoreSchema, simplicity: scoreSchema, }), strengths: z.array(z.string()).max(8), problems: z.array(z.string()).max(12), highestImpactFixes: z.array(z.string()).max(8), }); export type CustomWidgetJudgeResult = z.infer; export interface AiEvaluationResult { caseId: string; attempts: number; widget: HomarrCustomWidgetV2 | null; judge: CustomWidgetJudgeResult | null; outputDirectory: string; errors: string[]; } interface OpenRouterResponse { choices?: Array<{ message?: { content?: string | null } }>; error?: { message?: string }; } export function buildEvaluationPrompt(testCase: CustomWidgetAiEvaluationCase): string { return buildCustomWidgetAiPrompt( undefined, null, null, `${testCase.request}\n\nVerified API notes:\n${testCase.apiNotes}`, testCase.documentationUrl, ); } export function buildRepairPrompt( originalPrompt: string, previousResponse: string, issues: readonly { path?: Array; message: string }[], ): string { const diagnostics = issues .map((issue) => `${issue.path?.length ? `${issue.path.join(".")}: ` : ""}${issue.message}`) .join("\n"); return `${originalPrompt}\n\nYour previous response did not validate. Correct only the generic contract/runtime problems below while preserving a polished design.\n\nDiagnostics:\n${diagnostics}\n\nPrevious response:\n${previousResponse}`; } export function buildJudgePrompt(testCase: CustomWidgetAiEvaluationCase, widget: HomarrCustomWidgetV2): string { return `You are a strict evaluator for a safe dashboard-widget language. Judge the widget against the request and API notes. Do not reward unsupported capabilities, invented API routes, or visual claims not present in the JSX. Request: ${testCase.request} Verified API notes: ${testCase.apiNotes} Validated widget: ${JSON.stringify(widget, null, 2)} Score every category from 0 to 100. The total must reflect the whole result. A pass requires total >= 80 and no category below 65. Return JSON only with this exact shape: {"total":0,"verdict":"fail","categories":{"schemaAndBindings":0,"apiAndRequestDesign":0,"runtimeCompatibility":0,"visualHierarchy":0,"responsiveAndTheme":0,"loadingEmptyErrorSuccess":0,"accessibility":0,"actionSafety":0,"simplicity":0},"strengths":[],"problems":[],"highestImpactFixes":[]}`; } export function judgePasses(result: CustomWidgetJudgeResult): boolean { return result.total >= 80 && Object.values(result.categories).every((score) => score >= 65); } export async function evaluateCustomWidgetCase(args: { testCase: CustomWidgetAiEvaluationCase; apiKey: string; outputRoot: string; maxLoops: number; generatorModel?: string; judgeModel?: string; }): Promise { const caseDirectory = path.join(args.outputRoot, args.testCase.id); await mkdir(caseDirectory, { recursive: true }); const originalPrompt = buildEvaluationPrompt(args.testCase); await writeFile(path.join(caseDirectory, "prompt.md"), originalPrompt, "utf8"); let prompt = originalPrompt; const errors: string[] = []; let bestWidget: HomarrCustomWidgetV2 | null = null; let bestJudge: CustomWidgetJudgeResult | null = null; for (let attempt = 1; attempt <= Math.min(args.maxLoops, MAX_AI_EVALUATION_LOOPS); attempt += 1) { let response: string; try { response = await callOpenRouter({ apiKey: args.apiKey, model: args.generatorModel ?? DEFAULT_GENERATOR_MODEL, prompt, json: false, }); } catch (error) { errors.push( `Attempt ${attempt}: generator request failed — ${error instanceof Error ? error.message : "Unknown error"}`, ); continue; } await writeFile(path.join(caseDirectory, `attempt-${attempt}.md`), response, "utf8"); const parsed = parseCustomWidgetAiResponse(response); if (!parsed.success) { const message = formatCustomWidgetImportIssues(parsed.issues); errors.push(`Attempt ${attempt}: ${message}`); prompt = buildRepairPrompt(originalPrompt, response, parsed.issues); continue; } const canonical = customWidgetDefinitionSchema.parse(parsed.widget); await writeWidgetFiles(caseDirectory, canonical, `attempt-${attempt}`); let judge: CustomWidgetJudgeResult; try { const judgeRaw = await callOpenRouter({ apiKey: args.apiKey, model: args.judgeModel ?? DEFAULT_JUDGE_MODEL, prompt: buildJudgePrompt(args.testCase, canonical), json: true, }); await writeFile(path.join(caseDirectory, `judge-${attempt}.json`), judgeRaw, "utf8"); judge = parseJudgeResult(judgeRaw); } catch (error) { errors.push( `Attempt ${attempt}: judge response invalid — ${error instanceof Error ? error.message : "Unknown error"}`, ); continue; } if (!bestJudge || judge.total > bestJudge.total) { bestJudge = judge; bestWidget = canonical; await writeWidgetFiles(caseDirectory, canonical, "best"); await writeFile(path.join(caseDirectory, "best-judge.json"), JSON.stringify(judge, null, 2), "utf8"); } if (judgePasses(judge)) { await writeFile(path.join(caseDirectory, "result.json"), JSON.stringify(judge, null, 2), "utf8"); return { caseId: args.testCase.id, attempts: attempt, widget: canonical, judge, outputDirectory: caseDirectory, errors, }; } errors.push(`Attempt ${attempt}: judge ${judge.total}/100 — ${judge.highestImpactFixes.join("; ")}`); prompt = buildRepairPrompt( originalPrompt, response, judge.highestImpactFixes.map((message) => ({ message })), ); } return { caseId: args.testCase.id, attempts: Math.min(args.maxLoops, MAX_AI_EVALUATION_LOOPS), widget: bestWidget, judge: bestJudge, outputDirectory: caseDirectory, errors, }; } export function parseJudgeResult(raw: string): CustomWidgetJudgeResult { const fenced = /```(?:json)?\s*([\s\S]*?)```/iu.exec(raw)?.[1]; const parsed: unknown = JSON.parse(fenced ?? raw); const result = judgeResultSchema.parse(parsed); const normalized = { ...result, total: Math.round(Object.values(result.categories).reduce((sum, score) => sum + score, 0) / 9), }; return { ...normalized, verdict: judgePasses(normalized) ? "pass" : "fail" }; } async function writeWidgetFiles(directory: string, widget: HomarrCustomWidgetV2, basename: string) { const { template, ...manifest } = widget; await Promise.all([ writeFile( path.join(directory, `${basename}.widget.json`), JSON.stringify({ ...manifest, template: "__HOMARR_TEMPLATE__" }, null, 2), "utf8", ), writeFile(path.join(directory, `${basename}.widget.jsx`), template, "utf8"), ]); } async function callOpenRouter(args: { apiKey: string; model: string; prompt: string; json: boolean }): Promise { const response = await fetch("https://openrouter.ai/api/v1/chat/completions", { method: "POST", headers: { Authorization: `Bearer ${args.apiKey}`, "Content-Type": "application/json", "HTTP-Referer": "https://homarr.dev", "X-Title": "Homarr Custom Widget AI Evaluation", }, body: JSON.stringify({ model: args.model, messages: [{ role: "user", content: args.prompt }], temperature: args.json ? 0 : 0.25, max_tokens: args.json ? 8_000 : 20_000, ...(args.json ? { reasoning: { effort: "low", exclude: true } } : {}), ...(args.json ? { response_format: { type: "json_object" } } : {}), }), signal: AbortSignal.timeout(180_000), }); const payload = (await response.json()) as OpenRouterResponse; if (!response.ok) throw new Error(`OpenRouter request failed (${response.status}): ${payload.error?.message ?? "Unknown error"}`); const content = payload.choices?.[0]?.message?.content; if (!content) throw new Error("OpenRouter returned no message content"); return content; }