Simplify Custom JSX v2 authoring

This commit is contained in:
Thomas Camlong committed 2026-07-21 08:19:22 +02:00
1 parent 1faaf29c3b
commit 05dbad8069
177 files changed
+4775 -43426

No files matched your search

@@ -0,0 +1,49 @@
export interface CustomWidgetAiEvaluationCase {
id: string;
request: string;
documentationUrl: string;
apiNotes: string;
}
export const CUSTOM_WIDGET_AI_EVALUATION_CASES: readonly CustomWidgetAiEvaluationCase[] = [
{
id: "pokedex",
documentationUrl: "https://pokeapi.co/docs/v2",
request:
"Create a polished Pokédex browser with searchable Pokémon, a responsive result grid, manual detail loading, sprites, types, abilities, base-stat progress bars, and clear loading, empty, and failure states.",
apiNotes:
"Use GET /api/v2/pokemon?limit=<option>&offset=<option> for the list and GET /api/v2/pokemon/{param:name} as a manual detail query. PokeAPI needs no authentication.",
},
{
id: "portainer-containers",
documentationUrl: "https://docs.portainer.io/api/examples",
request:
"Create an excellent Portainer container dashboard: environment option, running/stopped summary, responsive container list, health/status badges, and explicit start, stop, and restart actions with confirmation and refresh after success.",
apiNotes:
"Use X-API-Key auth. List containers with GET /api/endpoints/{option:endpointId}/docker/containers/json?all=true. Actions are POST /api/endpoints/{option:endpointId}/docker/containers/{param:id}/start, /stop, and /restart and invalidate the list query.",
},
{
id: "tautulli-activity",
documentationUrl: "https://github.com/Tautulli/Tautulli/wiki/Tautulli-API-Reference",
request:
"Create a beautiful Tautulli activity widget with active stream cards, user/player details, progress, transcode/direct-play badges, bandwidth summary, and a refresh control that works in narrow and wide tiles.",
apiNotes:
"Use an apiKeyQuery source with parameter name apikey. Query GET /api/v2 with query cmd=get_activity. The response payload is under response.data and sessions is an array.",
},
{
id: "bambubuddy-printer",
documentationUrl: "https://wiki.bambuddy.cool/reference/api/",
request:
"Create a premium BambuBuddy printer status widget with printer selector, current job, progress, remaining time, nozzle and bed temperatures, connection state, and safe pause/resume/stop controls when supported.",
apiNotes:
"Use X-API-Key auth and base path /api/v1. GET /printers lists printers and GET /printers/{id}/status returns state, progress, remaining_time, temperatures.nozzle, temperatures.bed, and hms_status. The official reference does not document pause, resume, or stop endpoints, so omit those actions. Configure the selected printer with choicesFrom on a widget option and use that option in the status path.",
},
{
id: "home-assistant-control",
documentationUrl: "https://developers.home-assistant.io/docs/api/rest/",
request:
"Create a refined Home Assistant room widget with temperature and humidity readings, light status, a room/entity option, and an actionable light toggle. Use calm hierarchy and responsive controls, not a pile of nested cards.",
apiNotes:
"Use bearer auth. GET /api/states/{option:sensorEntity} and /api/states/{option:lightEntity} load entity state. POST /api/services/light/turn_on and /turn_off accept a body with entity_id from an option and should invalidate the light query.",
},
] as const;
@@ -0,0 +1,235 @@
import { mkdir, writeFile } from "node:fs/promises";
import path from "node:path";
import { z } from "zod/v4";
import { buildCustomWidgetAiPrompt } from "../src/core/ai-prompt";
import { customWidgetDefinitionSchema } from "../src/core/custom-jsx-schema";
import type { HomarrCustomWidgetV2 } from "../src/core/custom-jsx-schema";
import { formatCustomWidgetImportIssues, parseCustomWidgetAiResponse } from "../src/core/import";
import type { CustomWidgetAiEvaluationCase } from "./ai-evaluation-cases";
export const DEFAULT_GENERATOR_MODEL = "deepseek/deepseek-v4-pro";
export const DEFAULT_JUDGE_MODEL = "deepseek/deepseek-v4-flash";
export const MAX_AI_EVALUATION_LOOPS = 10;
const scoreSchema = z.number().int().min(0).max(100);
const judgeResultSchema = z.strictObject({
total: scoreSchema,
verdict: z.enum(["pass", "fail"]),
categories: z.strictObject({
schemaAndBindings: scoreSchema,
apiAndRequestDesign: scoreSchema,
runtimeCompatibility: scoreSchema,
visualHierarchy: scoreSchema,
responsiveAndTheme: scoreSchema,
loadingEmptyErrorSuccess: scoreSchema,
accessibility: scoreSchema,
actionSafety: scoreSchema,
simplicity: scoreSchema,
}),
strengths: z.array(z.string()).max(8),
problems: z.array(z.string()).max(12),
highestImpactFixes: z.array(z.string()).max(8),
});
export type CustomWidgetJudgeResult = z.infer<typeof judgeResultSchema>;
export interface AiEvaluationResult {
caseId: string;
attempts: number;
widget: HomarrCustomWidgetV2 | null;
judge: CustomWidgetJudgeResult | null;
outputDirectory: string;
errors: string[];
}
interface OpenRouterResponse {
choices?: Array<{ message?: { content?: string | null } }>;
error?: { message?: string };
}
export function buildEvaluationPrompt(testCase: CustomWidgetAiEvaluationCase): string {
return buildCustomWidgetAiPrompt(
undefined,
null,
null,
`${testCase.request}\n\nVerified API notes:\n${testCase.apiNotes}`,
testCase.documentationUrl,
);
}
export function buildRepairPrompt(
originalPrompt: string,
previousResponse: string,
issues: readonly { path?: Array<string | number>; message: string }[],
): string {
const diagnostics = issues
.map((issue) => `${issue.path?.length ? `${issue.path.join(".")}: ` : ""}${issue.message}`)
.join("\n");
return `${originalPrompt}\n\nYour previous response did not validate. Correct only the generic contract/runtime problems below while preserving a polished design.\n\nDiagnostics:\n${diagnostics}\n\nPrevious response:\n${previousResponse}`;
}
export function buildJudgePrompt(testCase: CustomWidgetAiEvaluationCase, widget: HomarrCustomWidgetV2): string {
return `You are a strict evaluator for a safe dashboard-widget language. Judge the widget against the request and API notes. Do not reward unsupported capabilities, invented API routes, or visual claims not present in the JSX.
Request:
${testCase.request}
Verified API notes:
${testCase.apiNotes}
Validated widget:
${JSON.stringify(widget, null, 2)}
Score every category from 0 to 100. The total must reflect the whole result. A pass requires total >= 80 and no category below 65. Return JSON only with this exact shape:
{"total":0,"verdict":"fail","categories":{"schemaAndBindings":0,"apiAndRequestDesign":0,"runtimeCompatibility":0,"visualHierarchy":0,"responsiveAndTheme":0,"loadingEmptyErrorSuccess":0,"accessibility":0,"actionSafety":0,"simplicity":0},"strengths":[],"problems":[],"highestImpactFixes":[]}`;
}
export function judgePasses(result: CustomWidgetJudgeResult): boolean {
return result.total >= 80 && Object.values(result.categories).every((score) => score >= 65);
}
export async function evaluateCustomWidgetCase(args: {
testCase: CustomWidgetAiEvaluationCase;
apiKey: string;
outputRoot: string;
maxLoops: number;
generatorModel?: string;
judgeModel?: string;
}): Promise<AiEvaluationResult> {
const caseDirectory = path.join(args.outputRoot, args.testCase.id);
await mkdir(caseDirectory, { recursive: true });
const originalPrompt = buildEvaluationPrompt(args.testCase);
await writeFile(path.join(caseDirectory, "prompt.md"), originalPrompt, "utf8");
let prompt = originalPrompt;
const errors: string[] = [];
let bestWidget: HomarrCustomWidgetV2 | null = null;
let bestJudge: CustomWidgetJudgeResult | null = null;
for (let attempt = 1; attempt <= Math.min(args.maxLoops, MAX_AI_EVALUATION_LOOPS); attempt += 1) {
let response: string;
try {
response = await callOpenRouter({
apiKey: args.apiKey,
model: args.generatorModel ?? DEFAULT_GENERATOR_MODEL,
prompt,
json: false,
});
} catch (error) {
errors.push(
`Attempt ${attempt}: generator request failed — ${error instanceof Error ? error.message : "Unknown error"}`,
);
continue;
}
await writeFile(path.join(caseDirectory, `attempt-${attempt}.md`), response, "utf8");
const parsed = parseCustomWidgetAiResponse(response);
if (!parsed.success) {
const message = formatCustomWidgetImportIssues(parsed.issues);
errors.push(`Attempt ${attempt}: ${message}`);
prompt = buildRepairPrompt(originalPrompt, response, parsed.issues);
continue;
}
const canonical = customWidgetDefinitionSchema.parse(parsed.widget);
await writeWidgetFiles(caseDirectory, canonical, `attempt-${attempt}`);
let judge: CustomWidgetJudgeResult;
try {
const judgeRaw = await callOpenRouter({
apiKey: args.apiKey,
model: args.judgeModel ?? DEFAULT_JUDGE_MODEL,
prompt: buildJudgePrompt(args.testCase, canonical),
json: true,
});
await writeFile(path.join(caseDirectory, `judge-${attempt}.json`), judgeRaw, "utf8");
judge = parseJudgeResult(judgeRaw);
} catch (error) {
errors.push(
`Attempt ${attempt}: judge response invalid — ${error instanceof Error ? error.message : "Unknown error"}`,
);
continue;
}
if (!bestJudge || judge.total > bestJudge.total) {
bestJudge = judge;
bestWidget = canonical;
await writeWidgetFiles(caseDirectory, canonical, "best");
await writeFile(path.join(caseDirectory, "best-judge.json"), JSON.stringify(judge, null, 2), "utf8");
}
if (judgePasses(judge)) {
await writeFile(path.join(caseDirectory, "result.json"), JSON.stringify(judge, null, 2), "utf8");
return {
caseId: args.testCase.id,
attempts: attempt,
widget: canonical,
judge,
outputDirectory: caseDirectory,
errors,
};
}
errors.push(`Attempt ${attempt}: judge ${judge.total}/100 — ${judge.highestImpactFixes.join("; ")}`);
prompt = buildRepairPrompt(
originalPrompt,
response,
judge.highestImpactFixes.map((message) => ({ message })),
);
}
return {
caseId: args.testCase.id,
attempts: Math.min(args.maxLoops, MAX_AI_EVALUATION_LOOPS),
widget: bestWidget,
judge: bestJudge,
outputDirectory: caseDirectory,
errors,
};
}
export function parseJudgeResult(raw: string): CustomWidgetJudgeResult {
const fenced = /```(?:json)?\s*([\s\S]*?)```/iu.exec(raw)?.[1];
const parsed: unknown = JSON.parse(fenced ?? raw);
const result = judgeResultSchema.parse(parsed);
const normalized = {
...result,
total: Math.round(Object.values(result.categories).reduce((sum, score) => sum + score, 0) / 9),
};
return { ...normalized, verdict: judgePasses(normalized) ? "pass" : "fail" };
}
async function writeWidgetFiles(directory: string, widget: HomarrCustomWidgetV2, basename: string) {
const { template, ...manifest } = widget;
await Promise.all([
writeFile(
path.join(directory, `${basename}.widget.json`),
JSON.stringify({ ...manifest, template: "__HOMARR_TEMPLATE__" }, null, 2),
"utf8",
),
writeFile(path.join(directory, `${basename}.widget.jsx`), template, "utf8"),
]);
}
async function callOpenRouter(args: { apiKey: string; model: string; prompt: string; json: boolean }): Promise<string> {
const response = await fetch("https://openrouter.ai/api/v1/chat/completions", {
method: "POST",
headers: {
Authorization: `Bearer ${args.apiKey}`,
"Content-Type": "application/json",
"HTTP-Referer": "https://homarr.dev",
"X-Title": "Homarr Custom Widget AI Evaluation",
},
body: JSON.stringify({
model: args.model,
messages: [{ role: "user", content: args.prompt }],
temperature: args.json ? 0 : 0.25,
max_tokens: args.json ? 8_000 : 20_000,
...(args.json ? { reasoning: { effort: "low", exclude: true } } : {}),
...(args.json ? { response_format: { type: "json_object" } } : {}),
}),
signal: AbortSignal.timeout(180_000),
});
const payload = (await response.json()) as OpenRouterResponse;
if (!response.ok)
throw new Error(`OpenRouter request failed (${response.status}): ${payload.error?.message ?? "Unknown error"}`);
const content = payload.choices?.[0]?.message?.content;
if (!content) throw new Error("OpenRouter returned no message content");
return content;
}
@@ -1,326 +0,0 @@
import type {
CustomJsxAuthoringCatalog,
CustomJsxComponentApi,
CustomJsxLiteralValue,
CustomJsxPropDescriptor,
} from "../src/core/component-catalog-types";
import type { CustomJsxComponentPackage } from "../src/core/component-types";
export const GENERATED_COMPONENT_REFERENCE_PATHS = ["components.md", "homarr-components.md"] as const;
export type GeneratedComponentReferencePath = (typeof GENERATED_COMPONENT_REFERENCE_PATHS)[number];
const packageReferences = [
{
packageName: "@mantine/core",
path: "mantine-core.md",
title: "Mantine Core",
},
{
packageName: "@mantine/dates",
path: "mantine-dates.md",
title: "Mantine Dates",
},
{
packageName: "@mantine/charts",
path: "mantine-charts.md",
title: "Mantine Charts",
},
{
packageName: "@homarr/widgets",
path: "homarr-components.md",
title: "Homarr components",
},
] as const satisfies readonly {
packageName: CustomJsxComponentPackage;
path: string;
title: string;
}[];
export function renderCustomWidgetComponentReferences(
catalog: Readonly<CustomJsxAuthoringCatalog>,
): Readonly<Record<GeneratedComponentReferencePath, string>> {
validateCatalogReferences(catalog);
const references = Object.fromEntries([
["components.md", renderCatalogOverview(catalog)],
["homarr-components.md", renderPackageReference(catalog, "@homarr/widgets", "Homarr components")],
]) as Record<GeneratedComponentReferencePath, string>;
return references;
}
export function getComponentReferenceAnchor(name: string): string {
return `component-${[...name]
.map((character) =>
/^[a-z0-9-]$/iu.test(character) ? character.toLowerCase() : `-${character.codePointAt(0)?.toString(16)}-`,
)
.join("")}`;
}
function renderCatalogOverview(catalog: Readonly<CustomJsxAuthoringCatalog>): string {
const lines = [
"# Component catalog",
"",
generatedNotice(catalog),
"",
"This is the complete Custom JSX component boundary for this Homarr release. The machine-readable `component-catalog.json` contains exact generated component, prop, binding, and safety metadata; these Markdown pages provide the human-readable inventory and upstream Mantine indexes. Ordinary serializable props pass through unless the capability boundary explicitly blocks them.",
"",
"## Package references",
"",
...packageReferences.map(({ packageName, path, title }) => {
const components = catalog.components.filter((component) => component.package === packageName);
const enabledCount = components.filter((component) => component.safety !== "denied").length;
const deniedCount = components.length - enabledCount;
return `- [${title}](${path}) — ${enabledCount} enabled, ${deniedCount} denied`;
}),
"",
"## Authoring boundary",
"",
"- Use the exact component names below. Compound names such as `Tabs.List`, `Card.Section`, and `Radio.Group` are distinct catalog entries.",
"- Use the official Mantine LLM links for component props and examples. Homarr's live catalog is authoritative for availability, bindings, and blocked capabilities.",
'- Use `bind="name"` only when a component record declares bind metadata. Read the temporary in-memory value from `inputs.name`; it is never persisted.',
"- Ordinary authored callbacks remain blocked. Restricted collection callbacks, directly invoked zero-argument derived-value IIFEs, and `RecursiveList`'s child template may use the immutable local `const` grammar in [JSX runtime](runtime.md#immutable-local-bindings).",
`- Use [\`RecursiveList\`](homarr-components.md#${getComponentReferenceAnchor("RecursiveList")}) for custom arbitrary-depth trees. Do not author recursion or pass a renderer callback to Mantine Tree.`,
"- Unknown ordinary Mantine props are advisory and are not silently discarded. Every capability listed in the blocked table is rejected.",
"",
"## Complete enabled inventory",
"",
...renderInventoryByPackage(catalog, false),
"",
"## Complete denied inventory",
"",
...renderInventoryByPackage(catalog, true),
"",
"## Blocked capabilities",
"",
"| Kind | Name | Reason |",
"| --- | --- | --- |",
...catalog.blockedCapabilities.map(
(capability) =>
`| ${escapeTableCell(capability.kind)} | ${renderCode(capability.name)} | ${escapeTableCell(capability.reason)} |`,
),
"",
];
return `${lines.join("\n").trimEnd()}\n`;
}
function renderPackageReference(
catalog: Readonly<CustomJsxAuthoringCatalog>,
packageName: CustomJsxComponentPackage,
title: string,
): string {
const components = catalog.components.filter((component) => component.package === packageName);
const enabled = components.filter((component) => component.safety !== "denied");
const denied = components.filter((component) => component.safety === "denied");
const lines = [
`# ${title}`,
"",
generatedNotice(catalog),
"",
`Package: \`${packageName}\`. The component-specific props below supplement ordinary safe serializable Mantine props and style props. Component-specific records override ordinary behavior with the same name.`,
"",
...packageGuidance(packageName),
"",
"## Enabled inventory",
"",
enabled.length > 0 ? renderCodeList(enabled.map(({ name }) => name)) : "_None._",
"",
"## Denied inventory",
"",
denied.length > 0 ? renderCodeList(denied.map(({ name }) => name)) : "_None._",
"",
"## Component records",
"",
...components.flatMap((component) => renderComponentRecord(catalog, component)),
];
return `${lines.join("\n").trimEnd()}\n`;
}
function renderComponentRecord(
catalog: Readonly<CustomJsxAuthoringCatalog>,
component: Readonly<CustomJsxComponentApi>,
): string[] {
const details = [
`<a id="${getComponentReferenceAnchor(component.name)}"></a>`,
`### \`${component.name}\``,
"",
`- Safety: \`${component.safety}\``,
`- Category: \`${component.category}\``,
`- Documentation: [upstream component API](${component.documentationUrl})`,
`- Temporary bind: ${renderBinding(component)}`,
`- Subcomponents: ${component.subcomponents.length > 0 ? renderCodeList(component.subcomponents) : "none"}`,
`- Component-specific blocked props: ${
component.blockedProps.length > 0
? component.blockedProps
.map(({ name, reason }) => `${renderCode(name)} — ${escapeTableCell(reason)}`)
.join(", ")
: "none"
}`,
"- Accessibility:",
...(component.accessibilityRequirements.length > 0
? component.accessibilityRequirements.map((requirement) => ` - ${requirement}`)
: [" - No additional catalog requirement; preserve semantic labels and text alternatives."]),
];
if (component.description) details.push("", component.description);
if (component.deniedReason) details.push("", `Denied reason: ${component.deniedReason}.`);
details.push("", "#### Component-specific props", "");
if (component.props.length === 0) {
details.push(
component.safety === "denied"
? "_No authorable props because this component is denied._"
: "_No component-specific props. Use the shared prop API._",
);
} else {
details.push(renderPropTable(catalog, component.props));
}
details.push("");
return details;
}
function renderPropTable(
catalog: Readonly<CustomJsxAuthoringCatalog>,
props: readonly Readonly<CustomJsxPropDescriptor>[],
): string {
const lines = [
"| Prop | Required | Type | Literal values | Description |",
"| --- | --- | --- | --- | --- |",
...props.map((prop) => {
const type = catalog.types[prop.typeRef];
if (type === undefined) throw new Error(`Prop '${prop.name}' references missing type ${prop.typeRef}`);
return `| ${renderCode(prop.name)} | ${prop.required ? "yes" : "no"} | ${renderCode(type)} | ${renderLiteralValues(prop.literalValues)} | ${escapeTableCell(prop.description ?? "—")} |`;
}),
];
return lines.join("\n");
}
function renderInventoryByPackage(catalog: Readonly<CustomJsxAuthoringCatalog>, denied: boolean): string[] {
return packageReferences.flatMap(({ packageName, path, title }) => {
const components = catalog.components.filter(
(component) => component.package === packageName && (component.safety === "denied") === denied,
);
return [
`### ${title}`,
"",
components.length > 0
? components
.map((component) => {
const target =
component.package === "@homarr/widgets"
? `${path}#${getComponentReferenceAnchor(component.name)}`
: component.documentationUrl;
return `[\`${escapeMarkdownLinkLabel(component.name)}\`](${target})${component.deniedReason ? ` — ${component.deniedReason}` : ""}`;
})
.join(", ")
: "_None._",
"",
];
});
}
function packageGuidance(packageName: CustomJsxComponentPackage): string[] {
switch (packageName) {
case "@mantine/core":
return [
"## Package guidance",
"",
"Use declarative props and `bind` adapters instead of event callbacks. Homarr scopes supported overlays to the widget root. Use Homarr's `RecursiveList` when a tree needs a custom arbitrary-depth node template.",
];
case "@mantine/dates":
return [
"## Package guidance",
"",
"Date bindings are temporary serialized inputs. Persisted locale, time-zone, and date choices belong in widget options. Include a textual date or agenda when a visual date control carries meaning.",
];
case "@mantine/charts":
return [
"## Package guidance",
"",
"Pass sanitized JSON data through serializable props. Use [immutable local `const` blocks](runtime.md#immutable-local-bindings) for repeated derived chart values; authored formatters and renderer callbacks remain blocked. Include a textual summary and do not rely on color alone.",
];
case "@homarr/widgets":
return [
"## Package guidance",
"",
"Homarr components own controlled request, action, pagination, disclosure, icon, and recursive traversal behavior. Manual request parameters come from the invoking component's `params` prop. `RecursiveList` is the only supported arbitrary-depth authored renderer and its child template may use the restricted safe-block grammar.",
];
}
}
function renderBinding(component: Readonly<CustomJsxComponentApi>): string {
if (!component.bind) return "not available";
const conditionalType = getConditionalBindingType(component.name);
return `\`${component.bind.type}\` via \`${component.bind.initialProp}\`; read from \`inputs.<bindName>\`${conditionalType ? `; ${conditionalType}` : ""}`;
}
function getConditionalBindingType(componentName: string): string | null {
if (componentName === "Accordion") return "becomes `string[]` when `multiple` is true";
if (["Chip.Group", "ChipGroup"].includes(componentName)) return "becomes `string[]` when `multiple` is true";
if (componentName === "TreeSelect") return "becomes `string[]` when `mode` is `multiple` or `checkbox`";
if (
["DatePicker", "DatePickerInput", "MonthPicker", "MonthPickerInput", "YearPicker", "YearPickerInput"].includes(
componentName,
)
) {
return "becomes `string[]` when `type` is `multiple` or `range`";
}
if (["DateTimePicker", "InlineDateTimePicker"].includes(componentName))
return "becomes `string[]` when `type` is `range`; `multiple` is not supported";
return null;
}
function renderLiteralValues(values: readonly CustomJsxLiteralValue[] | undefined): string {
return values && values.length > 0
? values.map((value) => renderCode(JSON.stringify(value) ?? "null")).join(", ")
: "—";
}
function renderCodeList(values: readonly string[]): string {
return values.map(renderCode).join(", ");
}
function renderCode(value: string): string {
return `<code>${escapeHtml(value)}</code>`;
}
function escapeTableCell(value: string): string {
return escapeHtml(value).replaceAll("|", "&#124;");
}
function escapeHtml(value: string): string {
return value.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll("|", "&#124;");
}
function escapeMarkdownLinkLabel(value: string): string {
return value.replaceAll("[", "\\[").replaceAll("]", "\\]");
}
function generatedNotice(catalog: Readonly<CustomJsxAuthoringCatalog>): string {
return `> Generated from \`component-catalog-v${catalog.schemaVersion}.json\`. Catalog schema v${catalog.schemaVersion} · Custom Widget ${catalog.customWidgetVersion} · Mantine ${catalog.mantineVersion}. Do not edit this file by hand.`;
}
function validateCatalogReferences(catalog: Readonly<CustomJsxAuthoringCatalog>) {
const componentNames = catalog.components.map(({ name }) => name);
if (new Set(componentNames).size !== componentNames.length) throw new Error("Catalog component names must be unique");
for (const component of catalog.components) {
if (!packageReferences.some(({ packageName }) => packageName === component.package)) {
throw new Error(`Catalog component '${component.name}' uses unsupported package '${component.package}'`);
}
for (const prop of component.props) assertPropType(catalog, prop, `${component.name}.${prop.name}`);
}
for (const prop of catalog.globalProps) assertPropType(catalog, prop, prop.name);
}
function assertPropType(
catalog: Readonly<CustomJsxAuthoringCatalog>,
prop: Readonly<CustomJsxPropDescriptor>,
label: string,
) {
if (!Number.isSafeInteger(prop.typeRef) || catalog.types[prop.typeRef] === undefined) {
throw new Error(`Catalog prop '${label}' references missing type ${prop.typeRef}`);
}
}
@@ -0,0 +1,66 @@
import { mkdir, writeFile } from "node:fs/promises";
import path from "node:path";
import { CUSTOM_WIDGET_AI_EVALUATION_CASES } from "./ai-evaluation-cases";
import { DEFAULT_GENERATOR_MODEL, DEFAULT_JUDGE_MODEL, evaluateCustomWidgetCase } from "./ai-evaluation";
import type { AiEvaluationResult } from "./ai-evaluation";
const apiKey = process.env.OPENROUTER_API_KEY;
if (!apiKey) throw new Error("OPENROUTER_API_KEY is required for the live Custom Widget AI evaluation");
const requestedCase = process.argv.find((value) => value.startsWith("--case="))?.slice("--case=".length);
const configuredLoops = Number(process.env.CUSTOM_WIDGET_AI_MAX_LOOPS ?? 10);
const totalLoopBudget = Number.isInteger(configuredLoops) && configuredLoops > 0 ? Math.min(configuredLoops, 10) : 10;
const selectedCases = requestedCase
? CUSTOM_WIDGET_AI_EVALUATION_CASES.filter((testCase) => testCase.id === requestedCase)
: CUSTOM_WIDGET_AI_EVALUATION_CASES;
if (selectedCases.length === 0) throw new Error(`Unknown AI evaluation case '${requestedCase}'`);
if (totalLoopBudget < selectedCases.length) {
throw new Error(`CUSTOM_WIDGET_AI_MAX_LOOPS must be at least ${selectedCases.length} for this run`);
}
const runId = new Date().toISOString().replaceAll(/[:.]/gu, "-");
const outputRoot = path.resolve(process.cwd(), ".ai-evaluations", runId);
await mkdir(outputRoot, { recursive: true });
const results: AiEvaluationResult[] = [];
let remainingLoopBudget = totalLoopBudget;
for (const testCase of selectedCases) {
const remainingCases = selectedCases.length - results.length;
const maxLoops = Math.max(1, Math.floor(remainingLoopBudget / remainingCases));
process.stdout.write(`Evaluating ${testCase.id}...\n`);
const result = await evaluateCustomWidgetCase({
testCase,
apiKey,
outputRoot,
maxLoops,
generatorModel: process.env.OPENROUTER_GENERATOR_MODEL ?? DEFAULT_GENERATOR_MODEL,
judgeModel: process.env.OPENROUTER_JUDGE_MODEL ?? DEFAULT_JUDGE_MODEL,
});
results.push(result);
remainingLoopBudget -= result.attempts;
process.stdout.write(
result.judge
? ` ${result.judge.total}/100 after ${result.attempts} attempt(s)\n`
: ` failed after ${result.attempts} attempt(s)\n`,
);
}
const summary = {
generatedAt: new Date().toISOString(),
generatorModel: process.env.OPENROUTER_GENERATOR_MODEL ?? DEFAULT_GENERATOR_MODEL,
judgeModel: process.env.OPENROUTER_JUDGE_MODEL ?? DEFAULT_JUDGE_MODEL,
results: results.map((result) => ({
caseId: result.caseId,
attempts: result.attempts,
score: result.judge?.total ?? null,
verdict: result.judge?.verdict ?? "fail",
categories: result.judge?.categories ?? null,
errors: result.errors,
outputDirectory: path.relative(process.cwd(), result.outputDirectory),
})),
};
await writeFile(path.join(outputRoot, "summary.json"), JSON.stringify(summary, null, 2), "utf8");
process.stdout.write(`Results saved to ${path.relative(process.cwd(), outputRoot)}\n`);
if (results.some((result) => result.judge?.verdict !== "pass")) process.exitCode = 1;
@@ -1,20 +0,0 @@
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { resolve } from "node:path";
import type { CustomJsxAuthoringCatalog } from "../src/core/component-catalog-types";
import { renderCustomWidgetComponentReferences } from "./component-docs";
const packageRoot = resolve(import.meta.dirname, "..");
const sourcePath = resolve(packageRoot, "src/core/component-catalog.generated.json");
const publicDirectory = resolve(packageRoot, "../../apps/docs/static/custom-widgets");
const publicPath = resolve(publicDirectory, "component-catalog-v1.json");
const skillReferenceDirectory = resolve(packageRoot, "../../.agents/skills/homarr-custom-widget/references");
const contents = await readFile(sourcePath, "utf8");
const catalog = JSON.parse(contents) as CustomJsxAuthoringCatalog;
const references = renderCustomWidgetComponentReferences(catalog);
await Promise.all([mkdir(publicDirectory, { recursive: true }), mkdir(skillReferenceDirectory, { recursive: true })]);
await Promise.all([
writeFile(publicPath, contents),
...Object.entries(references).map(([path, content]) => writeFile(resolve(skillReferenceDirectory, path), content)),
]);
@@ -151,7 +151,11 @@ async function generateCatalog() {
components,
};
await writeFile(resolve(packageRoot, "src/core/component-catalog.generated.json"), `${JSON.stringify(catalog)}\n`);
const serialized = `${JSON.stringify(catalog)}\n`;
await Promise.all([
writeFile(resolve(packageRoot, "src/core/component-catalog.generated.json"), serialized),
writeFile(resolve(repositoryRoot, "apps/docs/static/custom-widgets/component-catalog-v1.json"), serialized),
]);
}
function inspectMantineComponent(component: CatalogSourceComponent): InspectedComponent {
@@ -506,7 +510,6 @@ const HOMARR_COMPONENT_DEFINITIONS = [
{ name: "Collapsible", category: "interaction", safety: "allowed" },
{ name: "StatBar", category: "interaction", safety: "allowed" },
{ name: "TypeBadge", category: "interaction", safety: "allowed" },
{ name: "RecursiveList", category: "content", safety: "wrapped" },
{ name: "SubFetch", category: "network", safety: "wrapped" },
{ name: "SubData", category: "network", safety: "wrapped" },
{ name: "ActionButton", category: "network", safety: "wrapped" },
@@ -580,7 +583,6 @@ const HOMARR_COMPONENT_DESCRIPTIONS: Readonly<Record<string, string>> = {
Collapsible: "Shows and hides a bounded content section.",
StatBar: "Displays a compact labeled statistic and progress value.",
TypeBadge: "Displays a compact semantic type badge.",
RecursiveList: "Renders an arbitrary-depth tree through a bounded, Homarr-owned traversal.",
SubFetch: "Runs a named manual query with invocation-time parameters.",
SubData: "Reads a nested value from a named manual-query result.",
ActionButton: "Runs a named user-triggered action.",
@@ -627,17 +629,6 @@ const HOMARR_COMPONENT_PROPS: Readonly<Record<string, Readonly<Record<string, Ho
type: requiredProp("string"),
size: optionalProp("'xs' | 'sm' | 'md' | 'lg' | 'xl'", ["xs", "sm", "md", "lg", "xl"]),
},
RecursiveList: {
data: requiredProp("unknown"),
childrenPath: requiredProp("string"),
keyPath: requiredProp("string"),
maxDepth: optionalProp("number"),
maxNodes: optionalProp("number"),
defaultExpandedDepth: optionalProp("number"),
indent: optionalProp("MantineSpacing"),
gap: optionalProp("MantineSpacing"),
showLines: optionalProp("boolean"),
},
SubFetch: {
requestId: optionalProp("string", undefined, "Identifier of the named Custom Widget query."),
params: optionalProp("Record<string, string | number | boolean>"),
@@ -1,49 +0,0 @@
import { readFile, writeFile } from "node:fs/promises";
import { resolve } from "node:path";
import { customJsxAuthoringCatalog } from "../src/core/component-catalog";
import { customJsxExamples } from "../src/core/examples";
import { getCustomWidgetJsonSchema } from "../src/core/schema";
export const CUSTOM_WIDGET_SKILL_SOURCE_PATHS = [
"SKILL.md",
"references/schema.md",
"references/widget-schema.json",
"references/runtime.md",
"references/security.md",
"references/components.md",
"references/component-catalog.json",
"references/examples.json",
"references/mantine-core.md",
"references/mantine-dates.md",
"references/mantine-charts.md",
"references/homarr-components.md",
] as const;
const packageRoot = resolve(import.meta.dirname, "..");
const repositoryRoot = resolve(packageRoot, "../..");
const skillRoot = resolve(repositoryRoot, ".agents/skills/homarr-custom-widget");
const outputPath = resolve(packageRoot, "src/core/skill-content.generated.json");
await writeFile(
resolve(skillRoot, "references/widget-schema.json"),
`${JSON.stringify(getCustomWidgetJsonSchema(), null, 2)}\n`,
);
await writeFile(
resolve(skillRoot, "references/component-catalog.json"),
`${JSON.stringify(customJsxAuthoringCatalog)}\n`,
);
await writeFile(resolve(skillRoot, "references/examples.json"), `${JSON.stringify(customJsxExamples, null, 2)}\n`);
const files = await Promise.all(
CUSTOM_WIDGET_SKILL_SOURCE_PATHS.map(async (path) => ({
path,
content: normalizeMarkdown(await readFile(resolve(skillRoot, path), "utf8")),
})),
);
await writeFile(outputPath, `${JSON.stringify({ schemaVersion: 1, files }, null, 2)}\n`);
function normalizeMarkdown(value: string) {
return `${value.replace(/\r\n?/gu, "\n").trimEnd()}\n`;
}
@@ -0,0 +1,138 @@
import { customJsxRequestSchema } from "../src/core/request-schema";
import { executeCustomWidgetRequest } from "../src/server/request-executor";
import { renderRequestTarget, resolveCustomWidgetRequestValues } from "../src/server/request-manifest";
const pokemonListRequest = customJsxRequestSchema.parse({
path: "/api/v2/pokemon",
query: { limit: { $option: "limit" }, offset: 0 },
});
const pokemonDetailRequest = customJsxRequestSchema.parse({
path: "/api/v2/pokemon/{param:name}",
trigger: "manual",
});
const listTarget = renderRequestTarget(
"https://pokeapi.co",
pokemonListRequest,
resolveCustomWidgetRequestValues(pokemonListRequest, { limit: 3 }),
);
const list = await fetchJson(listTarget, {});
if (!isRecord(list) || !Array.isArray(list.results) || list.results.length !== 3) {
throw new Error("PokeAPI list response did not match the expected shape");
}
const firstName = isRecord(list.results[0]) && typeof list.results[0].name === "string" ? list.results[0].name : null;
if (!firstName) throw new Error("PokeAPI list did not contain a Pokémon name");
const detailTarget = renderRequestTarget(
"https://pokeapi.co",
pokemonDetailRequest,
resolveCustomWidgetRequestValues(pokemonDetailRequest, {}, { name: firstName }),
);
const detail = await fetchJson(detailTarget, {});
if (!isRecord(detail) || detail.name !== firstName || !Array.isArray(detail.stats)) {
throw new Error("PokeAPI detail response did not match the expected shape");
}
process.stdout.write("PokeAPI list and detail requests passed\n");
const portainerBaseUrl = process.env.PORTAINER_BASE_URL;
const portainerApiKey = process.env.PORTAINER_API_KEY;
if (!portainerBaseUrl || !portainerApiKey) {
if (process.argv.includes("--pokeapi-only")) {
process.stdout.write("Portainer check skipped (--pokeapi-only)\n");
process.exit(0);
}
throw new Error("PORTAINER_BASE_URL and PORTAINER_API_KEY are required for the Portainer reference check");
}
const endpointId = process.env.PORTAINER_ENDPOINT_ID ?? "2";
const containersRequest = customJsxRequestSchema.parse({
path: "/api/endpoints/{option:endpointId}/docker/containers/json",
query: { all: true },
});
const containersTarget = renderRequestTarget(
portainerBaseUrl,
containersRequest,
resolveCustomWidgetRequestValues(containersRequest, { endpointId }),
);
const containers = await fetchJson(containersTarget, { "X-API-Key": portainerApiKey });
if (!Array.isArray(containers)) throw new Error("Portainer containers response was not an array");
const actionRequests = ["start", "stop", "restart"].map((action) =>
customJsxRequestSchema.parse({
kind: "action",
method: "POST",
path: `/api/endpoints/{option:endpointId}/docker/containers/{param:id}/${action}`,
confirmation: `${action} this container?`,
invalidates: ["containers"],
}),
);
for (const request of actionRequests) {
const target = renderRequestTarget(
portainerBaseUrl,
request,
resolveCustomWidgetRequestValues(request, { endpointId }, { id: "container/id" }),
);
if (!target.pathname.includes("container%2Fid")) throw new Error("Portainer action target did not encode its ID");
}
process.stdout.write(
`Portainer container query passed (${containers.length} containers); start/stop/restart targets passed\n`,
);
const mutationContainerId = process.env.PORTAINER_TEST_CONTAINER_ID;
if (mutationContainerId) {
if (process.env.PORTAINER_ALLOW_MUTATIONS !== "1") {
throw new Error("Set PORTAINER_ALLOW_MUTATIONS=1 to test actions against PORTAINER_TEST_CONTAINER_ID");
}
const container = containers.find(
(candidate) =>
isRecord(candidate) && (candidate.Id === mutationContainerId || candidate.ID === mutationContainerId),
);
if (!isRecord(container)) throw new Error("PORTAINER_TEST_CONTAINER_ID was not returned by the container query");
const initiallyRunning =
container.State === "running" || (isRecord(container.State) && container.State.Running === true);
const actionOrder = initiallyRunning ? ["restart", "stop", "start"] : ["start", "restart", "stop"];
try {
for (const action of actionOrder) await runPortainerAction(action, mutationContainerId);
} finally {
await runPortainerAction(initiallyRunning ? "start" : "stop", mutationContainerId, true);
}
process.stdout.write(
`Portainer ${actionOrder.join("/")} actions passed and the original running state was restored\n`,
);
}
function isRecord(value: unknown): value is Record<string, unknown> {
return value !== null && typeof value === "object" && !Array.isArray(value);
}
async function fetchJson(target: URL, headers: Record<string, string>): Promise<unknown> {
const response = await fetch(target, { headers, signal: AbortSignal.timeout(15_000) });
if (!response.ok) throw new Error(`Reference API returned HTTP ${response.status}`);
return response.json() as Promise<unknown>;
}
async function runPortainerAction(action: string, containerId: string, ignoreConflict = false) {
const request = customJsxRequestSchema.parse({
kind: "action",
method: "POST",
path: `/api/endpoints/{option:endpointId}/docker/containers/{param:id}/${action}`,
});
const target = renderRequestTarget(
portainerBaseUrl,
request,
resolveCustomWidgetRequestValues(request, { endpointId }, { id: containerId }),
);
const response = await executeCustomWidgetRequest({
baseUrl: portainerBaseUrl,
targetUrl: target,
method: "POST",
networkScope: "private",
kind: "action",
auth: {
type: "apiKeyHeader",
headerName: "X-API-Key",
secrets: [{ kind: "apiKey", value: portainerApiKey }],
},
});
if (!response.ok && !(ignoreConflict && response.status === 304)) {
throw new Error(`Portainer ${action} returned HTTP ${response.status}`);
}
}