feat!: release v2 (#6545)
Co-authored-by: Artur <145776561+arturict@users.noreply.github.com> Co-authored-by: arturict <arturict@users.noreply.github.com> Co-authored-by: Thomas <thomas@ajnart.fr> Co-authored-by: Afonso Jorge Ramos <afonsojorgeramos@gmail.com> Co-authored-by: Danura30082 <danura30082+github@gmail.com> Co-authored-by: Kushie <122986913+kushiemoon-dev@users.noreply.github.com> Co-authored-by: ajnart <thomas@ajnart.fr> Co-authored-by: ajnart <thomas@ajnart.dev> Co-authored-by: chenjun <chenjun@foxmail.com> Co-authored-by: Wikibear <wikibear@users.noreply.github.com> Co-authored-by: Bergasha <bergasha@gmail.com> Co-authored-by: leafyelin <43306196+leafyelin@users.noreply.github.com> Co-authored-by: Lucas Araujo <lucas.lra@gmail.com> Co-authored-by: GreenMonito <86085570+GreenMonito@users.noreply.github.com> Co-authored-by: Nerdy-Technician <roffo@nerdy-technician.social> Co-authored-by: Jun <34545896+jcstrong@users.noreply.github.com> Co-authored-by: Edmond Abraham <54119952+gitedmond@users.noreply.github.com> Co-authored-by: thrawnx <tight.bulb9384@fastmail.com> Co-authored-by: Chris <58918656+nordada@users.noreply.github.com> Co-authored-by: yi111 <153097222+Yi-111-a@users.noreply.github.com> Co-authored-by: Yanis <naamaneyanis@gmail.com> Co-authored-by: Wikibear <me@sej.de> Co-authored-by: alessiottavo <93441886+alessiottavo@users.noreply.github.com>
This commit is contained in:
3286 files changed
+284386
-139176
No files matched your search
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,999 @@
|
||||
import { getCustomWidgetSourceAuthType } from "../src/core/request-schema";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
import { z } from "zod/v4";
|
||||
|
||||
import { buildCustomWidgetAiPrompt } from "../src/core/ai-prompt";
|
||||
import { getCustomWidgetSkillReference } from "../src/core/authoring-resources";
|
||||
import { customWidgetDefinitionSchema } from "../src/core/custom-jsx-schema";
|
||||
import type { HomarrCustomWidgetV2 } from "../src/core/custom-jsx-schema";
|
||||
import { formatCustomWidgetImportIssues, parseCustomWidgetAiResponse } from "../src/core/import";
|
||||
import type { CustomWidgetAiEvaluationCase } from "./ai-evaluation-cases";
|
||||
import type { CustomWidgetAiExpectation } from "./ai-evaluation-cases";
|
||||
|
||||
export const DEFAULT_GENERATOR_MODEL = "deepseek/deepseek-v4.1-flash";
|
||||
export const DEFAULT_JUDGE_MODEL = "openai/gpt-5.6-luna";
|
||||
export const DEFAULT_AI_PROVIDER_BASE_URL = "https://openrouter.ai/api/v1";
|
||||
export const MAX_AI_EVALUATION_LOOPS = 10;
|
||||
|
||||
let aiProviderRequestCount = 0;
|
||||
let observedAiProviderSpend = 0;
|
||||
|
||||
const getPositiveLimit = (value: string | undefined, fallback: number) => {
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
||||
};
|
||||
|
||||
export const getAiEvaluationBudgetUsage = () => ({
|
||||
requests: aiProviderRequestCount,
|
||||
observedSpendUsd: observedAiProviderSpend,
|
||||
});
|
||||
|
||||
export const reserveAiEvaluationProviderRequest = () => {
|
||||
const requestLimit = getPositiveLimit(process.env.CUSTOM_WIDGET_AI_MAX_PROVIDER_CALLS, 100);
|
||||
const spendLimit = Number(process.env.CUSTOM_WIDGET_AI_MAX_SPEND_USD);
|
||||
if (!Number.isFinite(spendLimit) || spendLimit <= 0) {
|
||||
throw new Error(
|
||||
"CUSTOM_WIDGET_AI_MAX_SPEND_USD must define this evaluator process's share of the approved total budget.",
|
||||
);
|
||||
}
|
||||
if (aiProviderRequestCount >= requestLimit) {
|
||||
throw new Error(`AI evaluation request cap reached (${requestLimit}).`);
|
||||
}
|
||||
if (observedAiProviderSpend >= spendLimit) {
|
||||
throw new Error(`AI evaluation observed-spend cap reached ($${spendLimit.toFixed(2)}).`);
|
||||
}
|
||||
aiProviderRequestCount += 1;
|
||||
};
|
||||
|
||||
export const recordAiEvaluationProviderSpend = (cost: unknown) => {
|
||||
if (typeof cost !== "number" || !Number.isFinite(cost) || cost < 0) return;
|
||||
observedAiProviderSpend += cost;
|
||||
};
|
||||
|
||||
export const getAiEvaluationReasoningOptions = (model: string) => ({
|
||||
effort:
|
||||
model === DEFAULT_GENERATOR_MODEL
|
||||
? ("xhigh" as const)
|
||||
: model === DEFAULT_JUDGE_MODEL
|
||||
? ("high" as const)
|
||||
: ("medium" as const),
|
||||
exclude: true,
|
||||
});
|
||||
|
||||
export const getAiEvaluationProviderPreferences = (model: string) => {
|
||||
if (model !== DEFAULT_GENERATOR_MODEL) return undefined;
|
||||
return {
|
||||
order: ["deepinfra/fp8"],
|
||||
quantizations: ["fp8"],
|
||||
allow_fallbacks: true,
|
||||
};
|
||||
};
|
||||
export function getAiEvaluationMaxOutputTokens(purpose: "generation" | "judge", configuredValue: string | undefined) {
|
||||
const defaultValue = 32_768;
|
||||
const minimum = purpose === "judge" ? 2_000 : 4_096;
|
||||
if (configuredValue === undefined) return defaultValue;
|
||||
const configured = Number(configuredValue);
|
||||
if (!Number.isInteger(configured) || configured <= 0) return defaultValue;
|
||||
return Math.min(defaultValue, Math.max(minimum, configured));
|
||||
}
|
||||
export const getAiProviderChatCompletionsUrl = (baseUrl = DEFAULT_AI_PROVIDER_BASE_URL) =>
|
||||
`${baseUrl.replace(/\/+$/u, "")}/chat/completions`;
|
||||
export const resolveAiEvaluationProviderConfig = (environment: Record<string, string | undefined>) => {
|
||||
const configuredBaseUrl = environment.AI_PROVIDER_BASE_URL?.trim();
|
||||
const baseUrl = (configuredBaseUrl || DEFAULT_AI_PROVIDER_BASE_URL).replace(/\/+$/u, "");
|
||||
const homarrProvider = baseUrl.endsWith("/api/ai/v1");
|
||||
const openRouterProvider = baseUrl === DEFAULT_AI_PROVIDER_BASE_URL;
|
||||
const providerDefaultModel = homarrProvider ? "homarr/model" : DEFAULT_GENERATOR_MODEL;
|
||||
const generatorApiKey =
|
||||
environment.AI_PROVIDER_API_KEY ?? (openRouterProvider ? environment.OPENROUTER_API_KEY : undefined);
|
||||
const judgeBaseUrl = (
|
||||
environment.AI_JUDGE_PROVIDER_BASE_URL ??
|
||||
(homarrProvider && environment.OPENROUTER_API_KEY ? DEFAULT_AI_PROVIDER_BASE_URL : baseUrl)
|
||||
).replace(/\/+$/u, "");
|
||||
const openRouterJudge = judgeBaseUrl === DEFAULT_AI_PROVIDER_BASE_URL;
|
||||
return {
|
||||
apiKey: generatorApiKey,
|
||||
baseUrl,
|
||||
generatorModel:
|
||||
environment.AI_PROVIDER_MODEL ??
|
||||
(openRouterProvider ? environment.OPENROUTER_GENERATOR_MODEL : undefined) ??
|
||||
providerDefaultModel,
|
||||
judgeModel:
|
||||
environment.AI_JUDGE_PROVIDER_MODEL ??
|
||||
environment.AI_PROVIDER_JUDGE_MODEL ??
|
||||
(openRouterJudge ? environment.OPENROUTER_JUDGE_MODEL : undefined) ??
|
||||
(judgeBaseUrl === baseUrl && homarrProvider ? providerDefaultModel : DEFAULT_JUDGE_MODEL),
|
||||
judgeBaseUrl,
|
||||
judgeApiKey:
|
||||
environment.AI_JUDGE_PROVIDER_API_KEY ??
|
||||
(openRouterJudge ? environment.OPENROUTER_API_KEY : undefined) ??
|
||||
generatorApiKey,
|
||||
};
|
||||
};
|
||||
|
||||
const scoreSchema = z.number().int().min(0).max(100);
|
||||
const judgeCategoriesSchema = z.strictObject({
|
||||
schemaAndBindings: scoreSchema,
|
||||
apiAndRequestDesign: scoreSchema,
|
||||
runtimeCompatibility: scoreSchema,
|
||||
goalFulfillment: scoreSchema,
|
||||
visualQuality: scoreSchema,
|
||||
responsiveAndTheme: scoreSchema,
|
||||
loadingEmptyErrorSuccess: scoreSchema,
|
||||
dailyUsefulness: scoreSchema,
|
||||
complexityDiscipline: scoreSchema,
|
||||
accessibility: scoreSchema,
|
||||
actionSafety: scoreSchema,
|
||||
});
|
||||
const categoryReasonSchema = z.string().min(1).max(600);
|
||||
const judgeResultSchema = z.strictObject({
|
||||
total: scoreSchema,
|
||||
verdict: z.enum(["pass", "fail"]),
|
||||
dailyUseDecision: z.enum(["would-use-daily", "promising-but-not-daily", "not-practical"]),
|
||||
categories: judgeCategoriesSchema,
|
||||
categoryReasons: z.strictObject({
|
||||
schemaAndBindings: categoryReasonSchema,
|
||||
apiAndRequestDesign: categoryReasonSchema,
|
||||
runtimeCompatibility: categoryReasonSchema,
|
||||
goalFulfillment: categoryReasonSchema,
|
||||
visualQuality: categoryReasonSchema,
|
||||
responsiveAndTheme: categoryReasonSchema,
|
||||
loadingEmptyErrorSuccess: categoryReasonSchema,
|
||||
dailyUsefulness: categoryReasonSchema,
|
||||
complexityDiscipline: categoryReasonSchema,
|
||||
accessibility: categoryReasonSchema,
|
||||
actionSafety: categoryReasonSchema,
|
||||
}),
|
||||
strengths: z.array(z.string().min(1).max(400)).max(6),
|
||||
problems: z.array(z.string().min(1).max(500)).max(12),
|
||||
fatalProblems: z.array(z.string().min(1).max(500)).max(8),
|
||||
highestImpactFixes: z.array(z.string().min(1).max(500)).max(6),
|
||||
});
|
||||
|
||||
const categoryWeights = {
|
||||
schemaAndBindings: 8,
|
||||
apiAndRequestDesign: 10,
|
||||
runtimeCompatibility: 8,
|
||||
goalFulfillment: 15,
|
||||
visualQuality: 15,
|
||||
responsiveAndTheme: 8,
|
||||
loadingEmptyErrorSuccess: 8,
|
||||
dailyUsefulness: 12,
|
||||
complexityDiscipline: 8,
|
||||
accessibility: 4,
|
||||
actionSafety: 4,
|
||||
} as const satisfies Record<keyof z.infer<typeof judgeCategoriesSchema>, number>;
|
||||
|
||||
export type CustomWidgetJudgeResult = z.infer<typeof judgeResultSchema>;
|
||||
|
||||
export type CustomWidgetJudgePanelStatus = "pass" | "fail" | "inconclusive";
|
||||
|
||||
export interface CustomWidgetJudgePanelCall {
|
||||
call: number;
|
||||
raw: string | null;
|
||||
result: CustomWidgetJudgeResult | null;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
export interface CustomWidgetJudgePanel {
|
||||
status: CustomWidgetJudgePanelStatus;
|
||||
artifactHash: string;
|
||||
judges: CustomWidgetJudgeResult[];
|
||||
representative: CustomWidgetJudgeResult | null;
|
||||
passVotes: number;
|
||||
failVotes: number;
|
||||
medianTotal: number | null;
|
||||
scoreRange: { min: number; max: number } | null;
|
||||
medianCategories: CustomWidgetJudgeResult["categories"] | null;
|
||||
calls: CustomWidgetJudgePanelCall[];
|
||||
}
|
||||
|
||||
export interface AiEvaluationResult {
|
||||
caseId: string;
|
||||
attempts: number;
|
||||
widget: HomarrCustomWidgetV2 | null;
|
||||
judge: CustomWidgetJudgeResult | null;
|
||||
judgePanel: CustomWidgetJudgePanel | null;
|
||||
judgeStatus: CustomWidgetJudgePanelStatus;
|
||||
outputDirectory: string;
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
interface OpenRouterResponse {
|
||||
choices?: Array<{ message?: { content?: string | null } }>;
|
||||
error?: { message?: string };
|
||||
usage?: { cost?: number };
|
||||
}
|
||||
|
||||
export function buildEvaluationPrompt(testCase: CustomWidgetAiEvaluationCase): string {
|
||||
return buildCustomWidgetAiPrompt(
|
||||
undefined,
|
||||
getEvaluationResponseFixtureText(testCase),
|
||||
null,
|
||||
`${testCase.request}\n\nVerified API notes:\n${testCase.apiNotes}`,
|
||||
testCase.documentationUrl,
|
||||
);
|
||||
}
|
||||
|
||||
export function getEvaluationResponseFixtureText(testCase: CustomWidgetAiEvaluationCase): string | null {
|
||||
if (testCase.previewResponses?.length) {
|
||||
return testCase.previewResponses
|
||||
.map(({ pathIncludes, response }) => `${pathIncludes}:\n${JSON.stringify(response, null, 2)}`)
|
||||
.join("\n\n");
|
||||
}
|
||||
return testCase.sampleResponse === undefined ? null : JSON.stringify(testCase.sampleResponse, null, 2);
|
||||
}
|
||||
|
||||
const getExpectedBindingValue = (value: unknown): string => {
|
||||
if (typeof value === "string") return value;
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value))
|
||||
return JSON.stringify(value) ?? String(value);
|
||||
if ("$param" in value && typeof value.$param === "string") return `$param:${value.$param}`;
|
||||
if ("$option" in value && typeof value.$option === "string") return `$option:${value.$option}`;
|
||||
return JSON.stringify(value);
|
||||
};
|
||||
|
||||
const bindingMatchesExpectation = (
|
||||
value: unknown,
|
||||
expected: string | readonly string[],
|
||||
options: HomarrCustomWidgetV2["options"],
|
||||
): boolean => {
|
||||
if (Array.isArray(expected)) {
|
||||
return expected.some((candidate) => bindingMatchesExpectation(value, candidate, options));
|
||||
}
|
||||
const actual = getExpectedBindingValue(value);
|
||||
if (actual === expected) return true;
|
||||
if (expected === "$param:*" && actual.startsWith("$param:")) return true;
|
||||
if (expected === "$option:*" && actual.startsWith("$option:")) return true;
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
|
||||
if (!("$option" in value) || typeof value.$option !== "string") return false;
|
||||
const option = options[value.$option];
|
||||
if (!option) return false;
|
||||
return getExpectedBindingValue(option.default) === expected;
|
||||
};
|
||||
|
||||
export interface DeterministicEvaluationIssue {
|
||||
path?: Array<string | number>;
|
||||
message: string;
|
||||
}
|
||||
|
||||
export const getExpectedWidgetCase = (
|
||||
testCase: CustomWidgetAiEvaluationCase,
|
||||
expectedWidget: NonNullable<CustomWidgetAiEvaluationCase["expectedWidgets"]>[number],
|
||||
): CustomWidgetAiEvaluationCase => ({
|
||||
...testCase,
|
||||
id: `${testCase.id}-${expectedWidget.id}`,
|
||||
request: expectedWidget.request,
|
||||
apiNotes: expectedWidget.apiNotes ?? testCase.apiNotes,
|
||||
previewResponses: testCase.previewResponses?.filter((previewResponse) =>
|
||||
expectedWidget.expectations.requests.some(
|
||||
(request) =>
|
||||
request.pathIncludes === previewResponse.pathIncludes &&
|
||||
(previewResponse.kind === undefined || request.kind === previewResponse.kind) &&
|
||||
(previewResponse.method === undefined || request.method === previewResponse.method),
|
||||
),
|
||||
),
|
||||
expectations: expectedWidget.expectations,
|
||||
expectedWidgets: undefined,
|
||||
});
|
||||
|
||||
const getAuthName = (source: HomarrCustomWidgetV2["sources"][string] | undefined) =>
|
||||
typeof source?.auth === "object" ? source.auth.name : undefined;
|
||||
|
||||
const requestMatchesExpectation = (
|
||||
request: HomarrCustomWidgetV2["requests"][string],
|
||||
expected: CustomWidgetAiExpectation["requests"][number],
|
||||
options: HomarrCustomWidgetV2["options"],
|
||||
requests: HomarrCustomWidgetV2["requests"],
|
||||
) => {
|
||||
if (
|
||||
request.kind !== expected.kind ||
|
||||
request.method !== expected.method ||
|
||||
!request.path.includes(expected.pathIncludes) ||
|
||||
(expected.trigger !== undefined && request.trigger !== expected.trigger) ||
|
||||
(expected.permission !== undefined && request.permission !== expected.permission)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const queryMatches = Object.entries(expected.queryIncludes ?? {}).every(([key, value]) =>
|
||||
bindingMatchesExpectation(request.query?.[key], value, options),
|
||||
);
|
||||
if (!queryMatches) return false;
|
||||
const bodyMatches = Object.entries(expected.bodyIncludes ?? {}).every(([key, value]) => {
|
||||
if (typeof request.body !== "object" || request.body === null || Array.isArray(request.body)) return false;
|
||||
const body = request.body as Record<string, unknown>;
|
||||
return bindingMatchesExpectation(body[key], value, options);
|
||||
});
|
||||
if (!bodyMatches) return false;
|
||||
const invalidates = new Set(request.invalidates ?? []);
|
||||
if (!(expected.invalidates ?? []).every((requestId) => invalidates.has(requestId))) return false;
|
||||
const invalidatedRequests = [...invalidates]
|
||||
.map((requestId) => requests[requestId])
|
||||
.filter((candidate) => candidate !== undefined);
|
||||
if (
|
||||
!(expected.invalidatesPaths ?? []).every((expectedPath) =>
|
||||
invalidatedRequests.some((candidate) => candidate.path === expectedPath),
|
||||
)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (expected.requiresConfirmation === true && request.confirmation === undefined) return false;
|
||||
return true;
|
||||
};
|
||||
|
||||
const formatExpectedBinding = (value: string | readonly string[]): string => {
|
||||
if (Array.isArray(value)) return value.map((candidate) => formatExpectedBinding(candidate)).join(" or ");
|
||||
if (value === "$param:*") return "any $param binding";
|
||||
if (value === "$option:*") return "any $option binding";
|
||||
return JSON.stringify(value);
|
||||
};
|
||||
|
||||
const getExpectedRequestConstraintSummary = (expected: CustomWidgetAiExpectation["requests"][number]) => {
|
||||
const formatBindings = (bindings: Readonly<Record<string, string | readonly string[]>>) =>
|
||||
Object.entries(bindings)
|
||||
.map(([name, value]) => `${name}=${formatExpectedBinding(value)}`)
|
||||
.join(", ");
|
||||
const constraints: string[] = [];
|
||||
if (expected.trigger !== undefined) constraints.push(`trigger=${expected.trigger}`);
|
||||
if (expected.permission !== undefined) constraints.push(`permission=${expected.permission}`);
|
||||
if (expected.queryIncludes !== undefined) {
|
||||
constraints.push(`query bindings ${formatBindings(expected.queryIncludes)}`);
|
||||
}
|
||||
if (expected.bodyIncludes !== undefined) {
|
||||
constraints.push(`body bindings ${formatBindings(expected.bodyIncludes)}`);
|
||||
}
|
||||
if (expected.invalidatesPaths?.length) {
|
||||
constraints.push(`invalidates query request IDs for paths ${expected.invalidatesPaths.join(", ")}`);
|
||||
}
|
||||
if (expected.invalidates?.length) {
|
||||
constraints.push(`invalidates request IDs ${expected.invalidates.join(", ")}`);
|
||||
}
|
||||
if (expected.requiresConfirmation === true) constraints.push("confirmation is required");
|
||||
if (constraints.length === 0) return "";
|
||||
return ` Required: ${constraints.join("; ")}.`;
|
||||
};
|
||||
|
||||
export function getDeterministicEvaluationIssues(
|
||||
testCase: CustomWidgetAiEvaluationCase,
|
||||
widget: HomarrCustomWidgetV2,
|
||||
): DeterministicEvaluationIssue[] {
|
||||
const expectations = testCase.expectations;
|
||||
if (!expectations) return [];
|
||||
const issues: DeterministicEvaluationIssue[] = [];
|
||||
const source = widget.sources.default;
|
||||
if (expectations.sourceType !== undefined && source?.type !== expectations.sourceType) {
|
||||
issues.push({
|
||||
path: ["sources", "default", "type"],
|
||||
message: `Use the verified ${expectations.sourceType} source type.`,
|
||||
});
|
||||
}
|
||||
if (expectations.sourceBaseUrl !== undefined && (!source || source.baseUrl !== expectations.sourceBaseUrl)) {
|
||||
issues.push({
|
||||
path: ["sources", "default", "baseUrl"],
|
||||
message: `Use the verified source URL ${expectations.sourceBaseUrl}.`,
|
||||
});
|
||||
}
|
||||
if (
|
||||
expectations.sourceIntegrationKind !== undefined &&
|
||||
(!source || source.type !== "integration" || source.integrationKind !== expectations.sourceIntegrationKind)
|
||||
) {
|
||||
issues.push({
|
||||
path: ["sources", "default", "integrationKind"],
|
||||
message: `Use the verified ${expectations.sourceIntegrationKind} saved integration.`,
|
||||
});
|
||||
}
|
||||
const authType = getCustomWidgetSourceAuthType(source);
|
||||
if (expectations.sourceAuth !== undefined && authType !== expectations.sourceAuth) {
|
||||
issues.push({
|
||||
path: ["sources", "default", "auth"],
|
||||
message: `Use the verified ${expectations.sourceAuth} authentication mode.`,
|
||||
});
|
||||
}
|
||||
if (expectations.sourceNetworkScope !== undefined && source?.networkScope !== expectations.sourceNetworkScope) {
|
||||
issues.push({
|
||||
path: ["sources", "default", "networkScope"],
|
||||
message: `Use the verified ${expectations.sourceNetworkScope} network scope.`,
|
||||
});
|
||||
}
|
||||
if (expectations.sourceAuthName !== undefined && getAuthName(source) !== expectations.sourceAuthName) {
|
||||
issues.push({
|
||||
path: ["sources", "default", "auth", "name"],
|
||||
message: `Use '${expectations.sourceAuthName}' as the verified API-key name.`,
|
||||
});
|
||||
}
|
||||
if (
|
||||
expectations.minimumTemplateCharacters !== undefined &&
|
||||
widget.template.length < expectations.minimumTemplateCharacters
|
||||
) {
|
||||
issues.push({
|
||||
path: ["template"],
|
||||
message: `Build a substantive interface of at least ${expectations.minimumTemplateCharacters} JSX characters.`,
|
||||
});
|
||||
}
|
||||
const matchedRequestIds = new Set<string>();
|
||||
for (const expected of expectations.requests) {
|
||||
const match = Object.entries(widget.requests).find(([requestId, request]) => {
|
||||
if (matchedRequestIds.has(requestId)) return false;
|
||||
return requestMatchesExpectation(request, expected, widget.options, widget.requests);
|
||||
});
|
||||
if (!match) {
|
||||
issues.push({
|
||||
path: ["requests"],
|
||||
message: `${`Define the verified ${expected.trigger ?? ""} ${expected.kind} request ${expected.method} ${expected.pathIncludes}.`.replace(
|
||||
/\s+/gu,
|
||||
" ",
|
||||
)}${getExpectedRequestConstraintSummary(expected)}`,
|
||||
});
|
||||
} else {
|
||||
matchedRequestIds.add(match[0]);
|
||||
if (expected.requiresStatusBinding === true && !widget.template.includes(`status.${match[0]}`)) {
|
||||
issues.push({
|
||||
path: ["template"],
|
||||
message: `Read loading, error, and success state from status.${match[0]}; there is no global status.loading or status.ok.`,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const requiredText of expectations.templateIncludes ?? []) {
|
||||
if (!widget.template.includes(requiredText)) {
|
||||
issues.push({
|
||||
path: ["template"],
|
||||
message: `Render or use the required '${requiredText}' capability from the verified response and request.`,
|
||||
});
|
||||
}
|
||||
}
|
||||
for (const alternatives of expectations.templateIncludesAny ?? []) {
|
||||
if (alternatives.some((text) => widget.template.includes(text))) continue;
|
||||
issues.push({
|
||||
path: ["template"],
|
||||
message: `Render or use one equivalent capability: ${alternatives.map((text) => `'${text}'`).join(", ")}.`,
|
||||
});
|
||||
}
|
||||
for (const match of widget.template.matchAll(/<Pagination\b([^>]*)>/gu)) {
|
||||
const attributes = match[1] ?? "";
|
||||
const binding = attributes.match(/\bbind\s*=\s*["']([^"']+)["']/u)?.[1];
|
||||
if (binding && widget.template.includes(`inputs.${binding}`)) continue;
|
||||
issues.push({
|
||||
path: ["template"],
|
||||
message:
|
||||
"Wire Pagination with bind and use its inputs value in a supported request/helper, or render pagination context as text instead.",
|
||||
});
|
||||
}
|
||||
return issues;
|
||||
}
|
||||
|
||||
export interface DeterministicEvaluationMatch {
|
||||
expectedWidgetId: string;
|
||||
widgetIndex: number;
|
||||
testCase: CustomWidgetAiEvaluationCase;
|
||||
issues: DeterministicEvaluationIssue[];
|
||||
}
|
||||
|
||||
const getWidgetIndexPermutations = (widgetCount: number, expectedCount: number) => {
|
||||
const permutations: number[][] = [];
|
||||
const visit = (selected: number[]) => {
|
||||
if (selected.length === expectedCount) {
|
||||
permutations.push(selected);
|
||||
return;
|
||||
}
|
||||
for (let widgetIndex = 0; widgetIndex < widgetCount; widgetIndex += 1) {
|
||||
if (selected.includes(widgetIndex)) continue;
|
||||
visit([...selected, widgetIndex]);
|
||||
}
|
||||
};
|
||||
visit([]);
|
||||
return permutations;
|
||||
};
|
||||
|
||||
export function getDeterministicEvaluationMatches(
|
||||
testCase: CustomWidgetAiEvaluationCase,
|
||||
widgets: readonly HomarrCustomWidgetV2[],
|
||||
): DeterministicEvaluationMatch[] {
|
||||
const expectedWidgets = testCase.expectedWidgets;
|
||||
if (!expectedWidgets?.length) {
|
||||
const widget = widgets[0];
|
||||
if (!widget) return [];
|
||||
return [
|
||||
{
|
||||
expectedWidgetId: testCase.id,
|
||||
widgetIndex: 0,
|
||||
testCase,
|
||||
issues: getDeterministicEvaluationIssues(testCase, widget),
|
||||
},
|
||||
];
|
||||
}
|
||||
if (widgets.length < expectedWidgets.length) return [];
|
||||
let bestMatches: DeterministicEvaluationMatch[] = [];
|
||||
let bestIssueCount = Number.POSITIVE_INFINITY;
|
||||
for (const permutation of getWidgetIndexPermutations(widgets.length, expectedWidgets.length)) {
|
||||
const matches = expectedWidgets.map((expectedWidget, expectedIndex) => {
|
||||
const widgetIndex = permutation[expectedIndex] ?? -1;
|
||||
const expectedCase = getExpectedWidgetCase(testCase, expectedWidget);
|
||||
return {
|
||||
expectedWidgetId: expectedWidget.id,
|
||||
widgetIndex,
|
||||
testCase: expectedCase,
|
||||
issues: getDeterministicEvaluationIssues(expectedCase, widgets[widgetIndex] as HomarrCustomWidgetV2),
|
||||
};
|
||||
});
|
||||
const issueCount = matches.reduce((sum, match) => sum + match.issues.length, 0);
|
||||
if (issueCount >= bestIssueCount) continue;
|
||||
bestMatches = matches;
|
||||
bestIssueCount = issueCount;
|
||||
}
|
||||
return bestMatches;
|
||||
}
|
||||
|
||||
export function getDeterministicEvaluationSuiteIssues(
|
||||
testCase: CustomWidgetAiEvaluationCase,
|
||||
widgets: readonly HomarrCustomWidgetV2[],
|
||||
): DeterministicEvaluationIssue[] {
|
||||
if (!testCase.expectedWidgets?.length) {
|
||||
const widget = widgets[0];
|
||||
if (!widget) return [{ message: "Create the required widget." }];
|
||||
return getDeterministicEvaluationIssues(testCase, widget);
|
||||
}
|
||||
const issues: DeterministicEvaluationIssue[] = [];
|
||||
if (widgets.length !== testCase.expectedWidgets.length) {
|
||||
issues.push({
|
||||
message: `Create exactly ${testCase.expectedWidgets.length} independent widgets; received ${widgets.length}.`,
|
||||
});
|
||||
}
|
||||
const matches = getDeterministicEvaluationMatches(testCase, widgets);
|
||||
if (matches.length === 0) {
|
||||
issues.push({ message: "Every requested widget job needs its own complete manifest." });
|
||||
return issues;
|
||||
}
|
||||
for (const match of matches) {
|
||||
issues.push(
|
||||
...match.issues.map((issue) => ({
|
||||
...issue,
|
||||
message: `${match.expectedWidgetId}: ${issue.message}`,
|
||||
})),
|
||||
);
|
||||
}
|
||||
return issues;
|
||||
}
|
||||
|
||||
export function buildRepairPrompt(
|
||||
originalPrompt: string,
|
||||
previousResponse: string,
|
||||
issues: readonly { path?: Array<string | number>; message: string }[],
|
||||
): string {
|
||||
const diagnostics = issues
|
||||
.map((issue) => `${issue.path?.length ? `${issue.path.join(".")}: ` : ""}${issue.message}`)
|
||||
.join("\n");
|
||||
return `${originalPrompt}\n\nYour previous response did not validate. Correct only the generic contract/runtime problems below while preserving a polished design.\n\nDiagnostics:\n${diagnostics}\n\nPrevious response:\n${previousResponse}`;
|
||||
}
|
||||
|
||||
const getJudgeRuntimeContext = (widget: HomarrCustomWidgetV2) => {
|
||||
const sections = [getCustomWidgetSkillReference("runtime").content];
|
||||
const hasMutation = Object.values(widget.requests).some((request) => request.kind === "action");
|
||||
const hasProtectedSource = Object.values(widget.sources).some((source) => {
|
||||
const authType = getCustomWidgetSourceAuthType(source);
|
||||
return authType !== "none";
|
||||
});
|
||||
if (hasMutation || hasProtectedSource) sections.push(getCustomWidgetSkillReference("security").content);
|
||||
return sections.join("\n\n");
|
||||
};
|
||||
|
||||
export function buildJudgePrompt(testCase: CustomWidgetAiEvaluationCase, widget: HomarrCustomWidgetV2): string {
|
||||
return `You are a hostile-but-fair product review panel evaluating a safe dashboard widget. Most competent drafts should score 55-75, not 90. Judge only evidence present in the manifest and JSX. Never reward unsupported capabilities, invented API routes, aspirational claims, or code that merely validates.
|
||||
|
||||
The installed Homarr skill and runtime references below are authoritative. The verified API notes and representative response fixtures are authoritative for endpoint paths, authentication requirements, and response shapes unless the validated manifest contradicts them. Do not invent external endpoint or authentication objections from outside assumptions. Judge only the scoped Request below; an endpoint mentioned in broader API notes is available, not automatically required, and an optional fixture field omitted by the scoped Request is not a missing capability. Decorative icons paired with equivalent adjacent visible status text need no separate aria-label. A Badge containing explicit visible status text is not color-only. Date.toLocaleString(value, "en-US", "UTC") is an installed safe static helper for concise absolute UTC timestamps; do not require relative time. In particular, request state is exposed as status.<requestId> with loading/ok/status/error fields while successful payloads are exposed as data.<requestId>. RefreshButton is an installed runtime helper: it refreshes load queries by default; inside a successful manual result, requestId targets and reruns that active query with unchanged parameters. A bound Pagination should declare defaultValue={1}; resetKey={inputs.query} restores that default when its dependent scalar query changes. SubFetch without trigger="manual" runs automatically and reruns when bound params change, so never demand a hidden input, debounce callback, or raw event. When the scoped Request requires manual search, do not penalize the required re-trigger after query or page changes. A manual SubFetch synchronously hides its old result and returns to its trigger when its request ID, normalized parameters, or effective definition changes; it cannot display stale results under edited inputs or fetch the new parameters before another trigger. ActionButton supplies pending UI, native success/error notification, confirmation, and declared invalidation; do not demand duplicate local action state. Safe templates forbid local declarations and helper functions; do not penalize a repeated short literal label array used for distinct enum fields, or a repeated pure collection/count expression that must drive both an empty branch and rendered content, as an avoidable missing abstraction. Every component in this already-validated template exists in the installed release. Do not penalize those documented facts. The widget has already passed Homarr's real schema and JSX analyzer, which proves syntax and component compatibility but does not prove API correctness, visual quality, usefulness, or accessibility.
|
||||
|
||||
Task-relevant installed Homarr runtime references:
|
||||
${getJudgeRuntimeContext(widget)}
|
||||
|
||||
Request:
|
||||
${testCase.request}
|
||||
|
||||
Verified API notes:
|
||||
${testCase.apiNotes}
|
||||
|
||||
Representative API response used by the preview test:
|
||||
${getEvaluationResponseFixtureText(testCase) ?? "Not supplied."}
|
||||
|
||||
Validated widget:
|
||||
${JSON.stringify(widget, null, 2)}
|
||||
|
||||
Scoring calibration:
|
||||
- 95-100: exceptional, purpose-built quality; complete, beautiful, restrained, and something a demanding user would choose every day. Almost never award this.
|
||||
- 85-94: excellent with only small, specific defects. It must fully achieve the request and feel deliberately designed.
|
||||
- 70-84: good prototype or useful widget with visible compromises, generic design, missing polish, or avoidable complexity.
|
||||
- 50-69: functional but incomplete, awkward, visually ordinary, overbuilt, or impractical for repeated use.
|
||||
- 0-49: broken, misleading, unsafe, substantially incomplete, or incompatible.
|
||||
|
||||
Required review behavior:
|
||||
- Compare every requested capability with concrete manifest/JSX evidence. A missing or invented core capability is fatal and caps total at 79.
|
||||
- Judge whether the API design can actually reach the stated goal, including response paths, bindings, invalidation, and action safety.
|
||||
- Judge visual quality, not component count: hierarchy, density, whitespace, typography, restrained color, scanability, and avoidance of repetitive nested cards.
|
||||
- A purpose-specific asymmetric summary, divided hierarchy, responsive density, and restrained semantic accents can clear 75 without decorative chrome. Do not demand gradients, novelty, or a generic selected-row detail interaction.
|
||||
- Judge daily usefulness: information priority, interaction cost, refresh behavior, narrow-tile usability, and whether the widget is pleasant rather than demo-like.
|
||||
- Do not penalize a missing last-refreshed timestamp when the verified response has no freshness field; cache policy plus a labeled RefreshButton is complete. Do not penalize a deliberately bounded first-page view for lacking pagination unless the scoped Request asks for pagination or browsing the full collection.
|
||||
- A required manual search rerun is deliberate interaction, not daily-use friction. SubFetch owns failure and retry before its child renders; never demand an unreachable child error branch or a RefreshButton there.
|
||||
- Judge complexity discipline: penalize duplicate requests/options, unnecessary controls, excessive JSX, cleverness, and UI chrome that does not help the goal. Complexity must earn its place.
|
||||
- Recommend only interactions supported by the installed authoring contract. Do not suggest portals, modals, arbitrary event handlers, or other blocked capabilities. Prefer a responsive in-widget detail area when separation is useful.
|
||||
- A visually generic but valid widget should normally score below 75 for visualQuality. A widget that is attractive but inconvenient should score below 75 for dailyUsefulness.
|
||||
- Give a concrete evidence sentence for every category. List all score-capping issues under fatalProblems.
|
||||
- Return an empty fatalProblems array when there are no fatal problems; never put "none" or an explanation of their absence in that array.
|
||||
|
||||
Homarr computes the weighted total and final verdict itself. Your total and verdict are advisory, but must be internally honest. A pass requires a weighted total of at least 85, every category at least 75, goalFulfillment at least 85, complexityDiscipline at least 80, no fatal problem, and dailyUseDecision="would-use-daily". Return only the requested structured object.`;
|
||||
}
|
||||
|
||||
export function judgePasses(result: CustomWidgetJudgeResult): boolean {
|
||||
return (
|
||||
result.total >= 85 &&
|
||||
Object.values(result.categories).every((score) => score >= 75) &&
|
||||
result.categories.goalFulfillment >= 85 &&
|
||||
result.categories.complexityDiscipline >= 80 &&
|
||||
result.dailyUseDecision === "would-use-daily" &&
|
||||
result.fatalProblems.length === 0
|
||||
);
|
||||
}
|
||||
|
||||
const median = (values: readonly number[]) => {
|
||||
if (values.length === 0) return null;
|
||||
const sorted = values.toSorted((left, right) => left - right);
|
||||
const midpoint = Math.floor(sorted.length / 2);
|
||||
if (sorted.length % 2 === 1) return sorted[midpoint] ?? null;
|
||||
const left = sorted[midpoint - 1];
|
||||
const right = sorted[midpoint];
|
||||
if (left === undefined || right === undefined) return null;
|
||||
return Math.round((left + right) / 2);
|
||||
};
|
||||
|
||||
export function getCustomWidgetArtifactHash(widget: HomarrCustomWidgetV2) {
|
||||
return createHash("sha256").update(JSON.stringify(widget)).digest("hex");
|
||||
}
|
||||
|
||||
export function summarizeCustomWidgetJudgePanel(
|
||||
widget: HomarrCustomWidgetV2,
|
||||
calls: readonly CustomWidgetJudgePanelCall[],
|
||||
): CustomWidgetJudgePanel {
|
||||
const judges = calls.flatMap((call) => (call.result ? [call.result] : []));
|
||||
const passVotes = judges.filter(judgePasses).length;
|
||||
const failVotes = judges.length - passVotes;
|
||||
let status: CustomWidgetJudgePanelStatus = "inconclusive";
|
||||
if (judges.length >= 3) {
|
||||
const firstThreePassVotes = judges.slice(0, 3).filter(judgePasses).length;
|
||||
if (firstThreePassVotes === 3) status = "pass";
|
||||
if (firstThreePassVotes === 0) status = "fail";
|
||||
if (firstThreePassVotes > 0 && firstThreePassVotes < 3 && judges.length >= 5) {
|
||||
status = passVotes >= 3 ? "pass" : "fail";
|
||||
}
|
||||
}
|
||||
|
||||
const medianTotal = median(judges.map((judge) => judge.total));
|
||||
let representative: CustomWidgetJudgeResult | null = null;
|
||||
const representativeCandidates = judges.filter((judge) => {
|
||||
if (status === "pass") return judgePasses(judge);
|
||||
if (status === "fail") return !judgePasses(judge);
|
||||
return true;
|
||||
});
|
||||
const representativeMedian = median(representativeCandidates.map((judge) => judge.total));
|
||||
if (representativeMedian !== null) {
|
||||
representative =
|
||||
representativeCandidates.toSorted(
|
||||
(left, right) => Math.abs(left.total - representativeMedian) - Math.abs(right.total - representativeMedian),
|
||||
)[0] ?? null;
|
||||
}
|
||||
let scoreRange: CustomWidgetJudgePanel["scoreRange"] = null;
|
||||
if (judges.length > 0) {
|
||||
const totals = judges.map((judge) => judge.total);
|
||||
scoreRange = { min: Math.min(...totals), max: Math.max(...totals) };
|
||||
}
|
||||
let medianCategories: CustomWidgetJudgePanel["medianCategories"] = null;
|
||||
if (judges.length > 0) {
|
||||
medianCategories = Object.fromEntries(
|
||||
Object.keys(categoryWeights).map((category) => [
|
||||
category,
|
||||
median(judges.map((judge) => judge.categories[category as keyof typeof categoryWeights])) ?? 0,
|
||||
]),
|
||||
) as CustomWidgetJudgeResult["categories"];
|
||||
}
|
||||
return {
|
||||
status,
|
||||
artifactHash: getCustomWidgetArtifactHash(widget),
|
||||
judges,
|
||||
representative,
|
||||
passVotes,
|
||||
failVotes,
|
||||
medianTotal,
|
||||
scoreRange,
|
||||
medianCategories,
|
||||
calls: [...calls],
|
||||
};
|
||||
}
|
||||
|
||||
export function getJudgeResponseFormat() {
|
||||
return {
|
||||
type: "json_schema" as const,
|
||||
json_schema: {
|
||||
name: "homarr_custom_widget_review",
|
||||
strict: true,
|
||||
schema: z.toJSONSchema(judgeResultSchema, { io: "output" }),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function requestCustomWidgetJudge(args: {
|
||||
testCase: CustomWidgetAiEvaluationCase;
|
||||
widget: HomarrCustomWidgetV2;
|
||||
apiKey: string;
|
||||
baseUrl?: string;
|
||||
judgeModel?: string;
|
||||
sample?: number;
|
||||
}) {
|
||||
const perspectives = [
|
||||
"Prioritize API correctness and runtime compatibility while still applying every rubric gate.",
|
||||
"Prioritize daily usefulness and information design while still applying every rubric gate.",
|
||||
"Act as an adversarial completeness reviewer while still applying every rubric gate.",
|
||||
"Prioritize responsive visual quality and accessibility while still applying every rubric gate.",
|
||||
"Prioritize complexity discipline and action safety while still applying every rubric gate.",
|
||||
];
|
||||
const sample = args.sample ?? 1;
|
||||
const perspective = perspectives[(sample - 1) % perspectives.length] ?? perspectives[0];
|
||||
return callOpenRouter({
|
||||
apiKey: args.apiKey,
|
||||
baseUrl: args.baseUrl,
|
||||
model: args.judgeModel ?? DEFAULT_JUDGE_MODEL,
|
||||
prompt: `${buildJudgePrompt(args.testCase, args.widget)}\n\nIndependent review lens: ${perspective}`,
|
||||
purpose: "judge",
|
||||
temperature: Math.min(0.35, (sample - 1) * 0.08),
|
||||
});
|
||||
}
|
||||
|
||||
export async function judgeCustomWidgetCase(args: {
|
||||
testCase: CustomWidgetAiEvaluationCase;
|
||||
widget: HomarrCustomWidgetV2;
|
||||
apiKey: string;
|
||||
baseUrl?: string;
|
||||
judgeModel?: string;
|
||||
}) {
|
||||
const raw = await requestCustomWidgetJudge(args);
|
||||
return { raw, result: parseJudgeResult(raw) };
|
||||
}
|
||||
|
||||
export async function judgeCustomWidgetPanel(args: {
|
||||
testCase: CustomWidgetAiEvaluationCase;
|
||||
widget: HomarrCustomWidgetV2;
|
||||
apiKey: string;
|
||||
baseUrl?: string;
|
||||
judgeModel?: string;
|
||||
}) {
|
||||
const calls: CustomWidgetJudgePanelCall[] = [];
|
||||
const collect = async (count: number) => {
|
||||
const start = calls.length;
|
||||
const collected = await Promise.all(
|
||||
Array.from({ length: count }, async (_, index): Promise<CustomWidgetJudgePanelCall> => {
|
||||
const call = start + index + 1;
|
||||
try {
|
||||
const raw = await requestCustomWidgetJudge({ ...args, sample: call });
|
||||
return { call, raw, result: parseJudgeResult(raw), error: null };
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Unknown judge error";
|
||||
return { call, raw: null, result: null, error: message };
|
||||
}
|
||||
}),
|
||||
);
|
||||
calls.push(...collected);
|
||||
};
|
||||
|
||||
await collect(3);
|
||||
while (calls.flatMap((call) => (call.result ? [call.result] : [])).length < 3 && calls.length < 5) {
|
||||
await collect(1);
|
||||
}
|
||||
let panel = summarizeCustomWidgetJudgePanel(args.widget, calls);
|
||||
const splitInitialVote = panel.judges.length >= 3 && panel.passVotes > 0 && panel.failVotes > 0;
|
||||
if (splitInitialVote) {
|
||||
await collect(2);
|
||||
while (calls.flatMap((call) => (call.result ? [call.result] : [])).length < 5 && calls.length < 7) {
|
||||
await collect(1);
|
||||
}
|
||||
panel = summarizeCustomWidgetJudgePanel(args.widget, calls);
|
||||
}
|
||||
return panel;
|
||||
}
|
||||
|
||||
export async function evaluateCustomWidgetCase(args: {
|
||||
testCase: CustomWidgetAiEvaluationCase;
|
||||
apiKey: string;
|
||||
baseUrl?: string;
|
||||
judgeApiKey?: string;
|
||||
judgeBaseUrl?: string;
|
||||
outputRoot: string;
|
||||
maxLoops: number;
|
||||
generatorModel?: string;
|
||||
judgeModel?: string;
|
||||
}): Promise<AiEvaluationResult> {
|
||||
const caseDirectory = path.join(args.outputRoot, args.testCase.id);
|
||||
await mkdir(caseDirectory, { recursive: true });
|
||||
const originalPrompt = buildEvaluationPrompt(args.testCase);
|
||||
await writeFile(path.join(caseDirectory, "prompt.md"), originalPrompt, "utf8");
|
||||
|
||||
let prompt = originalPrompt;
|
||||
const errors: string[] = [];
|
||||
let bestWidget: HomarrCustomWidgetV2 | null = null;
|
||||
let bestJudge: CustomWidgetJudgeResult | null = null;
|
||||
let bestJudgePanel: CustomWidgetJudgePanel | null = null;
|
||||
for (let attempt = 1; attempt <= Math.min(args.maxLoops, MAX_AI_EVALUATION_LOOPS); attempt += 1) {
|
||||
let response: string;
|
||||
try {
|
||||
response = await callOpenRouter({
|
||||
apiKey: args.apiKey,
|
||||
baseUrl: args.baseUrl,
|
||||
model: args.generatorModel ?? DEFAULT_GENERATOR_MODEL,
|
||||
prompt,
|
||||
purpose: "generation",
|
||||
});
|
||||
} catch (error) {
|
||||
errors.push(
|
||||
`Attempt ${attempt}: generator request failed — ${error instanceof Error ? error.message : "Unknown error"}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
await writeFile(path.join(caseDirectory, `attempt-${attempt}.md`), response, "utf8");
|
||||
const parsed = parseCustomWidgetAiResponse(response);
|
||||
if (!parsed.success) {
|
||||
const message = formatCustomWidgetImportIssues(parsed.issues);
|
||||
errors.push(`Attempt ${attempt}: ${message}`);
|
||||
prompt = buildRepairPrompt(originalPrompt, response, parsed.issues);
|
||||
continue;
|
||||
}
|
||||
|
||||
const canonical = customWidgetDefinitionSchema.parse(parsed.widget);
|
||||
const deterministicIssues = getDeterministicEvaluationIssues(args.testCase, canonical);
|
||||
if (deterministicIssues.length > 0) {
|
||||
errors.push(
|
||||
`Attempt ${attempt}: deterministic scenario checks failed — ${deterministicIssues.map((issue) => issue.message).join("; ")}`,
|
||||
);
|
||||
prompt = buildRepairPrompt(originalPrompt, response, deterministicIssues);
|
||||
continue;
|
||||
}
|
||||
await writeWidgetFiles(caseDirectory, canonical, `attempt-${attempt}`);
|
||||
const judgePanel = await judgeCustomWidgetPanel({
|
||||
testCase: args.testCase,
|
||||
widget: canonical,
|
||||
apiKey: args.judgeApiKey ?? args.apiKey,
|
||||
baseUrl: args.judgeBaseUrl ?? args.baseUrl,
|
||||
judgeModel: args.judgeModel,
|
||||
});
|
||||
await writeFile(
|
||||
path.join(caseDirectory, `judge-panel-${attempt}.json`),
|
||||
JSON.stringify(judgePanel, null, 2),
|
||||
"utf8",
|
||||
);
|
||||
for (const call of judgePanel.calls) {
|
||||
if (call.raw) await writeFile(path.join(caseDirectory, `judge-${attempt}-${call.call}.json`), call.raw, "utf8");
|
||||
}
|
||||
const judge = judgePanel.representative;
|
||||
if (!judge) {
|
||||
errors.push(`Attempt ${attempt}: judge panel inconclusive — no valid judge response.`);
|
||||
bestWidget = canonical;
|
||||
bestJudgePanel = judgePanel;
|
||||
break;
|
||||
}
|
||||
if (!bestJudge || (judgePanel.medianTotal ?? -1) > (bestJudgePanel?.medianTotal ?? -1)) {
|
||||
bestJudge = judge;
|
||||
bestWidget = canonical;
|
||||
bestJudgePanel = judgePanel;
|
||||
await writeWidgetFiles(caseDirectory, canonical, "best");
|
||||
await writeFile(path.join(caseDirectory, "best-judge-panel.json"), JSON.stringify(judgePanel, null, 2), "utf8");
|
||||
}
|
||||
if (judgePanel.status === "pass") {
|
||||
await writeFile(path.join(caseDirectory, "result.json"), JSON.stringify(judgePanel, null, 2), "utf8");
|
||||
return {
|
||||
caseId: args.testCase.id,
|
||||
attempts: attempt,
|
||||
widget: canonical,
|
||||
judge,
|
||||
judgePanel,
|
||||
judgeStatus: "pass",
|
||||
outputDirectory: caseDirectory,
|
||||
errors,
|
||||
};
|
||||
}
|
||||
if (judgePanel.status === "inconclusive") {
|
||||
errors.push(`Attempt ${attempt}: judge panel inconclusive after ${judgePanel.calls.length} calls.`);
|
||||
break;
|
||||
}
|
||||
errors.push(
|
||||
`Attempt ${attempt}: judge panel failed ${judgePanel.passVotes}-${judgePanel.failVotes}, median ${judgePanel.medianTotal}/100 — ${judge.highestImpactFixes.join("; ")}`,
|
||||
);
|
||||
prompt = buildRepairPrompt(
|
||||
originalPrompt,
|
||||
response,
|
||||
judge.highestImpactFixes.map((message) => ({ message })),
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
caseId: args.testCase.id,
|
||||
attempts: Math.min(args.maxLoops, MAX_AI_EVALUATION_LOOPS),
|
||||
widget: bestWidget,
|
||||
judge: bestJudge,
|
||||
judgePanel: bestJudgePanel,
|
||||
judgeStatus: bestJudgePanel?.status ?? "inconclusive",
|
||||
outputDirectory: caseDirectory,
|
||||
errors,
|
||||
};
|
||||
}
|
||||
|
||||
export function parseJudgeResult(raw: string): CustomWidgetJudgeResult {
|
||||
const normalizedRaw = raw
|
||||
.trim()
|
||||
.replace(/^```(?:json)?\s*/iu, "")
|
||||
.replace(/\s*```$/u, "");
|
||||
const parsed: unknown = JSON.parse(normalizedRaw);
|
||||
const result = judgeResultSchema.parse(parsed);
|
||||
const fatalProblems = result.fatalProblems.filter(
|
||||
(problem) => !/^(?:none|no fatal (?:problems?|issues?))(?:\b|\s*[:.-])/iu.test(problem.trim()),
|
||||
);
|
||||
const weightedTotal = Object.entries(categoryWeights).reduce(
|
||||
(sum, [category, weight]) => sum + result.categories[category as keyof typeof categoryWeights] * weight,
|
||||
0,
|
||||
);
|
||||
const normalized = {
|
||||
...result,
|
||||
fatalProblems,
|
||||
total: Math.round(weightedTotal / 100),
|
||||
};
|
||||
return { ...normalized, verdict: judgePasses(normalized) ? "pass" : "fail" };
|
||||
}
|
||||
|
||||
async function writeWidgetFiles(directory: string, widget: HomarrCustomWidgetV2, basename: string) {
|
||||
await writeFile(path.join(directory, `${basename}.widget.json`), JSON.stringify(widget, null, 2), "utf8");
|
||||
}
|
||||
|
||||
async function callOpenRouter(args: {
|
||||
apiKey: string;
|
||||
baseUrl?: string;
|
||||
model: string;
|
||||
prompt: string;
|
||||
purpose: "generation" | "judge";
|
||||
temperature?: number;
|
||||
}): Promise<string> {
|
||||
const isJudge = args.purpose === "judge";
|
||||
let configuredMaxOutputTokens = process.env.CUSTOM_WIDGET_AI_GENERATION_MAX_OUTPUT_TOKENS;
|
||||
if (isJudge) configuredMaxOutputTokens = process.env.CUSTOM_WIDGET_AI_JUDGE_MAX_OUTPUT_TOKENS;
|
||||
const attempts = isJudge ? 1 : 2;
|
||||
for (let attempt = 1; attempt <= attempts; attempt += 1) {
|
||||
reserveAiEvaluationProviderRequest();
|
||||
const response = await fetch(getAiProviderChatCompletionsUrl(args.baseUrl), {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${args.apiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
"HTTP-Referer": "https://homarr.dev",
|
||||
"X-Title": "Homarr Custom Widget AI Evaluation",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: args.model,
|
||||
messages: [{ role: "user", content: args.prompt }],
|
||||
temperature: args.temperature ?? (isJudge ? 0 : 0.2),
|
||||
max_tokens: getAiEvaluationMaxOutputTokens(args.purpose, configuredMaxOutputTokens),
|
||||
reasoning: getAiEvaluationReasoningOptions(args.model),
|
||||
...(getAiEvaluationProviderPreferences(args.model)
|
||||
? { provider: getAiEvaluationProviderPreferences(args.model) }
|
||||
: {}),
|
||||
...(isJudge ? { response_format: getJudgeResponseFormat() } : {}),
|
||||
}),
|
||||
signal: AbortSignal.timeout(180_000),
|
||||
});
|
||||
const payload = (await response.json()) as OpenRouterResponse;
|
||||
recordAiEvaluationProviderSpend(payload.usage?.cost);
|
||||
if (!response.ok)
|
||||
throw new Error(`AI provider request failed (${response.status}): ${payload.error?.message ?? "Unknown error"}`);
|
||||
const content = payload.choices?.[0]?.message?.content;
|
||||
if (content) return content;
|
||||
if (attempt === attempts) {
|
||||
const suffix = isJudge ? "" : " after one retry";
|
||||
throw new Error(`AI provider returned no message content${suffix}`);
|
||||
}
|
||||
}
|
||||
throw new Error("AI provider returned no message content");
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
import { readdir, readFile } from "node:fs/promises";
|
||||
import { extname, join, relative, resolve } from "node:path";
|
||||
|
||||
const packageRoot = resolve(import.meta.dirname, "..");
|
||||
const repositoryRoot = resolve(packageRoot, "../..");
|
||||
const sourceRoots = [
|
||||
join(packageRoot, "src"),
|
||||
join(repositoryRoot, "packages/api/src/router/custom-widget"),
|
||||
join(repositoryRoot, "packages/widgets/src/custom-api"),
|
||||
join(repositoryRoot, "apps/nextjs/src/app/[locale]/manage/custom-widgets"),
|
||||
];
|
||||
const productionFiles = [];
|
||||
|
||||
async function collect(directory) {
|
||||
for (const entry of await readdir(directory, { withFileTypes: true })) {
|
||||
const path = join(directory, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
if (entry.name !== "test") await collect(path);
|
||||
} else if ([".ts", ".tsx"].includes(extname(entry.name)) && !/\.spec\.tsx?$/u.test(entry.name)) {
|
||||
productionFiles.push(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const sourceRoot of sourceRoots) await collect(sourceRoot);
|
||||
const failures = [];
|
||||
const graph = new Map();
|
||||
for (const file of productionFiles) {
|
||||
const source = await readFile(file, "utf8");
|
||||
const name = relative(repositoryRoot, file);
|
||||
if (file.startsWith(join(packageRoot, "src")) && /from\s+["']@homarr\/(?:api|widgets)(?:\/|["'])/u.test(source)) {
|
||||
failures.push(`${name} imports a forbidden adapter package`);
|
||||
}
|
||||
if (extname(file) === ".tsx" && /(?:runtime|workbench|custom-widgets|custom-api)/u.test(name)) {
|
||||
for (const line of source.split("\n")) {
|
||||
const match = line.match(/>\s*([A-Za-z][A-Za-z0-9 .,!?'-]*)\s*</u);
|
||||
if (match) failures.push(`${name} contains untranslated JSX text: ${JSON.stringify(match[1]?.trim())}`);
|
||||
}
|
||||
if (/\b(?:aria-label|description|label|placeholder|title)=["'][A-Za-z]/u.test(source)) {
|
||||
failures.push(`${name} contains an untranslated user-facing attribute`);
|
||||
}
|
||||
}
|
||||
const dependencies = [];
|
||||
for (const match of source.matchAll(/from\s+["'](\.[^"']+)["']/gu)) {
|
||||
const candidate = resolve(file, "..", match[1]);
|
||||
const resolved = productionFiles.find(
|
||||
(item) =>
|
||||
item === candidate ||
|
||||
item === `${candidate}.ts` ||
|
||||
item === `${candidate}.tsx` ||
|
||||
item === join(candidate, "index.ts"),
|
||||
);
|
||||
if (resolved) dependencies.push(resolved);
|
||||
}
|
||||
graph.set(file, dependencies);
|
||||
}
|
||||
|
||||
const visiting = new Set();
|
||||
const visited = new Set();
|
||||
function visit(file, path = []) {
|
||||
if (visiting.has(file)) {
|
||||
failures.push(`circular dependency: ${[...path, file].map((item) => relative(repositoryRoot, item)).join(" -> ")}`);
|
||||
return;
|
||||
}
|
||||
if (visited.has(file)) return;
|
||||
visiting.add(file);
|
||||
for (const dependency of graph.get(file) ?? []) visit(dependency, [...path, file]);
|
||||
visiting.delete(file);
|
||||
visited.add(file);
|
||||
}
|
||||
for (const file of productionFiles) visit(file);
|
||||
|
||||
const widgetAdapter = await readFile(join(repositoryRoot, "packages/widgets/src/custom-api/component.tsx"), "utf8");
|
||||
if (!/dynamic\(\(\)\s*=>\s*import\(["']\.\/custom-jsx-display["']\)/u.test(widgetAdapter)) {
|
||||
failures.push("Custom JSX board rendering must remain dynamically loaded");
|
||||
}
|
||||
const editorAdapter = await readFile(
|
||||
join(repositoryRoot, "packages/custom-widgets/src/workbench/code-editor.tsx"),
|
||||
"utf8",
|
||||
);
|
||||
if (!/lazy\(\(\)\s*=>\s*import\(["'].\/direct-code-mirror["']\)\)/u.test(editorAdapter)) {
|
||||
failures.push("CodeMirror must remain dynamically loaded after the client mounts");
|
||||
}
|
||||
const componentRegistry = await readFile(
|
||||
join(repositoryRoot, "packages/custom-widgets/src/core/component-registry.ts"),
|
||||
"utf8",
|
||||
);
|
||||
if (!/from\s+["'].\/component-catalog["']/u.test(componentRegistry)) {
|
||||
failures.push("The Custom JSX runtime registry must use the canonical authoring catalog");
|
||||
}
|
||||
if (/component-runtime\.generated\.json/u.test(componentRegistry)) {
|
||||
failures.push("The Custom JSX runtime registry must not use a second generated component catalog");
|
||||
}
|
||||
|
||||
if (failures.length > 0) {
|
||||
console.error(failures.join("\n"));
|
||||
process.exitCode = 1;
|
||||
} else {
|
||||
console.log(`Custom Widgets architecture checks passed for ${productionFiles.length} production modules.`);
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
import { CUSTOM_WIDGET_AI_EVALUATION_CASES } from "./ai-evaluation-cases";
|
||||
import { evaluateCustomWidgetAssistantCase } from "./ai-assistant-evaluation";
|
||||
import type { CustomWidgetAssistantEvaluationResult } from "./ai-assistant-evaluation";
|
||||
import {
|
||||
evaluateCustomWidgetCase,
|
||||
getAiEvaluationBudgetUsage,
|
||||
resolveAiEvaluationProviderConfig,
|
||||
} from "./ai-evaluation";
|
||||
import type { AiEvaluationResult } from "./ai-evaluation";
|
||||
|
||||
const {
|
||||
apiKey,
|
||||
baseUrl: providerBaseUrl,
|
||||
generatorModel,
|
||||
judgeApiKey,
|
||||
judgeBaseUrl,
|
||||
judgeModel,
|
||||
} = resolveAiEvaluationProviderConfig(process.env);
|
||||
if (!apiKey) {
|
||||
throw new Error("AI_PROVIDER_API_KEY or OPENROUTER_API_KEY is required for the live Custom Widget AI evaluation");
|
||||
}
|
||||
if (!judgeApiKey) {
|
||||
throw new Error(
|
||||
"AI_JUDGE_PROVIDER_API_KEY, AI_PROVIDER_API_KEY, or OPENROUTER_API_KEY is required for the live judge panel",
|
||||
);
|
||||
}
|
||||
if (
|
||||
providerBaseUrl === judgeBaseUrl &&
|
||||
generatorModel === judgeModel &&
|
||||
process.env.CUSTOM_WIDGET_AI_ALLOW_SELF_JUDGE !== "true"
|
||||
) {
|
||||
throw new Error(
|
||||
"The generator and judge must use different models. Set AI_JUDGE_PROVIDER_MODEL or explicitly opt in with CUSTOM_WIDGET_AI_ALLOW_SELF_JUDGE=true.",
|
||||
);
|
||||
}
|
||||
|
||||
const getPositiveConfiguration = (value: string | undefined, fallback: number) => {
|
||||
const parsed = Number(value);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) return fallback;
|
||||
return parsed;
|
||||
};
|
||||
const maxProviderCalls = getPositiveConfiguration(process.env.CUSTOM_WIDGET_AI_MAX_PROVIDER_CALLS, 100);
|
||||
const maxSpendUsd = Number(process.env.CUSTOM_WIDGET_AI_MAX_SPEND_USD);
|
||||
if (!Number.isFinite(maxSpendUsd) || maxSpendUsd <= 0) {
|
||||
throw new Error(
|
||||
"CUSTOM_WIDGET_AI_MAX_SPEND_USD is required. For parallel evaluator processes, allocate shares whose sum does not exceed the approved total budget.",
|
||||
);
|
||||
}
|
||||
|
||||
const requestedCase = process.argv.find((value) => value.startsWith("--case="))?.slice("--case=".length);
|
||||
const assistantMode = process.argv.includes("--assistant");
|
||||
const configuredLoops = Number(process.env.CUSTOM_WIDGET_AI_MAX_LOOPS ?? 10);
|
||||
const maxLoops = Number.isInteger(configuredLoops) && configuredLoops > 0 ? Math.min(configuredLoops, 10) : 10;
|
||||
let selectableCases = CUSTOM_WIDGET_AI_EVALUATION_CASES.filter((testCase) => !testCase.expectedWidgets?.length);
|
||||
if (assistantMode) {
|
||||
selectableCases = CUSTOM_WIDGET_AI_EVALUATION_CASES.filter(
|
||||
(testCase) =>
|
||||
(testCase.sampleResponse !== undefined || testCase.previewResponses?.length) &&
|
||||
(testCase.expectations !== undefined || testCase.expectedWidgets?.length),
|
||||
);
|
||||
}
|
||||
let selectedCases = selectableCases;
|
||||
if (requestedCase) selectedCases = selectableCases.filter((testCase) => testCase.id === requestedCase);
|
||||
if (selectedCases.length === 0) throw new Error(`Unknown AI evaluation case '${requestedCase}'`);
|
||||
|
||||
const runId = `${new Date().toISOString().replaceAll(/[:.]/gu, "-")}-${process.pid}`;
|
||||
const outputRoot = path.resolve(process.cwd(), ".ai-evaluations", runId);
|
||||
await mkdir(outputRoot, { recursive: true });
|
||||
|
||||
function getResultScoreFloor(result: AiEvaluationResult | CustomWidgetAssistantEvaluationResult) {
|
||||
if ("judgePanels" in result && result.judgePanels.length > 0) {
|
||||
const medians = result.judgePanels.flatMap((panel) => (panel.medianTotal === null ? [] : [panel.medianTotal]));
|
||||
if (medians.length > 0) return Math.min(...medians);
|
||||
}
|
||||
if ("judgePanel" in result && result.judgePanel?.medianTotal !== null) {
|
||||
return result.judgePanel?.medianTotal ?? null;
|
||||
}
|
||||
if ("judges" in result && result.judges.length > 0) {
|
||||
return Math.min(...result.judges.map((judge) => judge.total));
|
||||
}
|
||||
return result.judge?.total ?? null;
|
||||
}
|
||||
|
||||
const results: Array<AiEvaluationResult | CustomWidgetAssistantEvaluationResult> = [];
|
||||
for (const testCase of selectedCases) {
|
||||
process.stdout.write(`Evaluating ${testCase.id}...\n`);
|
||||
let result: AiEvaluationResult | CustomWidgetAssistantEvaluationResult;
|
||||
if (assistantMode) {
|
||||
result = await evaluateCustomWidgetAssistantCase({
|
||||
testCase,
|
||||
apiKey,
|
||||
baseUrl: providerBaseUrl,
|
||||
outputRoot,
|
||||
maxLoops,
|
||||
generatorModel,
|
||||
judgeApiKey,
|
||||
judgeBaseUrl,
|
||||
judgeModel,
|
||||
});
|
||||
} else {
|
||||
result = await evaluateCustomWidgetCase({
|
||||
testCase,
|
||||
apiKey,
|
||||
baseUrl: providerBaseUrl,
|
||||
outputRoot,
|
||||
maxLoops,
|
||||
generatorModel,
|
||||
judgeApiKey,
|
||||
judgeBaseUrl,
|
||||
judgeModel,
|
||||
});
|
||||
}
|
||||
results.push(result);
|
||||
if (result.judge) {
|
||||
const widgetCount = "widgets" in result ? result.widgets.length : 1;
|
||||
const scoreFloor = getResultScoreFloor(result);
|
||||
const status = result.judgeStatus === "pass" ? "passed" : result.judgeStatus;
|
||||
process.stdout.write(
|
||||
` ${scoreFloor}/100 floor across ${widgetCount} widget(s), ${status} after ${result.attempts} attempt(s)\n`,
|
||||
);
|
||||
} else {
|
||||
process.stdout.write(` failed after ${result.attempts} attempt(s)\n`);
|
||||
}
|
||||
}
|
||||
|
||||
const summary = {
|
||||
generatedAt: new Date().toISOString(),
|
||||
mode: assistantMode ? "assistant-tool-loop" : "manifest",
|
||||
providerBaseUrl,
|
||||
generatorModel,
|
||||
judgeBaseUrl,
|
||||
judgeModel,
|
||||
budget: {
|
||||
maxProviderCalls,
|
||||
maxSpendUsd,
|
||||
...getAiEvaluationBudgetUsage(),
|
||||
},
|
||||
results: results.map((result) => ({
|
||||
caseId: result.caseId,
|
||||
attempts: result.attempts,
|
||||
score: getResultScoreFloor(result),
|
||||
verdict: result.judgeStatus,
|
||||
categories: result.judge?.categories ?? null,
|
||||
errors: result.errors,
|
||||
calledTools: "calledTools" in result ? result.calledTools : undefined,
|
||||
widgets: "widgets" in result ? result.widgets.length : 1,
|
||||
widgetScores: "judges" in result ? result.judges.map((judge) => judge.total) : undefined,
|
||||
judgePanels:
|
||||
"judgePanels" in result
|
||||
? result.judgePanels.map(({ status, artifactHash, passVotes, failVotes, medianTotal, scoreRange }) => ({
|
||||
status,
|
||||
artifactHash,
|
||||
passVotes,
|
||||
failVotes,
|
||||
medianTotal,
|
||||
scoreRange,
|
||||
}))
|
||||
: result.judgePanel,
|
||||
efficiency: "efficiency" in result ? result.efficiency : undefined,
|
||||
outputDirectory: path.relative(process.cwd(), result.outputDirectory),
|
||||
})),
|
||||
};
|
||||
await writeFile(path.join(outputRoot, "summary.json"), JSON.stringify(summary, null, 2), "utf8");
|
||||
process.stdout.write(`Results saved to ${path.relative(process.cwd(), outputRoot)}\n`);
|
||||
|
||||
if (results.some((result) => result.judgeStatus !== "pass")) process.exitCode = 1;
|
||||
@@ -0,0 +1,735 @@
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
import * as Charts from "@mantine/charts";
|
||||
import * as Core from "@mantine/core";
|
||||
import * as Dates from "@mantine/dates";
|
||||
import ts from "typescript-compiler-api";
|
||||
|
||||
import {
|
||||
CUSTOM_JSX_AUTHORING_CATALOG_SCHEMA_VERSION,
|
||||
CUSTOM_WIDGET_AUTHORING_VERSION,
|
||||
} from "../src/core/component-catalog-types";
|
||||
import type {
|
||||
CustomJsxAuthoringCatalog,
|
||||
CustomJsxBlockedCapability,
|
||||
CustomJsxCatalogBindingType,
|
||||
CustomJsxComponentApi,
|
||||
CustomJsxLiteralValue,
|
||||
CustomJsxPropDescriptor,
|
||||
CustomJsxPropSource,
|
||||
} from "../src/core/component-catalog-types";
|
||||
import { getCatalogBlockedComponentProps, getCatalogDeniedComponentReason } from "../src/core/component-catalog-policy";
|
||||
import { commonSafeProps } from "../src/core/component-props";
|
||||
import { accessibilityByCategory, documentationUrl, subcomponentsByName } from "../src/core/component-descriptor";
|
||||
import { discoveredMantineComponents, discoveredSubcomponentsByName } from "../src/core/component-discovery";
|
||||
import type {
|
||||
CustomJsxComponentCategory,
|
||||
CustomJsxComponentPackage,
|
||||
CustomJsxComponentSafety,
|
||||
} from "../src/core/component-types";
|
||||
import {
|
||||
CUSTOM_JSX_BLOCKED_PROPERTIES,
|
||||
CUSTOM_JSX_BLOCKED_PROPS,
|
||||
CUSTOM_JSX_BLOCKED_STYLE_KEYS,
|
||||
isBlockedCustomJsxProp,
|
||||
} from "../src/jsx/policy";
|
||||
|
||||
void [Core, Dates, Charts];
|
||||
|
||||
const packageRoot = resolve(import.meta.dirname, "..");
|
||||
const repositoryRoot = resolve(packageRoot, "../..");
|
||||
const configPath = resolve(packageRoot, "tsconfig.json");
|
||||
const configFile = ts.readConfigFile(configPath, ts.sys.readFile);
|
||||
if (configFile.error) throw new Error(ts.flattenDiagnosticMessageText(configFile.error.messageText, "\n"));
|
||||
const config = ts.parseJsonConfigFileContent(configFile.config, ts.sys, packageRoot);
|
||||
const program = ts.createProgram({ rootNames: [...config.fileNames, import.meta.filename], options: config.options });
|
||||
const checker = program.getTypeChecker();
|
||||
const source = program.getSourceFile(import.meta.filename);
|
||||
if (!source) throw new Error("Unable to inspect the component catalog generator");
|
||||
|
||||
const moduleSymbols = new Map<string, ts.Symbol>();
|
||||
for (const declaration of source.statements) {
|
||||
if (!ts.isImportDeclaration(declaration) || !ts.isStringLiteral(declaration.moduleSpecifier)) continue;
|
||||
const moduleName = declaration.moduleSpecifier.text;
|
||||
if (!moduleName.startsWith("@mantine/")) continue;
|
||||
const symbol = checker.getSymbolAtLocation(declaration.moduleSpecifier);
|
||||
if (symbol) moduleSymbols.set(moduleName, symbol);
|
||||
}
|
||||
|
||||
type UninternedPropDescriptor = Omit<CustomJsxPropDescriptor, "typeRef"> & { type: string };
|
||||
type UninternedComponentApi = Omit<CustomJsxComponentApi, "props"> & { props: UninternedPropDescriptor[] };
|
||||
|
||||
interface InspectedProp extends UninternedPropDescriptor {
|
||||
global: boolean;
|
||||
}
|
||||
|
||||
interface InspectedComponent {
|
||||
description?: string;
|
||||
props: InspectedProp[];
|
||||
}
|
||||
|
||||
interface CatalogSourceComponent {
|
||||
name: string;
|
||||
package: CustomJsxComponentPackage;
|
||||
category: CustomJsxComponentCategory;
|
||||
safety: CustomJsxComponentSafety;
|
||||
subcomponents: readonly string[];
|
||||
accessibilityRequirements: readonly string[];
|
||||
documentationUrl: string;
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
const commonPropNames = new Set<string>(commonSafeProps);
|
||||
const inspectedComponents = new Map<string, InspectedComponent>();
|
||||
const globalCandidates = new Map<string, UninternedPropDescriptor[]>();
|
||||
|
||||
async function generateCatalog() {
|
||||
const sourceComponents = createSourceComponents();
|
||||
for (const component of sourceComponents) {
|
||||
if (component.package === "@homarr/widgets" || component.safety === "denied") continue;
|
||||
const inspected = inspectMantineComponent(component);
|
||||
inspectedComponents.set(component.name, inspected);
|
||||
for (const prop of inspected.props) {
|
||||
if (!prop.global) continue;
|
||||
const candidates = globalCandidates.get(prop.name) ?? [];
|
||||
const { global: _global, ...descriptor } = prop;
|
||||
candidates.push({ ...descriptor, source: "global" });
|
||||
globalCandidates.set(prop.name, candidates);
|
||||
}
|
||||
}
|
||||
|
||||
for (const name of commonSafeProps) {
|
||||
const candidates = globalCandidates.get(name) ?? [];
|
||||
if (candidates.length === 0) candidates.push(createFallbackProp(name, "global"));
|
||||
globalCandidates.set(name, candidates);
|
||||
}
|
||||
|
||||
const uninternedGlobalProps = [...globalCandidates.entries()]
|
||||
.map(([name, candidates]) => mergeGlobalProp(name, candidates))
|
||||
.toSorted((left, right) => compareStrings(left.name, right.name));
|
||||
const globalPropsByName = new Map(uninternedGlobalProps.map((prop) => [prop.name, prop]));
|
||||
|
||||
const uninternedComponents = sourceComponents
|
||||
.map((component) => buildComponentApi(component, globalPropsByName))
|
||||
.toSorted((left, right) => compareStrings(left.name, right.name));
|
||||
const types = [
|
||||
...new Set([
|
||||
...uninternedGlobalProps.map(({ type }) => type),
|
||||
...uninternedComponents.flatMap(({ props }) => props.map(({ type }) => type)),
|
||||
]),
|
||||
].toSorted(compareStrings);
|
||||
const typeReferences = new Map(types.map((type, index) => [type, index]));
|
||||
const internProp = ({ type, ...prop }: UninternedPropDescriptor): CustomJsxPropDescriptor => {
|
||||
const typeRef = typeReferences.get(type);
|
||||
if (typeRef === undefined) throw new Error(`Unable to intern prop type '${type}'`);
|
||||
return { ...prop, typeRef };
|
||||
};
|
||||
const globalProps = uninternedGlobalProps.map(internProp);
|
||||
const components = uninternedComponents.map(
|
||||
(component): CustomJsxComponentApi => ({ ...component, props: component.props.map(internProp) }),
|
||||
);
|
||||
|
||||
const mantineVersions = await Promise.all(
|
||||
["core", "dates", "charts"].map((packageName) =>
|
||||
readPackageVersion(resolve(repositoryRoot, `node_modules/@mantine/${packageName}/package.json`)),
|
||||
),
|
||||
);
|
||||
if (new Set(mantineVersions).size !== 1) {
|
||||
throw new Error(`Mantine package versions must match: ${mantineVersions.join(", ")}`);
|
||||
}
|
||||
const mantineVersion = mantineVersions[0];
|
||||
if (!mantineVersion) throw new Error("Unable to determine the installed Mantine version");
|
||||
|
||||
const catalog: CustomJsxAuthoringCatalog = {
|
||||
schemaVersion: CUSTOM_JSX_AUTHORING_CATALOG_SCHEMA_VERSION,
|
||||
mantineVersion,
|
||||
customWidgetVersion: CUSTOM_WIDGET_AUTHORING_VERSION,
|
||||
types,
|
||||
globalProps,
|
||||
blockedCapabilities: createBlockedCapabilities(),
|
||||
components,
|
||||
};
|
||||
|
||||
const serialized = `${JSON.stringify(catalog)}\n`;
|
||||
await Promise.all([
|
||||
writeFile(resolve(packageRoot, "src/core/component-catalog.generated.json"), serialized),
|
||||
writeFile(resolve(repositoryRoot, "apps/docs/public/custom-widgets/component-catalog-v1.json"), serialized),
|
||||
]);
|
||||
}
|
||||
|
||||
function inspectMantineComponent(component: CatalogSourceComponent): InspectedComponent {
|
||||
const moduleSymbol = moduleSymbols.get(component.package);
|
||||
const rootName = component.name.split(".")[0];
|
||||
let symbol =
|
||||
moduleSymbol && checker.getExportsOfModule(moduleSymbol).find((candidate) => candidate.name === rootName);
|
||||
let location = symbol?.valueDeclaration ?? symbol?.declarations?.[0] ?? source;
|
||||
let type = symbol && checker.getTypeOfSymbolAtLocation(symbol, location);
|
||||
for (const segment of component.name.split(".").slice(1)) {
|
||||
symbol = type?.getProperty(segment);
|
||||
if (!symbol) break;
|
||||
location = symbol.valueDeclaration ?? symbol.declarations?.[0] ?? location;
|
||||
type = checker.getTypeOfSymbolAtLocation(symbol, location);
|
||||
}
|
||||
|
||||
const signature = type?.getCallSignatures()[0] ?? type?.getConstructSignatures()[0];
|
||||
const parameter = signature?.parameters[0];
|
||||
const propType = parameter && checker.getTypeOfSymbolAtLocation(parameter, parameter.valueDeclaration ?? location);
|
||||
const props = propType
|
||||
? checker
|
||||
.getPropertiesOfType(propType)
|
||||
.flatMap((prop): InspectedProp[] => {
|
||||
const declaration = prop.valueDeclaration ?? prop.declarations?.[0] ?? location;
|
||||
const resolvedType = checker.getTypeOfSymbolAtLocation(prop, declaration);
|
||||
if (prop.name === "bind" || isBlockedCustomJsxProp(prop.name) || isCallableType(resolvedType)) return [];
|
||||
const global = isGlobalProp(prop);
|
||||
return [createPropDescriptor(prop, resolvedType, declaration, global ? "global" : "component", global)];
|
||||
})
|
||||
.toSorted((left, right) => compareStrings(left.name, right.name))
|
||||
: [];
|
||||
return {
|
||||
description: normalizeDescription(symbol && ts.displayPartsToString(symbol.getDocumentationComment(checker))),
|
||||
props,
|
||||
};
|
||||
}
|
||||
|
||||
function createSourceComponents(): CatalogSourceComponent[] {
|
||||
const mantineComponents = discoveredMantineComponents.map((component): CatalogSourceComponent => {
|
||||
const reason = getCatalogDeniedComponentReason(component.name);
|
||||
const category = reason ? ("blocked" as const) : component.category;
|
||||
return {
|
||||
name: component.name,
|
||||
package: component.package,
|
||||
category,
|
||||
safety: reason ? "denied" : "wrapped",
|
||||
subcomponents: discoveredSubcomponentsByName[component.name] ?? subcomponentsByName[component.name] ?? [],
|
||||
accessibilityRequirements: accessibilityByCategory[category],
|
||||
documentationUrl: documentationUrl(component.package, component.name),
|
||||
...(reason ? { reason } : {}),
|
||||
};
|
||||
});
|
||||
const homarrComponents = HOMARR_COMPONENT_DEFINITIONS.map(
|
||||
({ name, category, safety }): CatalogSourceComponent => ({
|
||||
name,
|
||||
package: "@homarr/widgets",
|
||||
category,
|
||||
safety,
|
||||
subcomponents: [],
|
||||
accessibilityRequirements: accessibilityByCategory[category],
|
||||
documentationUrl: documentationUrl("@homarr/widgets", name),
|
||||
}),
|
||||
);
|
||||
return [...mantineComponents, ...homarrComponents];
|
||||
}
|
||||
|
||||
function buildComponentApi(
|
||||
component: CatalogSourceComponent,
|
||||
globalPropsByName: ReadonlyMap<string, UninternedPropDescriptor>,
|
||||
): UninternedComponentApi {
|
||||
const blockedProps = getCatalogBlockedComponentProps(component.name);
|
||||
const blockedPropNames = new Set(blockedProps.map(({ name }) => name));
|
||||
const globalNames = new Set(globalPropsByName.keys());
|
||||
const inspected = inspectedComponents.get(component.name);
|
||||
const componentProps = new Map<string, UninternedPropDescriptor>();
|
||||
for (const prop of inspected?.props ?? []) {
|
||||
if (blockedPropNames.has(prop.name)) continue;
|
||||
const { global: _global, ...descriptor } = prop;
|
||||
const componentDescriptor = { ...descriptor, source: "component" as const };
|
||||
const globalProp = globalNames.has(prop.name) ? globalPropsByName.get(prop.name) : undefined;
|
||||
if (globalProp && samePropMetadata(globalProp, componentDescriptor)) continue;
|
||||
componentProps.set(prop.name, componentDescriptor);
|
||||
}
|
||||
if (component.safety !== "denied" && component.package === "@homarr/widgets") {
|
||||
const homarrProps = HOMARR_COMPONENT_PROPS[component.name];
|
||||
if (!homarrProps) throw new Error(`Homarr component '${component.name}' requires explicit catalog prop metadata`);
|
||||
for (const [propName, metadata] of Object.entries(homarrProps)) {
|
||||
const globalProp = globalPropsByName.get(propName);
|
||||
if (globalProp && samePropMetadata(globalProp, metadata)) continue;
|
||||
if (isBlockedCustomJsxProp(propName)) continue;
|
||||
componentProps.set(propName, { name: propName, source: "component", ...metadata });
|
||||
}
|
||||
}
|
||||
|
||||
const bindType = component.safety !== "denied" ? CUSTOM_JSX_BINDING_TYPES[component.name] : undefined;
|
||||
const description = inspected?.description ?? HOMARR_COMPONENT_DESCRIPTIONS[component.name];
|
||||
return {
|
||||
name: component.name,
|
||||
package: component.package,
|
||||
category: component.category,
|
||||
safety: component.safety,
|
||||
...(description ? { description } : {}),
|
||||
documentationUrl: component.documentationUrl,
|
||||
props: [...componentProps.values()].toSorted((left, right) => compareStrings(left.name, right.name)),
|
||||
blockedProps: [...blockedProps].toSorted((left, right) => compareStrings(left.name, right.name)),
|
||||
subcomponents: [...component.subcomponents].toSorted(),
|
||||
...(bindType
|
||||
? {
|
||||
bind: {
|
||||
type: bindType,
|
||||
initialProp: bindType === "boolean" ? ("defaultChecked" as const) : ("defaultValue" as const),
|
||||
resetProp: "resetKey" as const,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
accessibilityRequirements: [...component.accessibilityRequirements],
|
||||
...(component.safety === "denied" && component.reason ? { deniedReason: component.reason } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function createPropDescriptor(
|
||||
prop: ts.Symbol,
|
||||
type: ts.Type,
|
||||
declaration: ts.Node,
|
||||
sourceKind: CustomJsxPropSource,
|
||||
global: boolean,
|
||||
): InspectedProp {
|
||||
const literalValues = getLiteralValues(type);
|
||||
const description = normalizeDescription(ts.displayPartsToString(prop.getDocumentationComment(checker)));
|
||||
return {
|
||||
name: prop.name,
|
||||
type: simplifyType(type, declaration),
|
||||
required: (prop.flags & ts.SymbolFlags.Optional) === 0 && !containsUndefined(type),
|
||||
source: sourceKind,
|
||||
...(literalValues ? { literalValues } : {}),
|
||||
...(description ? { description } : {}),
|
||||
global,
|
||||
};
|
||||
}
|
||||
|
||||
function createFallbackProp(name: string, sourceKind: CustomJsxPropSource): UninternedPropDescriptor {
|
||||
const literalValues = FALLBACK_LITERAL_VALUES[name];
|
||||
return {
|
||||
name,
|
||||
type: FALLBACK_PROP_TYPES[name] ?? inferFallbackType(name),
|
||||
required: false,
|
||||
source: sourceKind,
|
||||
...(literalValues ? { literalValues: [...literalValues] } : {}),
|
||||
...(FALLBACK_PROP_DESCRIPTIONS[name] ? { description: FALLBACK_PROP_DESCRIPTIONS[name] } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function mergeGlobalProp(name: string, candidates: UninternedPropDescriptor[]): UninternedPropDescriptor {
|
||||
const fallback = createFallbackProp(name, "global");
|
||||
const type = mostFrequent(candidates.map((candidate) => candidate.type)) ?? fallback.type;
|
||||
const descriptions = candidates.flatMap(({ description }) => (description ? [description] : []));
|
||||
const literalValues = uniqueLiterals(candidates.flatMap(({ literalValues }) => literalValues ?? []));
|
||||
return {
|
||||
name,
|
||||
type,
|
||||
required: false,
|
||||
source: "global",
|
||||
...(literalValues.length > 0 ? { literalValues } : {}),
|
||||
...((mostFrequent(descriptions) ?? fallback.description)
|
||||
? { description: mostFrequent(descriptions) ?? fallback.description }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
function samePropMetadata(globalProp: UninternedPropDescriptor, componentProp: HomarrPropMetadata): boolean {
|
||||
return (
|
||||
globalProp.type === componentProp.type &&
|
||||
globalProp.required === componentProp.required &&
|
||||
JSON.stringify(globalProp.literalValues ?? []) === JSON.stringify(componentProp.literalValues ?? [])
|
||||
);
|
||||
}
|
||||
|
||||
function isCallableType(type: ts.Type): boolean {
|
||||
if (type.getCallSignatures().length > 0) return true;
|
||||
return type.isUnionOrIntersection() && type.types.some(isCallableType);
|
||||
}
|
||||
|
||||
function isGlobalProp(prop: ts.Symbol): boolean {
|
||||
if (commonPropNames.has(prop.name)) return true;
|
||||
return (prop.declarations ?? []).some((declaration) => {
|
||||
const path = declaration.getSourceFile().fileName.replaceAll("\\", "/");
|
||||
return path.includes("/node_modules/@types/react/") || path.includes("/@mantine/core/lib/core/Box/");
|
||||
});
|
||||
}
|
||||
|
||||
function containsUndefined(type: ts.Type): boolean {
|
||||
if ((type.flags & ts.TypeFlags.Undefined) !== 0) return true;
|
||||
return type.isUnion() && type.types.some(containsUndefined);
|
||||
}
|
||||
|
||||
function simplifyType(type: ts.Type, declaration: ts.Node): string {
|
||||
const withoutUndefined = type.isUnion()
|
||||
? type.types.filter((part) => (part.flags & ts.TypeFlags.Undefined) === 0)
|
||||
: [type];
|
||||
const simplified = withoutUndefined.map(simpleTypeName);
|
||||
if (simplified.every((value): value is string => value !== null)) return [...new Set(simplified)].join(" | ");
|
||||
|
||||
const rendered = checker
|
||||
.typeToString(type, declaration, ts.TypeFormatFlags.UseAliasDefinedOutsideCurrentScope)
|
||||
.replace(/\s+/gu, " ")
|
||||
.replace(/\s*\|\s*undefined\b/gu, "")
|
||||
.trim();
|
||||
if (rendered.length <= 240) return rendered === "any" ? "unknown" : rendered;
|
||||
const alias = type.aliasSymbol?.name;
|
||||
if (alias && !alias.startsWith("__")) return alias;
|
||||
return inferWideType(type);
|
||||
}
|
||||
|
||||
function simpleTypeName(type: ts.Type): string | null {
|
||||
if ((type.flags & ts.TypeFlags.String) !== 0) return "string";
|
||||
if ((type.flags & ts.TypeFlags.Number) !== 0) return "number";
|
||||
if ((type.flags & ts.TypeFlags.Boolean) !== 0) return "boolean";
|
||||
if ((type.flags & ts.TypeFlags.Null) !== 0) return "null";
|
||||
if ((type.flags & ts.TypeFlags.StringLiteral) !== 0) return JSON.stringify((type as ts.StringLiteralType).value);
|
||||
if ((type.flags & ts.TypeFlags.NumberLiteral) !== 0) return String((type as ts.NumberLiteralType).value);
|
||||
if ((type.flags & ts.TypeFlags.BooleanLiteral) !== 0)
|
||||
return (type as ts.Type & { intrinsicName?: string }).intrinsicName ?? "boolean";
|
||||
return null;
|
||||
}
|
||||
|
||||
function inferWideType(type: ts.Type): string {
|
||||
const parts = type.isUnionOrIntersection() ? type.types : [type];
|
||||
const kinds = new Set(
|
||||
parts.map((part) => {
|
||||
if ((part.flags & (ts.TypeFlags.String | ts.TypeFlags.StringLiteral)) !== 0) return "string";
|
||||
if ((part.flags & (ts.TypeFlags.Number | ts.TypeFlags.NumberLiteral)) !== 0) return "number";
|
||||
if ((part.flags & (ts.TypeFlags.Boolean | ts.TypeFlags.BooleanLiteral)) !== 0) return "boolean";
|
||||
if ((part.flags & ts.TypeFlags.Null) !== 0) return "null";
|
||||
return "object";
|
||||
}),
|
||||
);
|
||||
return [...kinds].toSorted().join(" | ") || "unknown";
|
||||
}
|
||||
|
||||
function getLiteralValues(type: ts.Type): CustomJsxLiteralValue[] | undefined {
|
||||
const values: CustomJsxLiteralValue[] = [];
|
||||
let hasNonLiteralValue = false;
|
||||
const visit = (candidate: ts.Type) => {
|
||||
if (candidate.isUnion()) {
|
||||
candidate.types.forEach(visit);
|
||||
return;
|
||||
}
|
||||
if ((candidate.flags & ts.TypeFlags.StringLiteral) !== 0) values.push((candidate as ts.StringLiteralType).value);
|
||||
else if ((candidate.flags & ts.TypeFlags.NumberLiteral) !== 0)
|
||||
values.push((candidate as ts.NumberLiteralType).value);
|
||||
else if ((candidate.flags & ts.TypeFlags.BooleanLiteral) !== 0)
|
||||
values.push((candidate as ts.Type & { intrinsicName?: string }).intrinsicName === "true");
|
||||
else if ((candidate.flags & ts.TypeFlags.Null) !== 0) values.push(null);
|
||||
else if ((candidate.flags & ts.TypeFlags.Undefined) === 0) hasNonLiteralValue = true;
|
||||
};
|
||||
visit(type);
|
||||
const unique = uniqueLiterals(values);
|
||||
if (hasNonLiteralValue && unique.every((value) => value === null || typeof value === "boolean")) return undefined;
|
||||
return unique.length > 0 && unique.length <= 64 ? unique : undefined;
|
||||
}
|
||||
|
||||
function uniqueLiterals(values: CustomJsxLiteralValue[]): CustomJsxLiteralValue[] {
|
||||
return [...new Map(values.map((value) => [`${typeof value}:${String(value)}`, value])).values()].toSorted(
|
||||
(left, right) => compareStrings(String(left), String(right)),
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeDescription(value: string | undefined): string | undefined {
|
||||
const normalized = value?.replace(/\s+/gu, " ").trim();
|
||||
return normalized || undefined;
|
||||
}
|
||||
|
||||
function mostFrequent(values: string[]): string | undefined {
|
||||
const counts = new Map<string, number>();
|
||||
for (const value of values) counts.set(value, (counts.get(value) ?? 0) + 1);
|
||||
return [...counts.entries()].toSorted(
|
||||
([leftValue, leftCount], [rightValue, rightCount]) =>
|
||||
rightCount - leftCount || leftValue.length - rightValue.length || compareStrings(leftValue, rightValue),
|
||||
)[0]?.[0];
|
||||
}
|
||||
|
||||
function compareStrings(left: string, right: string): number {
|
||||
return left < right ? -1 : left > right ? 1 : 0;
|
||||
}
|
||||
|
||||
function inferFallbackType(name: string): string {
|
||||
if (/^(?:is|has|with|allow|disabled|readOnly|loading|opened|checked|multiple|grow|wrap|striped|animated)/u.test(name))
|
||||
return "boolean";
|
||||
if (/^(?:count|index|max|min|offset|order|page|pageSize|size|span|step|total|value)$/u.test(name)) return "number";
|
||||
if (/^(?:colors|values|sections|data)$/u.test(name)) return "unknown[]";
|
||||
if (/^(?:children|description|fallback|icon|label|leftSection|rightSection|title)$/u.test(name)) return "ReactNode";
|
||||
if (/^(?:params|enabledParams|disabledParams|style|styles)$/u.test(name)) return "Record<string, unknown>";
|
||||
return "string | number | boolean";
|
||||
}
|
||||
|
||||
function createBlockedCapabilities(): CustomJsxBlockedCapability[] {
|
||||
const exactNames = new Set([
|
||||
...CUSTOM_JSX_BLOCKED_PROPS,
|
||||
...CUSTOM_JSX_BLOCKED_STYLE_KEYS,
|
||||
...[...CUSTOM_JSX_BLOCKED_PROPERTIES].filter((name) => name !== "bind"),
|
||||
]);
|
||||
return [
|
||||
...[...exactNames].toSorted().map(
|
||||
(name): CustomJsxBlockedCapability => ({
|
||||
kind: "prop",
|
||||
name,
|
||||
reason: blockedPropReason(name),
|
||||
}),
|
||||
),
|
||||
{
|
||||
kind: "prop-pattern",
|
||||
name: "on*",
|
||||
reason: "Authored event callbacks can execute code outside the declarative runtime.",
|
||||
},
|
||||
{
|
||||
kind: "prop-pattern",
|
||||
name: "*Ref",
|
||||
reason: "Refs expose underlying React components and DOM nodes.",
|
||||
},
|
||||
{
|
||||
kind: "prop-pattern",
|
||||
name: "__*",
|
||||
reason: "Private and prototype-related properties are outside the supported authoring surface.",
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
function blockedPropReason(name: string): string {
|
||||
if (["component", "renderRoot"].includes(name))
|
||||
return "Polymorphic roots can replace the Homarr-owned element boundary.";
|
||||
if (["portalProps", "popoverTarget", "popoverTargetAction", "withinPortal"].includes(name))
|
||||
return "Portal targeting can render or interact outside the widget root.";
|
||||
if (["dangerouslySetInnerHTML", "srcDoc"].includes(name))
|
||||
return "Raw HTML injection is not available to authored widgets.";
|
||||
if (["className", "classNames", "classes"].includes(name))
|
||||
return "Unscoped class injection can escape widget CSS isolation.";
|
||||
if (name === "children") return "Children must be expressed as interpreted JSX, not passed as a raw prop.";
|
||||
if (name.toLowerCase().includes("ref")) return "Refs expose underlying React components and DOM nodes.";
|
||||
if (name.startsWith("form")) return "External form submission is outside the named-request security boundary.";
|
||||
return "This capability is blocked by the Custom JSX safety boundary.";
|
||||
}
|
||||
|
||||
async function readPackageVersion(path: string): Promise<string> {
|
||||
const parsed = JSON.parse(await readFile(path, "utf8")) as { version?: unknown };
|
||||
if (typeof parsed.version !== "string") throw new Error(`Package at ${path} does not declare a version`);
|
||||
return parsed.version;
|
||||
}
|
||||
|
||||
const HOMARR_COMPONENT_DEFINITIONS = [
|
||||
{ name: "PaginatedList", category: "interaction", safety: "allowed" },
|
||||
{ name: "TabsContainer", category: "interaction", safety: "allowed" },
|
||||
{ name: "TabPanel", category: "interaction", safety: "allowed" },
|
||||
{ name: "Collapsible", category: "interaction", safety: "allowed" },
|
||||
{ name: "StatBar", category: "interaction", safety: "allowed" },
|
||||
{ name: "TypeBadge", category: "interaction", safety: "allowed" },
|
||||
{ name: "SubFetch", category: "network", safety: "wrapped" },
|
||||
{ name: "SubData", category: "network", safety: "wrapped" },
|
||||
{ name: "ActionButton", category: "network", safety: "wrapped" },
|
||||
{ name: "ToggleSwitch", category: "network", safety: "wrapped" },
|
||||
{ name: "RefreshButton", category: "network", safety: "wrapped" },
|
||||
{ name: "TablerIcon", category: "content", safety: "wrapped" },
|
||||
] as const satisfies readonly {
|
||||
name: string;
|
||||
category: CustomJsxComponentCategory;
|
||||
safety: CustomJsxComponentSafety;
|
||||
}[];
|
||||
|
||||
const stringBindingComponents = [
|
||||
"Accordion",
|
||||
"Autocomplete",
|
||||
"Chip.Group",
|
||||
"ChipGroup",
|
||||
"ColorInput",
|
||||
"ColorPicker",
|
||||
"DateInput",
|
||||
"DatePicker",
|
||||
"DatePickerInput",
|
||||
"DateTimePicker",
|
||||
"InlineDateTimePicker",
|
||||
"JsonInput",
|
||||
"MaskInput",
|
||||
"MiniCalendar",
|
||||
"MonthPicker",
|
||||
"MonthPickerInput",
|
||||
"NativeSelect",
|
||||
"PasswordInput",
|
||||
"PinInput",
|
||||
"Radio.Group",
|
||||
"RadioGroup",
|
||||
"SegmentedControl",
|
||||
"Select",
|
||||
"Tabs",
|
||||
"TextInput",
|
||||
"Textarea",
|
||||
"TimeInput",
|
||||
"TimePicker",
|
||||
"TreeSelect",
|
||||
"YearPicker",
|
||||
"YearPickerInput",
|
||||
] as const;
|
||||
|
||||
const CUSTOM_JSX_BINDING_TYPES: Readonly<Record<string, CustomJsxCatalogBindingType>> = {
|
||||
...Object.fromEntries(stringBindingComponents.map((name) => [name, "string"] as const)),
|
||||
Checkbox: "boolean",
|
||||
Menu: "boolean",
|
||||
Popover: "boolean",
|
||||
Switch: "boolean",
|
||||
NumberInput: "number",
|
||||
Pagination: "number",
|
||||
Rating: "number",
|
||||
Slider: "number",
|
||||
Stepper: "number",
|
||||
"Checkbox.Group": "string[]",
|
||||
CheckboxGroup: "string[]",
|
||||
MultiSelect: "string[]",
|
||||
"Switch.Group": "string[]",
|
||||
SwitchGroup: "string[]",
|
||||
TagsInput: "string[]",
|
||||
RangeSlider: "number[]",
|
||||
};
|
||||
|
||||
const HOMARR_COMPONENT_DESCRIPTIONS: Readonly<Record<string, string>> = {
|
||||
PaginatedList: "Renders a bounded page of collection items.",
|
||||
TabsContainer: "Renders a declarative tab container.",
|
||||
TabPanel: "Renders content for one declarative tab.",
|
||||
Collapsible: "Shows and hides a bounded content section.",
|
||||
StatBar: "Displays a compact labeled statistic and progress value.",
|
||||
TypeBadge: "Displays a compact semantic type badge.",
|
||||
SubFetch: "Runs a named manual query with invocation-time parameters.",
|
||||
SubData: "Reads a nested value from a named manual-query result.",
|
||||
ActionButton: "Runs a named user-triggered action.",
|
||||
ToggleSwitch: "Runs named actions for the enabled and disabled states.",
|
||||
RefreshButton: "Refreshes one active named query or all widget queries.",
|
||||
TablerIcon: "Resolves an installed Tabler icon by its safe name.",
|
||||
};
|
||||
|
||||
type HomarrPropMetadata = Omit<UninternedPropDescriptor, "name" | "source">;
|
||||
|
||||
const optionalProp = (
|
||||
type: string,
|
||||
literalValues?: CustomJsxLiteralValue[],
|
||||
description?: string,
|
||||
): HomarrPropMetadata => ({
|
||||
type,
|
||||
required: false,
|
||||
...(literalValues ? { literalValues } : {}),
|
||||
...(description ? { description } : {}),
|
||||
});
|
||||
const requiredProp = (
|
||||
type: string,
|
||||
literalValues?: CustomJsxLiteralValue[],
|
||||
description?: string,
|
||||
): HomarrPropMetadata => ({
|
||||
type,
|
||||
required: true,
|
||||
...(literalValues ? { literalValues } : {}),
|
||||
...(description ? { description } : {}),
|
||||
});
|
||||
|
||||
const HOMARR_COMPONENT_PROPS: Readonly<Record<string, Readonly<Record<string, HomarrPropMetadata>>>> = {
|
||||
PaginatedList: { pageSize: optionalProp("number") },
|
||||
TabsContainer: { defaultTab: optionalProp("string") },
|
||||
TabPanel: { value: requiredProp("string"), label: optionalProp("string") },
|
||||
Collapsible: { title: requiredProp("string"), defaultOpen: optionalProp("boolean") },
|
||||
StatBar: {
|
||||
value: requiredProp("number"),
|
||||
max: optionalProp("number"),
|
||||
label: optionalProp("string"),
|
||||
color: optionalProp("string"),
|
||||
},
|
||||
TypeBadge: {
|
||||
type: requiredProp("string"),
|
||||
size: optionalProp("'xs' | 'sm' | 'md' | 'lg' | 'xl'", ["xs", "sm", "md", "lg", "xl"]),
|
||||
},
|
||||
SubFetch: {
|
||||
requestId: optionalProp("string", undefined, "Identifier of the named Custom Widget query."),
|
||||
params: optionalProp("Record<string, string | number | boolean>"),
|
||||
refreshInterval: optionalProp("number"),
|
||||
loadingLabel: optionalProp("string"),
|
||||
errorMessage: optionalProp("string"),
|
||||
fallback: optionalProp("ReactNode"),
|
||||
triggerContent: optionalProp("ReactNode", undefined, "Custom accessible trigger shown before a manual query runs."),
|
||||
triggerAriaLabel: optionalProp("string", undefined, "Accessible label for a custom manual-query trigger."),
|
||||
path: optionalProp("string"),
|
||||
as: optionalProp("'json' | 'text'", ["json", "text"]),
|
||||
trigger: optionalProp("'auto' | 'manual'", ["auto", "manual"]),
|
||||
},
|
||||
SubData: {
|
||||
path: optionalProp("string"),
|
||||
as: optionalProp("'Text' | 'Title' | 'Badge' | 'Code' | 'Image'", ["Text", "Title", "Badge", "Code", "Image"]),
|
||||
order: optionalProp("1 | 2 | 3 | 4 | 5 | 6", [1, 2, 3, 4, 5, 6]),
|
||||
size: optionalProp("string"),
|
||||
color: optionalProp("string"),
|
||||
variant: optionalProp("string"),
|
||||
fw: optionalProp("number"),
|
||||
c: optionalProp("string"),
|
||||
alt: optionalProp("string"),
|
||||
fit: optionalProp("string"),
|
||||
w: optionalProp("string | number"),
|
||||
h: optionalProp("string | number"),
|
||||
radius: optionalProp("string | number"),
|
||||
},
|
||||
ActionButton: {
|
||||
requestId: optionalProp("string", undefined, "Identifier of the named Custom Widget action."),
|
||||
params: optionalProp("Record<string, string | number | boolean>"),
|
||||
label: optionalProp("string"),
|
||||
color: optionalProp("string"),
|
||||
variant: optionalProp("string"),
|
||||
size: optionalProp("string"),
|
||||
confirmMessage: optionalProp("string"),
|
||||
successMessage: optionalProp("string"),
|
||||
errorMessage: optionalProp("string"),
|
||||
icon: optionalProp("string"),
|
||||
disabled: optionalProp("boolean | string"),
|
||||
fullWidth: optionalProp("boolean | string"),
|
||||
},
|
||||
ToggleSwitch: {
|
||||
requestId: optionalProp("string", undefined, "Identifier of the named Custom Widget action."),
|
||||
enabledParams: optionalProp("Record<string, string | number | boolean>"),
|
||||
disabledParams: optionalProp("Record<string, string | number | boolean>"),
|
||||
initialValue: optionalProp("boolean | string"),
|
||||
label: optionalProp("string"),
|
||||
color: optionalProp("string"),
|
||||
size: optionalProp("string"),
|
||||
errorMessage: optionalProp("string"),
|
||||
disabled: optionalProp("boolean | string"),
|
||||
},
|
||||
RefreshButton: {
|
||||
requestId: optionalProp("string", undefined, "Optional named query to refresh instead of all widget queries."),
|
||||
label: optionalProp("string"),
|
||||
color: optionalProp("string"),
|
||||
variant: optionalProp("string"),
|
||||
size: optionalProp("string"),
|
||||
},
|
||||
TablerIcon: {
|
||||
name: requiredProp("string"),
|
||||
size: optionalProp("string | number"),
|
||||
color: optionalProp("string"),
|
||||
stroke: optionalProp("string | number"),
|
||||
},
|
||||
};
|
||||
|
||||
const FALLBACK_PROP_TYPES: Readonly<Record<string, string>> = {
|
||||
key: "string | number",
|
||||
id: "string",
|
||||
style: "CSSProperties",
|
||||
title: "string",
|
||||
role: "AriaRole",
|
||||
"aria-label": "string",
|
||||
"aria-describedby": "string",
|
||||
"aria-hidden": "boolean | 'true' | 'false'",
|
||||
tabIndex: "number",
|
||||
requestId: "string",
|
||||
refreshInterval: "number",
|
||||
path: "string",
|
||||
name: "string",
|
||||
color: "MantineColor",
|
||||
radius: "MantineRadius",
|
||||
variant: "string",
|
||||
size: "MantineSize | string | number",
|
||||
defaultValue: "unknown",
|
||||
defaultChecked: "boolean",
|
||||
};
|
||||
|
||||
const FALLBACK_LITERAL_VALUES: Readonly<Record<string, readonly CustomJsxLiteralValue[]>> = {
|
||||
target: ["_blank", "_parent", "_self", "_top"],
|
||||
orientation: ["horizontal", "vertical"],
|
||||
};
|
||||
|
||||
const FALLBACK_PROP_DESCRIPTIONS: Readonly<Record<string, string>> = {
|
||||
key: "Stable identity used by React when rendering collections.",
|
||||
requestId: "Identifier of the named Custom Widget request.",
|
||||
bind: "Name of the temporary in-memory input value.",
|
||||
resetKey: "Scalar dependency that resets a temporary input to its declared default when changed.",
|
||||
};
|
||||
|
||||
await generateCatalog();
|
||||
@@ -0,0 +1,192 @@
|
||||
import { customJsxRequestSchema } from "../src/core/request-schema";
|
||||
import { executeCustomWidgetRequest } from "../src/server/request-executor";
|
||||
import { renderRequestTarget, resolveCustomWidgetRequestValues } from "../src/server/request-manifest";
|
||||
|
||||
if (process.argv.includes("--public-ai-apis-only")) {
|
||||
await verifyPublicAiEvaluationApis();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const pokemonListRequest = customJsxRequestSchema.parse({
|
||||
path: "/api/v2/pokemon",
|
||||
query: { limit: { $option: "limit" }, offset: 0 },
|
||||
});
|
||||
const pokemonDetailRequest = customJsxRequestSchema.parse({
|
||||
path: "/api/v2/pokemon/{param:name}",
|
||||
trigger: "manual",
|
||||
});
|
||||
|
||||
const listTarget = renderRequestTarget(
|
||||
"https://pokeapi.co",
|
||||
pokemonListRequest,
|
||||
resolveCustomWidgetRequestValues(pokemonListRequest, { limit: 3 }),
|
||||
);
|
||||
const list = await fetchJson(listTarget, {});
|
||||
if (!isRecord(list) || !Array.isArray(list.results) || list.results.length !== 3) {
|
||||
throw new Error("PokeAPI list response did not match the expected shape");
|
||||
}
|
||||
const firstName = isRecord(list.results[0]) && typeof list.results[0].name === "string" ? list.results[0].name : null;
|
||||
if (!firstName) throw new Error("PokeAPI list did not contain a Pokémon name");
|
||||
const detailTarget = renderRequestTarget(
|
||||
"https://pokeapi.co",
|
||||
pokemonDetailRequest,
|
||||
resolveCustomWidgetRequestValues(pokemonDetailRequest, {}, { name: firstName }),
|
||||
);
|
||||
const detail = await fetchJson(detailTarget, {});
|
||||
if (!isRecord(detail) || detail.name !== firstName || !Array.isArray(detail.stats)) {
|
||||
throw new Error("PokeAPI detail response did not match the expected shape");
|
||||
}
|
||||
process.stdout.write("PokeAPI list and detail requests passed\n");
|
||||
|
||||
const portainerBaseUrl = process.env.PORTAINER_BASE_URL;
|
||||
const portainerApiKey = process.env.PORTAINER_API_KEY;
|
||||
if (!portainerBaseUrl || !portainerApiKey) {
|
||||
if (process.argv.includes("--pokeapi-only")) {
|
||||
process.stdout.write("Portainer check skipped (--pokeapi-only)\n");
|
||||
process.exit(0);
|
||||
}
|
||||
throw new Error("PORTAINER_BASE_URL and PORTAINER_API_KEY are required for the Portainer reference check");
|
||||
}
|
||||
const endpointId = process.env.PORTAINER_ENDPOINT_ID ?? "2";
|
||||
const containersRequest = customJsxRequestSchema.parse({
|
||||
path: "/api/endpoints/{option:endpointId}/docker/containers/json",
|
||||
query: { all: true },
|
||||
});
|
||||
const containersTarget = renderRequestTarget(
|
||||
portainerBaseUrl,
|
||||
containersRequest,
|
||||
resolveCustomWidgetRequestValues(containersRequest, { endpointId }),
|
||||
);
|
||||
const containers = await fetchJson(containersTarget, { "X-API-Key": portainerApiKey });
|
||||
if (!Array.isArray(containers)) throw new Error("Portainer containers response was not an array");
|
||||
|
||||
const actionRequests = ["start", "stop", "restart"].map((action) =>
|
||||
customJsxRequestSchema.parse({
|
||||
kind: "action",
|
||||
method: "POST",
|
||||
path: `/api/endpoints/{option:endpointId}/docker/containers/{param:id}/${action}`,
|
||||
confirmation: `${action} this container?`,
|
||||
invalidates: ["containers"],
|
||||
}),
|
||||
);
|
||||
for (const request of actionRequests) {
|
||||
const target = renderRequestTarget(
|
||||
portainerBaseUrl,
|
||||
request,
|
||||
resolveCustomWidgetRequestValues(request, { endpointId }, { id: "container/id" }),
|
||||
);
|
||||
if (!target.pathname.includes("container%2Fid")) throw new Error("Portainer action target did not encode its ID");
|
||||
}
|
||||
process.stdout.write(
|
||||
`Portainer container query passed (${containers.length} containers); start/stop/restart targets passed\n`,
|
||||
);
|
||||
|
||||
const mutationContainerId = process.env.PORTAINER_TEST_CONTAINER_ID;
|
||||
if (mutationContainerId) {
|
||||
if (process.env.PORTAINER_ALLOW_MUTATIONS !== "1") {
|
||||
throw new Error("Set PORTAINER_ALLOW_MUTATIONS=1 to test actions against PORTAINER_TEST_CONTAINER_ID");
|
||||
}
|
||||
const container = containers.find(
|
||||
(candidate) =>
|
||||
isRecord(candidate) && (candidate.Id === mutationContainerId || candidate.ID === mutationContainerId),
|
||||
);
|
||||
if (!isRecord(container)) throw new Error("PORTAINER_TEST_CONTAINER_ID was not returned by the container query");
|
||||
const initiallyRunning =
|
||||
container.State === "running" || (isRecord(container.State) && container.State.Running === true);
|
||||
const actionOrder = initiallyRunning ? ["restart", "stop", "start"] : ["start", "restart", "stop"];
|
||||
try {
|
||||
for (const action of actionOrder) await runPortainerAction(action, mutationContainerId);
|
||||
} finally {
|
||||
await runPortainerAction(initiallyRunning ? "start" : "stop", mutationContainerId, true);
|
||||
}
|
||||
process.stdout.write(
|
||||
`Portainer ${actionOrder.join("/")} actions passed and the original running state was restored\n`,
|
||||
);
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
async function fetchJson(target: URL, headers: Record<string, string>): Promise<unknown> {
|
||||
const response = await fetch(target, { headers, signal: AbortSignal.timeout(15_000) });
|
||||
if (!response.ok) throw new Error(`Reference API returned HTTP ${response.status}`);
|
||||
return response.json() as Promise<unknown>;
|
||||
}
|
||||
|
||||
async function runPortainerAction(action: string, containerId: string, ignoreConflict = false) {
|
||||
const request = customJsxRequestSchema.parse({
|
||||
kind: "action",
|
||||
method: "POST",
|
||||
path: `/api/endpoints/{option:endpointId}/docker/containers/{param:id}/${action}`,
|
||||
});
|
||||
const target = renderRequestTarget(
|
||||
portainerBaseUrl,
|
||||
request,
|
||||
resolveCustomWidgetRequestValues(request, { endpointId }, { id: containerId }),
|
||||
);
|
||||
const response = await executeCustomWidgetRequest({
|
||||
baseUrl: portainerBaseUrl,
|
||||
targetUrl: target,
|
||||
method: "POST",
|
||||
networkScope: "private",
|
||||
kind: "action",
|
||||
auth: {
|
||||
type: "apiKeyHeader",
|
||||
headerName: "X-API-Key",
|
||||
secrets: [{ kind: "apiKey", value: portainerApiKey }],
|
||||
},
|
||||
});
|
||||
if (!response.ok && !(ignoreConflict && response.status === 304)) {
|
||||
throw new Error(`Portainer ${action} returned HTTP ${response.status}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function verifyPublicAiEvaluationApis() {
|
||||
const bored = await fetchJson(new URL("https://bored-api.appbrewery.com/random"), {});
|
||||
if (!isRecord(bored) || typeof bored.activity !== "string" || typeof bored.participants !== "number") {
|
||||
throw new Error("Bored response did not match the evaluation fixture contract");
|
||||
}
|
||||
process.stdout.write("Bored activity request passed\n");
|
||||
|
||||
const coinMarketCapRequest = customJsxRequestSchema.parse({
|
||||
path: "/public-api/v3/cryptocurrency/quotes/latest",
|
||||
query: { id: "1,1027,5426", convert: "USD" },
|
||||
});
|
||||
const coinMarketCapTarget = renderRequestTarget(
|
||||
"https://pro-api.coinmarketcap.com",
|
||||
coinMarketCapRequest,
|
||||
resolveCustomWidgetRequestValues(coinMarketCapRequest, {}),
|
||||
);
|
||||
const response = await fetchJson(coinMarketCapTarget, {});
|
||||
const assets = isRecord(response) ? response.data : undefined;
|
||||
if (
|
||||
!Array.isArray(assets) ||
|
||||
!["BTC", "ETH", "SOL"].every((symbol) =>
|
||||
assets.some((asset) => isRecord(asset) && asset.symbol === symbol && Array.isArray(asset.quote)),
|
||||
)
|
||||
) {
|
||||
throw new Error("CoinMarketCap response did not match the evaluation fixture contract");
|
||||
}
|
||||
process.stdout.write("CoinMarketCap keyless watchlist request passed\n");
|
||||
|
||||
const agifyApiKey = process.env.AGIFY_API_KEY;
|
||||
if (!agifyApiKey) {
|
||||
process.stdout.write("Agify live request skipped (AGIFY_API_KEY is not configured)\n");
|
||||
return;
|
||||
}
|
||||
const agifyTarget = new URL("https://api.agify.io/");
|
||||
agifyTarget.searchParams.set("name", "michael");
|
||||
agifyTarget.searchParams.set("country_id", "US");
|
||||
agifyTarget.searchParams.set("apikey", agifyApiKey);
|
||||
const prediction = await fetchJson(agifyTarget, {});
|
||||
if (
|
||||
!isRecord(prediction) ||
|
||||
typeof prediction.name !== "string" ||
|
||||
!(typeof prediction.age === "number" || prediction.age === null) ||
|
||||
typeof prediction.count !== "number"
|
||||
) {
|
||||
throw new Error("Agify response did not match the evaluation fixture contract");
|
||||
}
|
||||
process.stdout.write("Agify authenticated lookup request passed\n");
|
||||
}
|
||||
Reference in new issue
Block a user