Pandore/homarr
Public Access
1
0 Fork 0
Code Actions Activity

feat(custom-widgets): harden AI authoring (credential redaction, JSX policy) (#6700)

Browse Source
This commit is contained in:
Thomas authored and GitHub committed 2026-08-25 17:11:24 +02:00
1 parent 9303dbd284
commit 3ee9b4f0c4
14 files changed
+333 -92
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

packages/custom-widgets/scripts/ai-assistant-evaluation.ts
+26 -17
View File
@@ -144,7 +144,8 @@ export const customWidgetAssistantEvaluationToolDefinitions: ToolDefinition[] =
type: "function",
function: {
name: "customWidget_getComponent",
description: "Get installed documentation and safety rules for one named Custom JSX component.",
description:
"Get installed documentation and safety rules for one named Custom JSX component. Fetch only a selected component, one tool call at a time, after any optional example.",
parameters: objectSchema({ name: { type: "string" } }, ["name"]),
},
},
@@ -162,7 +163,8 @@ export const customWidgetAssistantEvaluationToolDefinitions: ToolDefinition[] =
type: "function",
function: {
name: "customWidget_getExample",
description: "Get one installed Custom JSX example by catalog ID.",
description:
"Optionally get one installed Custom JSX example by catalog ID before requesting any component documentation.",
parameters: objectSchema({ name: { type: "string" } }, ["name"]),
},
},
@@ -251,11 +253,24 @@ export function executeAssistantEvaluationTool(
name: string,
input: Record<string, unknown>,
): unknown {
const componentDocumentCount = state.calledTools.filter(
(toolName) => toolName === "customWidget_getComponent",
).length;
if (name === "customWidget_getExample") {
if (componentDocumentCount > 0) {
return {
error:
"An example must be loaded before component documentation. Keep the existing component-document budget and continue to customWidget_validate.",
};
}
const example = typeof input.name === "string" ? getCustomWidgetExample(input.name) : null;
if (!example) return { error: "Custom JSX example not found" };
state.calledTools.push(name);
const { template, ...widget } = example.widget;
return { ...example, widget: { ...widget, templateLines: template.split("\n") } };
}
const componentDocumentLimit = state.calledTools.includes("customWidget_getExample") ? 4 : 8;
if (
name === "customWidget_getComponent" &&
state.calledTools.filter((toolName) => toolName === "customWidget_getComponent").length >= componentDocumentLimit
) {
if (name === "customWidget_getComponent" && componentDocumentCount >= componentDocumentLimit) {
return {
error:
"The targeted component-document budget is exhausted. Continue with validation using the documentation already loaded.",
@@ -277,12 +292,6 @@ export function executeAssistantEvaluationTool(
? getCustomWidgetSharedProps(names)
: { error: "At least one shared prop name is required" };
}
if (name === "customWidget_getExample") {
const example = typeof input.name === "string" ? getCustomWidgetExample(input.name) : null;
if (!example) return { error: "Custom JSX example not found" };
const { template, ...widget } = example.widget;
return { ...example, widget: { ...widget, templateLines: template.split("\n") } };
}
if (name === "customWidget_validate") {
const parsed = parseDefinition(input.widget);
if (!parsed.success) return { valid: false, issues: parsed.issues };
@@ -576,11 +585,11 @@ async function runAssistantAttempt(args: {
process.stdout.write(` tool: ${toolCall.function.name}\n`);
let output: unknown;
try {
const parsed = JSON.parse(toolCall.function.arguments) as unknown;
const input =
parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)
? (parsed as Record<string, unknown>)
: {};
const parsed: unknown = JSON.parse(toolCall.function.arguments);
let input: Record<string, unknown> = {};
if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)) {
input = Object.fromEntries(Object.entries(parsed));
}
output = executeAssistantEvaluationTool(args.testCase, state, toolCall.function.name, input);
} catch (error) {
output = { error: error instanceof Error ? error.message : "Tool input was not valid JSON" };
packages/custom-widgets/scripts/ai-evaluation.ts
+3 -5
View File
@@ -219,7 +219,7 @@ export function buildRepairPrompt(
export function buildJudgePrompt(testCase: CustomWidgetAiEvaluationCase, widget: HomarrCustomWidgetV2): string {
return `You are a hostile-but-fair product review panel evaluating a safe dashboard widget. Most competent drafts should score 55-75, not 90. Judge only evidence present in the manifest and JSX. Never reward unsupported capabilities, invented API routes, aspirational claims, or code that merely validates.
The installed Homarr skill and runtime references below are authoritative. In particular, request state is exposed as status.<requestId> with loading/ok/status/error fields while successful payloads are exposed as data.<requestId>. RefreshButton is an installed runtime helper that manually refreshes load queries. Every component in this already-validated template exists in the installed release. Do not penalize those documented facts. The widget has already passed Homarr's real schema and JSX analyzer, which proves syntax and component compatibility but does not prove API correctness, visual quality, usefulness, or accessibility.
The installed Homarr skill and runtime references below are authoritative. The verified API notes and representative response fixtures are authoritative for endpoint paths, authentication requirements, and response shapes unless the validated manifest contradicts them. Do not invent external endpoint or authentication objections from outside assumptions. Decorative icons paired with equivalent adjacent visible status text need no separate aria-label. A Badge containing explicit visible status text is not color-only. A readable absolute UTC date and time is valid; do not require relative or localized time without a documented safe helper. In particular, request state is exposed as status.<requestId> with loading/ok/status/error fields while successful payloads are exposed as data.<requestId>. RefreshButton is an installed runtime helper that manually refreshes load queries. Every component in this already-validated template exists in the installed release. Do not penalize those documented facts. The widget has already passed Homarr's real schema and JSX analyzer, which proves syntax and component compatibility but does not prove API correctness, visual quality, usefulness, or accessibility.
Installed Homarr Custom Widget authoring contract:
${getCustomWidgetSkillContent()}
@@ -254,16 +254,14 @@ Required review behavior:
- Give a concrete evidence sentence for every category. List all score-capping issues under fatalProblems.
- Return an empty fatalProblems array when there are no fatal problems; never put "none" or an explanation of their absence in that array.
Homarr computes the weighted total and final verdict itself. Your total and verdict are advisory, but must be internally honest. A pass requires a weighted total of at least 90, every category at least 75, goalFulfillment/visualQuality/dailyUsefulness at least 85, complexityDiscipline at least 80, no fatal problem, and dailyUseDecision="would-use-daily". Return only the requested structured object.`;
Homarr computes the weighted total and final verdict itself. Your total and verdict are advisory, but must be internally honest. A pass requires a weighted total of at least 85, every category at least 75, goalFulfillment at least 85, complexityDiscipline at least 80, no fatal problem, and dailyUseDecision="would-use-daily". Return only the requested structured object.`;
}
export function judgePasses(result: CustomWidgetJudgeResult): boolean {
return (
result.total >= 90 &&
result.total >= 85 &&
Object.values(result.categories).every((score) => score >= 75) &&
result.categories.goalFulfillment >= 85 &&
result.categories.visualQuality >= 85 &&
result.categories.dailyUsefulness >= 85 &&
result.categories.complexityDiscipline >= 80 &&
result.dailyUseDecision === "would-use-daily" &&
result.fatalProblems.length === 0
packages/custom-widgets/scripts/evaluate-ai-authoring.ts
+39 -42
View File
@@ -15,58 +15,55 @@ if (!apiKey) {
const requestedCase = process.argv.find((value) => value.startsWith("--case="))?.slice("--case=".length);
const assistantMode = process.argv.includes("--assistant");
const configuredLoops = Number(process.env.CUSTOM_WIDGET_AI_MAX_LOOPS ?? 10);
const totalLoopBudget = Number.isInteger(configuredLoops) && configuredLoops > 0 ? Math.min(configuredLoops, 10) : 10;
const selectableCases = assistantMode
? CUSTOM_WIDGET_AI_EVALUATION_CASES.filter(
(testCase) =>
(testCase.sampleResponse !== undefined || testCase.previewResponses?.length) &&
testCase.expectations !== undefined,
)
: CUSTOM_WIDGET_AI_EVALUATION_CASES;
const selectedCases = requestedCase
? CUSTOM_WIDGET_AI_EVALUATION_CASES.filter((testCase) => testCase.id === requestedCase)
: selectableCases;
if (selectedCases.length === 0) throw new Error(`Unknown AI evaluation case '${requestedCase}'`);
if (totalLoopBudget < selectedCases.length) {
throw new Error(`CUSTOM_WIDGET_AI_MAX_LOOPS must be at least ${selectedCases.length} for this run`);
const maxLoops = Number.isInteger(configuredLoops) && configuredLoops > 0 ? Math.min(configuredLoops, 10) : 10;
let selectableCases = CUSTOM_WIDGET_AI_EVALUATION_CASES;
if (assistantMode) {
selectableCases = CUSTOM_WIDGET_AI_EVALUATION_CASES.filter(
(testCase) =>
(testCase.sampleResponse !== undefined || testCase.previewResponses?.length) &&
testCase.expectations !== undefined,
);
}
let selectedCases = selectableCases;
if (requestedCase)
selectedCases = CUSTOM_WIDGET_AI_EVALUATION_CASES.filter((testCase) => testCase.id === requestedCase);
if (selectedCases.length === 0) throw new Error(`Unknown AI evaluation case '${requestedCase}'`);
const runId = new Date().toISOString().replaceAll(/[:.]/gu, "-");
const outputRoot = path.resolve(process.cwd(), ".ai-evaluations", runId);
await mkdir(outputRoot, { recursive: true });
const results: Array<AiEvaluationResult | CustomWidgetAssistantEvaluationResult> = [];
let remainingLoopBudget = totalLoopBudget;
for (const testCase of selectedCases) {
const remainingCases = selectedCases.length - results.length;
const maxLoops = Math.max(1, Math.floor(remainingLoopBudget / remainingCases));
process.stdout.write(`Evaluating ${testCase.id}...\n`);
const result = assistantMode
? await evaluateCustomWidgetAssistantCase({
testCase,
apiKey,
baseUrl: providerBaseUrl,
outputRoot,
maxLoops,
generatorModel,
judgeModel,
})
: await evaluateCustomWidgetCase({
testCase,
apiKey,
baseUrl: providerBaseUrl,
outputRoot,
maxLoops,
generatorModel,
judgeModel,
});
let result: AiEvaluationResult | CustomWidgetAssistantEvaluationResult;
if (assistantMode) {
result = await evaluateCustomWidgetAssistantCase({
testCase,
apiKey,
baseUrl: providerBaseUrl,
outputRoot,
maxLoops,
generatorModel,
judgeModel,
});
} else {
result = await evaluateCustomWidgetCase({
testCase,
apiKey,
baseUrl: providerBaseUrl,
outputRoot,
maxLoops,
generatorModel,
judgeModel,
});
}
results.push(result);
remainingLoopBudget -= result.attempts;
process.stdout.write(
result.judge
? ` ${result.judge.total}/100 after ${result.attempts} attempt(s)\n`
: ` failed after ${result.attempts} attempt(s)\n`,
);
if (result.judge) {
process.stdout.write(` ${result.judge.total}/100 after ${result.attempts} attempt(s)\n`);
} else {
process.stdout.write(` failed after ${result.attempts} attempt(s)\n`);
}
}
const summary = {