feat(assistant): optimize MCP and custom widget authoring (#6743)
* feat(assistant): optimize MCP and custom widget authoring * fix(assistant): isolate server-safe JSX analyzer * refactor(assistant): generate MCP tool staging * fix(assistant): preserve widget placement follow-up * fix(ci): allow fast gate to finish * fix(custom-widgets): address review findings * fix(custom-widgets): split preview evidence module * style: format assistant custom widget files * fix(custom-widgets): sync formatted skill schema
This commit is contained in:
Thomas authored and 
GitHub committed
1 parent
38f4b0e1f8
commit
5e23659d99
95 files changed
+6799
-1778
No files matched your search
File diff suppressed because it is too large.
Load diff
@@ -1,3 +1,26 @@
|
||||
export interface CustomWidgetAiExpectation {
|
||||
sourceBaseUrl: string;
|
||||
sourceNetworkScope?: "public" | "private" | "loopback";
|
||||
sourceAuth: "none" | "bearer" | "basic" | "apiKeyHeader" | "apiKeyQuery";
|
||||
sourceAuthName?: string;
|
||||
minimumTemplateCharacters?: number;
|
||||
requests: Array<{
|
||||
kind: "query" | "action";
|
||||
method: "GET" | "POST";
|
||||
pathIncludes: string;
|
||||
trigger?: "load" | "manual";
|
||||
permission?: "view" | "modify" | "full";
|
||||
queryIncludes?: Record<string, string | readonly string[]>;
|
||||
bodyIncludes?: Record<string, string | readonly string[]>;
|
||||
invalidates?: string[];
|
||||
invalidatesPaths?: string[];
|
||||
requiresConfirmation?: boolean;
|
||||
requiresStatusBinding?: boolean;
|
||||
}>;
|
||||
templateIncludes?: string[];
|
||||
templateIncludesAny?: string[][];
|
||||
}
|
||||
|
||||
export interface CustomWidgetAiEvaluationCase {
|
||||
id: string;
|
||||
request: string;
|
||||
@@ -6,24 +29,34 @@ export interface CustomWidgetAiEvaluationCase {
|
||||
sampleResponse?: unknown;
|
||||
previewResponses?: Array<{
|
||||
pathIncludes: string;
|
||||
kind?: "query" | "action";
|
||||
method?: "GET" | "POST";
|
||||
response: unknown;
|
||||
}>;
|
||||
minimumPreviewCycles?: number;
|
||||
expectations?: {
|
||||
sourceBaseUrl: string;
|
||||
sourceAuth: "none" | "apiKeyQuery";
|
||||
sourceAuthName?: string;
|
||||
requests: Array<{
|
||||
kind: "query" | "action";
|
||||
method: "GET" | "POST";
|
||||
pathIncludes: string;
|
||||
trigger?: "load" | "manual";
|
||||
queryIncludes?: Record<string, string>;
|
||||
}>;
|
||||
templateIncludes?: string[];
|
||||
expectations?: CustomWidgetAiExpectation;
|
||||
expectedWidgets?: Array<{
|
||||
id: string;
|
||||
request: string;
|
||||
apiNotes?: string;
|
||||
expectations: CustomWidgetAiExpectation;
|
||||
}>;
|
||||
research?: {
|
||||
query: string;
|
||||
requiredReferences: Array<"schema" | "runtime" | "security">;
|
||||
allowedReferences?: Array<"schema" | "runtime" | "security">;
|
||||
};
|
||||
}
|
||||
|
||||
const seerrSourceApiNotes =
|
||||
"Use the suggested self-hosted source http://seerr.local:5055/api/v1 with private network scope and X-Api-Key header authentication.";
|
||||
const seerrRequestOperationsApiNotes =
|
||||
"GET /request/count returns total, movie, tv, pending, approved, declined, processing, available, and completed counts. GET /request accepts take, skip, filter, sort, sortDirection, requestedBy, and mediaType; use fixed primitives take=10, skip=0, sort=added, and sortDirection=desc. Its response has pageInfo.pages/pageSize/results/page and a results array whose entries include id, type, numeric status, createdAt, profileName, requestedBy.displayName, and media.tmdbId/mediaType/status/status4k. Request status values are 1 Pending, 2 Approved, 3 Declined, 4 Failed, and 5 Completed. For managers, POST /request/{param:requestId}/approve and POST /request/{param:requestId}/decline change a pending request's status without a body; show them only for status 1, confirm both, and invalidate the count and list queries. This scoped widget intentionally uses text pageInfo context and does not require media-title joins, search requests, or interactive pagination.";
|
||||
const seerrMediaResearchApiNotes =
|
||||
"GET /search requires query and accepts optional page. Its entire response is an envelope with page, totalPages, totalResults, and results; mixed result entries include id, mediaType, title or name, overview, posterPath, backdropPath, voteAverage, releaseDate or firstAirDate, and mediaInfo.status/status4k, so render the array at response.results rather than mapping the envelope. TMDB artwork paths are relative: build a full https://image.tmdb.org/t/p/w780 URL from backdropPath with posterPath fallback. POST /request requires mediaType and mediaId; TV requests can use seasons='all'. Both request actions need modify permission, confirmation, and must invalidate the search query after success. This scoped widget intentionally does not require request counts, a request queue, status filters, or sorting.";
|
||||
const seerrMediaStatusApiNotes =
|
||||
"Both media.status and mediaInfo.status/status4k use 1 Unknown, 2 Pending, 3 Processing, 4 Partially Available, 5 Available, 6 Blocklisted, and 7 Deleted. Render readable labels rather than raw codes.";
|
||||
|
||||
export const CUSTOM_WIDGET_AI_EVALUATION_CASES: readonly CustomWidgetAiEvaluationCase[] = [
|
||||
{
|
||||
id: "pokedex",
|
||||
@@ -258,4 +291,271 @@ export const CUSTOM_WIDGET_AI_EVALUATION_CASES: readonly CustomWidgetAiEvaluatio
|
||||
templateIncludes: ["TextInput", "SubFetch", "age", "count"],
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "seerr-media-workflows",
|
||||
documentationUrl: "https://github.com/seerr-team/seerr/blob/develop/seerr-api.yml",
|
||||
request:
|
||||
"Research Seerr's current API once, then create two coordinated but independent Custom Widgets in this run. The first is a polished request-operations widget: prioritize total, pending, approved, and available in a responsive SimpleGrid, with Pending as the primary visual accent, Total as the strong anchor, and Approved/Available quieter; add compact processing and declined context, then show a recent queue with a two-level row hierarchy and strong identity such as Movie · TMDB 603 · Request #91. Give pending rows a restrained theme-aware accent. Keep requester and actions primary without repeating media type; use requestedBy.avatar only when present, set Avatar imageProps.alt from displayName, label profileName accurately as Media profile, and show missing data neutrally as Not provided. Separate identity from compact Created/Profile and Media/4K metadata groups plus a wrapping status/action row for narrow tiles. Show readable request/media/4K states, createdAt as clearly labeled UTC time, truthful pageInfo including total results, refresh, and safe confirmed approve and decline for pending requests. Preserve usable counts or queue when only its sibling request fails, and show subtle in-place refresh context when cached data remains. Give it resilient loading, empty, error, and success states. The second is an advanced media-research widget: use a bound search input and a manual SubFetch submit that never searches while typing and relies on native loading/error/retry, with functional pagination through bound request parameters. Reset pagination to page 1 when the query changes before the next manual run, keep a result-local control to rerun the same search, and render Pagination only when totalPages is greater than 1. Show exactly one concise active query/page/total-results summary. Keep its header to one compact line without explanatory copy. Render compact responsive results without redundant outer or per-item chrome; Image prefers backdropPath with posterPath fallback and uses a compact thumbnail beside content at base, widening only above xs without stacking a small capped image in a full-width slot. Clamp overview text, include release metadata, an explicit Rating label only when voteAverage is present, and compact readable mediaInfo plus 4K status badges, with resilient loading/no-results/error states and safe confirmed movie/full-series requests per result. Both must be excellent in narrow and wide tiles, independently validated and preview-tested, and persisted from their exact tested previews.",
|
||||
apiNotes: `${seerrSourceApiNotes} ${seerrRequestOperationsApiNotes} ${seerrMediaResearchApiNotes} ${seerrMediaStatusApiNotes}`,
|
||||
previewResponses: [
|
||||
{
|
||||
pathIncludes: "/request/count",
|
||||
kind: "query",
|
||||
method: "GET",
|
||||
response: {
|
||||
total: 42,
|
||||
movie: 25,
|
||||
tv: 17,
|
||||
pending: 4,
|
||||
approved: 9,
|
||||
declined: 2,
|
||||
processing: 6,
|
||||
available: 20,
|
||||
completed: 21,
|
||||
},
|
||||
},
|
||||
{
|
||||
pathIncludes: "/request",
|
||||
kind: "query",
|
||||
method: "GET",
|
||||
response: {
|
||||
pageInfo: { pages: 5, pageSize: 10, results: 42, page: 1 },
|
||||
results: [
|
||||
{
|
||||
id: 91,
|
||||
type: "movie",
|
||||
status: 2,
|
||||
createdAt: "2026-08-28T19:22:00.000Z",
|
||||
profileName: "HD-1080p",
|
||||
requestedBy: { id: 7, displayName: "Alex", avatar: "/avatar/alex" },
|
||||
media: { tmdbId: 603, mediaType: "movie", status: 3, status4k: 1 },
|
||||
},
|
||||
{
|
||||
id: 92,
|
||||
type: "tv",
|
||||
status: 1,
|
||||
createdAt: "2026-08-29T08:05:00.000Z",
|
||||
requestedBy: { id: 8, displayName: "Sam", avatar: "/avatar/sam" },
|
||||
media: { tmdbId: 1399, mediaType: "tv", status: 2, status4k: 1 },
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
pathIncludes: "/search",
|
||||
kind: "query",
|
||||
method: "GET",
|
||||
response: {
|
||||
page: 1,
|
||||
totalPages: 1,
|
||||
totalResults: 2,
|
||||
results: [
|
||||
{
|
||||
id: 603,
|
||||
mediaType: "movie",
|
||||
title: "The Matrix",
|
||||
overview: "A hacker discovers the world is a simulated reality.",
|
||||
posterPath: "/f89U3ADr1oiB1s9GkdPOEpXUk5H.jpg",
|
||||
backdropPath: "/icmmSD4vTTDKOq2vvdulafOGw93.jpg",
|
||||
voteAverage: 8.2,
|
||||
releaseDate: "1999-03-30",
|
||||
mediaInfo: { status: 3, status4k: 1 },
|
||||
},
|
||||
{
|
||||
id: 1399,
|
||||
mediaType: "tv",
|
||||
name: "Game of Thrones",
|
||||
overview: "Noble families compete for control of the realm.",
|
||||
posterPath: "/1XS1oqL89opfnbLl8WnZY1O1uJx.jpg",
|
||||
backdropPath: "/2OMB0ynKlyIenMJWI2Dy9IWT4c.jpg",
|
||||
voteAverage: 8.5,
|
||||
firstAirDate: "2011-04-17",
|
||||
mediaInfo: { status: 2, status4k: 1 },
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
pathIncludes: "/request",
|
||||
kind: "action",
|
||||
method: "POST",
|
||||
response: { id: 93, status: 1, type: "movie", media: { tmdbId: 603, mediaType: "movie" } },
|
||||
},
|
||||
{
|
||||
pathIncludes: "/request/{param:requestId}/approve",
|
||||
kind: "action",
|
||||
method: "POST",
|
||||
response: { id: 91, status: 2, type: "movie", modifiedBy: { displayName: "Admin" } },
|
||||
},
|
||||
{
|
||||
pathIncludes: "/request/{param:requestId}/decline",
|
||||
kind: "action",
|
||||
method: "POST",
|
||||
response: { id: 91, status: 3, type: "movie", modifiedBy: { displayName: "Admin" } },
|
||||
},
|
||||
],
|
||||
research: {
|
||||
query: "Seerr API search media create request get requests X-Api-Key",
|
||||
requiredReferences: ["schema", "runtime", "security"],
|
||||
allowedReferences: ["schema", "runtime", "security"],
|
||||
},
|
||||
expectedWidgets: [
|
||||
{
|
||||
id: "request-operations",
|
||||
request:
|
||||
"A Seerr request-operations widget with asymmetric Total/Pending priority plus quieter Approved/Available and processing/declined context, independently recoverable count/queue failures, subtle cached refresh context, divider-based rows with a pending accent, a narrow-first two-level queue, non-duplicated request/media identity, accessible requester/avatar and Media profile context, compact Created/Profile and Media/4K groups, readable request state, UTC time, truthful total-results page context, refresh, confirmed approve/decline for pending requests, and complete loading, empty, error, and success states.",
|
||||
apiNotes: `${seerrSourceApiNotes} ${seerrRequestOperationsApiNotes} ${seerrMediaStatusApiNotes}`,
|
||||
expectations: {
|
||||
sourceBaseUrl: "http://seerr.local:5055/api/v1",
|
||||
sourceNetworkScope: "private",
|
||||
sourceAuth: "apiKeyHeader",
|
||||
sourceAuthName: "X-Api-Key",
|
||||
minimumTemplateCharacters: 900,
|
||||
requests: [
|
||||
{
|
||||
kind: "query",
|
||||
method: "GET",
|
||||
pathIncludes: "/request/count",
|
||||
trigger: "load",
|
||||
requiresStatusBinding: true,
|
||||
},
|
||||
{
|
||||
kind: "query",
|
||||
method: "GET",
|
||||
pathIncludes: "/request",
|
||||
trigger: "load",
|
||||
queryIncludes: { take: "10", skip: "0", sort: "added", sortDirection: "desc" },
|
||||
requiresStatusBinding: true,
|
||||
},
|
||||
{
|
||||
kind: "action",
|
||||
method: "POST",
|
||||
pathIncludes: "/request/{param:requestId}/approve",
|
||||
trigger: "manual",
|
||||
permission: "modify",
|
||||
invalidatesPaths: ["/request/count", "/request"],
|
||||
requiresConfirmation: true,
|
||||
},
|
||||
{
|
||||
kind: "action",
|
||||
method: "POST",
|
||||
pathIncludes: "/request/{param:requestId}/decline",
|
||||
trigger: "manual",
|
||||
permission: "modify",
|
||||
invalidatesPaths: ["/request/count", "/request"],
|
||||
requiresConfirmation: true,
|
||||
},
|
||||
],
|
||||
templateIncludes: [
|
||||
"RefreshButton",
|
||||
"ActionButton",
|
||||
"SimpleGrid",
|
||||
"pageInfo",
|
||||
"results",
|
||||
"pending",
|
||||
"approved",
|
||||
"available",
|
||||
"processing",
|
||||
"declined",
|
||||
"createdAt",
|
||||
"Date.toLocaleString",
|
||||
"tmdbId",
|
||||
"status4k",
|
||||
"status.",
|
||||
"Failed",
|
||||
"Completed",
|
||||
"Not provided",
|
||||
"imageProps",
|
||||
],
|
||||
templateIncludesAny: [
|
||||
["TMDB", "Tmdb"],
|
||||
["Request #", "Request ID"],
|
||||
["UTC", "utc"],
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "media-research",
|
||||
request:
|
||||
"A compact Seerr media-research widget with manual bound search/pagination that resets to page 1 when the query changes, native SubFetch failure/retry, a result-local same-query rerun, conditional multi-page Pagination, one active query/page/total-results summary, a one-line header, a compact base thumbnail beside content that widens above xs, clamped overview, an explicit conditional Rating label, compact readable normal and 4K status badges, and confirmed direct requests for movies and full TV series.",
|
||||
apiNotes: `${seerrSourceApiNotes} ${seerrMediaResearchApiNotes} ${seerrMediaStatusApiNotes}`,
|
||||
expectations: {
|
||||
sourceBaseUrl: "http://seerr.local:5055/api/v1",
|
||||
sourceNetworkScope: "private",
|
||||
sourceAuth: "apiKeyHeader",
|
||||
sourceAuthName: "X-Api-Key",
|
||||
minimumTemplateCharacters: 900,
|
||||
requests: [
|
||||
{
|
||||
kind: "query",
|
||||
method: "GET",
|
||||
pathIncludes: "/search",
|
||||
trigger: "manual",
|
||||
queryIncludes: { query: "$param:query", page: "$param:page" },
|
||||
},
|
||||
{
|
||||
kind: "action",
|
||||
method: "POST",
|
||||
pathIncludes: "/request",
|
||||
trigger: "manual",
|
||||
permission: "modify",
|
||||
bodyIncludes: { mediaType: ["movie", "$param:*"], mediaId: "$param:*" },
|
||||
invalidatesPaths: ["/search"],
|
||||
requiresConfirmation: true,
|
||||
},
|
||||
{
|
||||
kind: "action",
|
||||
method: "POST",
|
||||
pathIncludes: "/request",
|
||||
trigger: "manual",
|
||||
permission: "modify",
|
||||
bodyIncludes: {
|
||||
mediaType: ["tv", "$param:*"],
|
||||
mediaId: "$param:*",
|
||||
seasons: ["all", "$param:*"],
|
||||
},
|
||||
invalidatesPaths: ["/search"],
|
||||
requiresConfirmation: true,
|
||||
},
|
||||
],
|
||||
templateIncludes: [
|
||||
"TextInput",
|
||||
"SubFetch",
|
||||
'trigger="manual"',
|
||||
"RefreshButton",
|
||||
'requestId="search"',
|
||||
"ActionButton",
|
||||
"Image",
|
||||
"Pagination",
|
||||
"defaultValue={1}",
|
||||
"resetKey",
|
||||
"inputs.page ?? 1",
|
||||
".results",
|
||||
"posterPath",
|
||||
"backdropPath",
|
||||
"https://image.tmdb.org/t/p/",
|
||||
"w780",
|
||||
"totalResults",
|
||||
"overview",
|
||||
"lineClamp",
|
||||
"voteAverage",
|
||||
"mediaInfo",
|
||||
"status4k",
|
||||
"Unknown",
|
||||
"Pending",
|
||||
"Processing",
|
||||
"Available",
|
||||
"Blocklisted",
|
||||
"Deleted",
|
||||
],
|
||||
templateIncludesAny: [
|
||||
["resetKey={inputs.query}", "resetKey={inputs.search}"],
|
||||
["Rating", "★", "/10"],
|
||||
["Partially Available", "Partially available"],
|
||||
["No matching", "No results", "No media", "Nothing found"],
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
] as const;
|
||||
@@ -4,11 +4,12 @@ import path from "node:path";
|
||||
import { z } from "zod/v4";
|
||||
|
||||
import { buildCustomWidgetAiPrompt } from "../src/core/ai-prompt";
|
||||
import { getCustomWidgetSkillContent } from "../src/core/authoring-resources";
|
||||
import { getCustomWidgetSkillReference } from "../src/core/authoring-resources";
|
||||
import { customWidgetDefinitionSchema } from "../src/core/custom-jsx-schema";
|
||||
import type { HomarrCustomWidgetV2 } from "../src/core/custom-jsx-schema";
|
||||
import { formatCustomWidgetImportIssues, parseCustomWidgetAiResponse } from "../src/core/import";
|
||||
import type { CustomWidgetAiEvaluationCase } from "./ai-evaluation-cases";
|
||||
import type { CustomWidgetAiExpectation } from "./ai-evaluation-cases";
|
||||
|
||||
// OpenRouter exposes the rolling "latest" route with a leading tilde. The non-tilde
|
||||
// deepseek/deepseek-v4-flash-latest alias is rejected by the chat-completions API.
|
||||
@@ -16,6 +17,18 @@ export const DEFAULT_GENERATOR_MODEL = "~deepseek/deepseek-v4-flash-latest";
|
||||
export const DEFAULT_JUDGE_MODEL = "~deepseek/deepseek-v4-flash-latest";
|
||||
export const DEFAULT_AI_PROVIDER_BASE_URL = "https://openrouter.ai/api/v1";
|
||||
export const MAX_AI_EVALUATION_LOOPS = 10;
|
||||
export function getAiEvaluationMaxOutputTokens(purpose: "generation" | "judge", configuredValue: string | undefined) {
|
||||
let defaultValue = 20_000;
|
||||
let minimum = 4_096;
|
||||
if (purpose === "judge") {
|
||||
defaultValue = 8_000;
|
||||
minimum = 2_000;
|
||||
}
|
||||
if (configuredValue === undefined) return defaultValue;
|
||||
const configured = Number(configuredValue);
|
||||
if (!Number.isInteger(configured) || configured <= 0) return defaultValue;
|
||||
return Math.min(defaultValue, Math.max(minimum, configured));
|
||||
}
|
||||
export const getAiProviderChatCompletionsUrl = (baseUrl = DEFAULT_AI_PROVIDER_BASE_URL) =>
|
||||
`${baseUrl.replace(/\/+$/u, "")}/chat/completions`;
|
||||
export const resolveAiEvaluationProviderConfig = (environment: Record<string, string | undefined>) => {
|
||||
@@ -128,19 +141,137 @@ export function getEvaluationResponseFixtureText(testCase: CustomWidgetAiEvaluat
|
||||
|
||||
const getExpectedBindingValue = (value: unknown): string => {
|
||||
if (typeof value === "string") return value;
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) return JSON.stringify(value);
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value))
|
||||
return JSON.stringify(value) ?? String(value);
|
||||
if ("$param" in value && typeof value.$param === "string") return `$param:${value.$param}`;
|
||||
if ("$option" in value && typeof value.$option === "string") return `$option:${value.$option}`;
|
||||
return JSON.stringify(value);
|
||||
};
|
||||
|
||||
const bindingMatchesExpectation = (
|
||||
value: unknown,
|
||||
expected: string | readonly string[],
|
||||
options: HomarrCustomWidgetV2["options"],
|
||||
): boolean => {
|
||||
if (Array.isArray(expected)) {
|
||||
return expected.some((candidate) => bindingMatchesExpectation(value, candidate, options));
|
||||
}
|
||||
const actual = getExpectedBindingValue(value);
|
||||
if (actual === expected) return true;
|
||||
if (expected === "$param:*" && actual.startsWith("$param:")) return true;
|
||||
if (expected === "$option:*" && actual.startsWith("$option:")) return true;
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
|
||||
if (!("$option" in value) || typeof value.$option !== "string") return false;
|
||||
const option = options[value.$option];
|
||||
if (!option) return false;
|
||||
return getExpectedBindingValue(option.default) === expected;
|
||||
};
|
||||
|
||||
export interface DeterministicEvaluationIssue {
|
||||
path?: Array<string | number>;
|
||||
message: string;
|
||||
}
|
||||
|
||||
export const getExpectedWidgetCase = (
|
||||
testCase: CustomWidgetAiEvaluationCase,
|
||||
expectedWidget: NonNullable<CustomWidgetAiEvaluationCase["expectedWidgets"]>[number],
|
||||
): CustomWidgetAiEvaluationCase => ({
|
||||
...testCase,
|
||||
id: `${testCase.id}-${expectedWidget.id}`,
|
||||
request: expectedWidget.request,
|
||||
apiNotes: expectedWidget.apiNotes ?? testCase.apiNotes,
|
||||
previewResponses: testCase.previewResponses?.filter((previewResponse) =>
|
||||
expectedWidget.expectations.requests.some(
|
||||
(request) =>
|
||||
request.pathIncludes === previewResponse.pathIncludes &&
|
||||
(previewResponse.kind === undefined || request.kind === previewResponse.kind) &&
|
||||
(previewResponse.method === undefined || request.method === previewResponse.method),
|
||||
),
|
||||
),
|
||||
expectations: expectedWidget.expectations,
|
||||
expectedWidgets: undefined,
|
||||
});
|
||||
|
||||
const getAuthName = (source: HomarrCustomWidgetV2["sources"][string] | undefined) =>
|
||||
typeof source?.auth === "object" ? source.auth.name : undefined;
|
||||
|
||||
const requestMatchesExpectation = (
|
||||
request: HomarrCustomWidgetV2["requests"][string],
|
||||
expected: CustomWidgetAiExpectation["requests"][number],
|
||||
options: HomarrCustomWidgetV2["options"],
|
||||
requests: HomarrCustomWidgetV2["requests"],
|
||||
) => {
|
||||
if (
|
||||
request.kind !== expected.kind ||
|
||||
request.method !== expected.method ||
|
||||
!request.path.includes(expected.pathIncludes) ||
|
||||
(expected.trigger !== undefined && request.trigger !== expected.trigger) ||
|
||||
(expected.permission !== undefined && request.permission !== expected.permission)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const queryMatches = Object.entries(expected.queryIncludes ?? {}).every(([key, value]) =>
|
||||
bindingMatchesExpectation(request.query?.[key], value, options),
|
||||
);
|
||||
if (!queryMatches) return false;
|
||||
const bodyMatches = Object.entries(expected.bodyIncludes ?? {}).every(([key, value]) => {
|
||||
if (typeof request.body !== "object" || request.body === null || Array.isArray(request.body)) return false;
|
||||
const body = request.body as Record<string, unknown>;
|
||||
return bindingMatchesExpectation(body[key], value, options);
|
||||
});
|
||||
if (!bodyMatches) return false;
|
||||
const invalidates = new Set(request.invalidates ?? []);
|
||||
if (!(expected.invalidates ?? []).every((requestId) => invalidates.has(requestId))) return false;
|
||||
const invalidatedRequests = [...invalidates]
|
||||
.map((requestId) => requests[requestId])
|
||||
.filter((candidate) => candidate !== undefined);
|
||||
if (
|
||||
!(expected.invalidatesPaths ?? []).every((path) => invalidatedRequests.some((candidate) => candidate.path === path))
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (expected.requiresConfirmation === true && request.confirmation === undefined) return false;
|
||||
return true;
|
||||
};
|
||||
|
||||
const getExpectedRequestConstraintSummary = (expected: CustomWidgetAiExpectation["requests"][number]) => {
|
||||
const formatBinding = (value: string | readonly string[]): string => {
|
||||
if (Array.isArray(value)) return value.map((candidate) => formatBinding(candidate)).join(" or ");
|
||||
if (value === "$param:*") return "any $param binding";
|
||||
if (value === "$option:*") return "any $option binding";
|
||||
return JSON.stringify(value);
|
||||
};
|
||||
const formatBindings = (bindings: Readonly<Record<string, string | readonly string[]>>) =>
|
||||
Object.entries(bindings)
|
||||
.map(([name, value]) => `${name}=${formatBinding(value)}`)
|
||||
.join(", ");
|
||||
const constraints: string[] = [];
|
||||
if (expected.trigger !== undefined) constraints.push(`trigger=${expected.trigger}`);
|
||||
if (expected.permission !== undefined) constraints.push(`permission=${expected.permission}`);
|
||||
if (expected.queryIncludes !== undefined) {
|
||||
constraints.push(`query bindings ${formatBindings(expected.queryIncludes)}`);
|
||||
}
|
||||
if (expected.bodyIncludes !== undefined) {
|
||||
constraints.push(`body bindings ${formatBindings(expected.bodyIncludes)}`);
|
||||
}
|
||||
if (expected.invalidatesPaths?.length) {
|
||||
constraints.push(`invalidates query request IDs for paths ${expected.invalidatesPaths.join(", ")}`);
|
||||
}
|
||||
if (expected.invalidates?.length) {
|
||||
constraints.push(`invalidates request IDs ${expected.invalidates.join(", ")}`);
|
||||
}
|
||||
if (expected.requiresConfirmation === true) constraints.push("confirmation is required");
|
||||
if (constraints.length === 0) return "";
|
||||
return ` Required: ${constraints.join("; ")}.`;
|
||||
};
|
||||
|
||||
export function getDeterministicEvaluationIssues(
|
||||
testCase: CustomWidgetAiEvaluationCase,
|
||||
widget: HomarrCustomWidgetV2,
|
||||
): Array<{ path?: Array<string | number>; message: string }> {
|
||||
): DeterministicEvaluationIssue[] {
|
||||
const expectations = testCase.expectations;
|
||||
if (!expectations) return [];
|
||||
const issues: Array<{ path?: Array<string | number>; message: string }> = [];
|
||||
const issues: DeterministicEvaluationIssue[] = [];
|
||||
const source = widget.sources.default;
|
||||
if (!source || source.baseUrl !== expectations.sourceBaseUrl) {
|
||||
issues.push({
|
||||
@@ -155,43 +286,49 @@ export function getDeterministicEvaluationIssues(
|
||||
message: `Use the verified ${expectations.sourceAuth} authentication mode.`,
|
||||
});
|
||||
}
|
||||
if (
|
||||
expectations.sourceAuth === "apiKeyQuery" &&
|
||||
typeof source?.auth === "object" &&
|
||||
source.auth.name !== expectations.sourceAuthName
|
||||
) {
|
||||
if (expectations.sourceNetworkScope !== undefined && source?.networkScope !== expectations.sourceNetworkScope) {
|
||||
issues.push({
|
||||
path: ["sources", "default", "networkScope"],
|
||||
message: `Use the verified ${expectations.sourceNetworkScope} network scope.`,
|
||||
});
|
||||
}
|
||||
if (expectations.sourceAuthName !== undefined && getAuthName(source) !== expectations.sourceAuthName) {
|
||||
issues.push({
|
||||
path: ["sources", "default", "auth", "name"],
|
||||
message: `Use '${expectations.sourceAuthName}' as the verified API-key query parameter.`,
|
||||
message: `Use '${expectations.sourceAuthName}' as the verified API-key name.`,
|
||||
});
|
||||
}
|
||||
if (
|
||||
expectations.minimumTemplateCharacters !== undefined &&
|
||||
widget.template.length < expectations.minimumTemplateCharacters
|
||||
) {
|
||||
issues.push({
|
||||
path: ["template"],
|
||||
message: `Build a substantive interface of at least ${expectations.minimumTemplateCharacters} JSX characters.`,
|
||||
});
|
||||
}
|
||||
const matchedRequestIds = new Set<string>();
|
||||
for (const expected of expectations.requests) {
|
||||
const match = Object.entries(widget.requests).find(([requestId, request]) => {
|
||||
if (matchedRequestIds.has(requestId)) return false;
|
||||
if (
|
||||
request.kind !== expected.kind ||
|
||||
request.method !== expected.method ||
|
||||
!request.path.includes(expected.pathIncludes) ||
|
||||
(expected.trigger !== undefined && request.trigger !== expected.trigger)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
return Object.entries(expected.queryIncludes ?? {}).every(
|
||||
([key, value]) => getExpectedBindingValue(request.query?.[key]) === value,
|
||||
);
|
||||
return requestMatchesExpectation(request, expected, widget.options, widget.requests);
|
||||
});
|
||||
if (!match) {
|
||||
issues.push({
|
||||
path: ["requests"],
|
||||
message:
|
||||
`Define the verified ${expected.trigger ?? ""} ${expected.kind} request ${expected.method} ${expected.pathIncludes} with its documented query bindings.`.replace(
|
||||
/\s+/gu,
|
||||
" ",
|
||||
),
|
||||
message: `${`Define the verified ${expected.trigger ?? ""} ${expected.kind} request ${expected.method} ${expected.pathIncludes}.`.replace(
|
||||
/\s+/gu,
|
||||
" ",
|
||||
)}${getExpectedRequestConstraintSummary(expected)}`,
|
||||
});
|
||||
} else {
|
||||
matchedRequestIds.add(match[0]);
|
||||
if (expected.requiresStatusBinding === true && !widget.template.includes(`status.${match[0]}`)) {
|
||||
issues.push({
|
||||
path: ["template"],
|
||||
message: `Read loading, error, and success state from status.${match[0]}; there is no global status.loading or status.ok.`,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const requiredText of expectations.templateIncludes ?? []) {
|
||||
@@ -202,6 +339,116 @@ export function getDeterministicEvaluationIssues(
|
||||
});
|
||||
}
|
||||
}
|
||||
for (const alternatives of expectations.templateIncludesAny ?? []) {
|
||||
if (alternatives.some((text) => widget.template.includes(text))) continue;
|
||||
issues.push({
|
||||
path: ["template"],
|
||||
message: `Render or use one equivalent capability: ${alternatives.map((text) => `'${text}'`).join(", ")}.`,
|
||||
});
|
||||
}
|
||||
for (const match of widget.template.matchAll(/<Pagination\b([^>]*)>/gu)) {
|
||||
const attributes = match[1] ?? "";
|
||||
const binding = attributes.match(/\bbind\s*=\s*["']([^"']+)["']/u)?.[1];
|
||||
if (binding && widget.template.includes(`inputs.${binding}`)) continue;
|
||||
issues.push({
|
||||
path: ["template"],
|
||||
message:
|
||||
"Wire Pagination with bind and use its inputs value in a supported request/helper, or render pagination context as text instead.",
|
||||
});
|
||||
}
|
||||
return issues;
|
||||
}
|
||||
|
||||
export interface DeterministicEvaluationMatch {
|
||||
expectedWidgetId: string;
|
||||
widgetIndex: number;
|
||||
testCase: CustomWidgetAiEvaluationCase;
|
||||
issues: DeterministicEvaluationIssue[];
|
||||
}
|
||||
|
||||
const getWidgetIndexPermutations = (widgetCount: number, expectedCount: number) => {
|
||||
const permutations: number[][] = [];
|
||||
const visit = (selected: number[]) => {
|
||||
if (selected.length === expectedCount) {
|
||||
permutations.push(selected);
|
||||
return;
|
||||
}
|
||||
for (let widgetIndex = 0; widgetIndex < widgetCount; widgetIndex += 1) {
|
||||
if (selected.includes(widgetIndex)) continue;
|
||||
visit([...selected, widgetIndex]);
|
||||
}
|
||||
};
|
||||
visit([]);
|
||||
return permutations;
|
||||
};
|
||||
|
||||
export function getDeterministicEvaluationMatches(
|
||||
testCase: CustomWidgetAiEvaluationCase,
|
||||
widgets: readonly HomarrCustomWidgetV2[],
|
||||
): DeterministicEvaluationMatch[] {
|
||||
const expectedWidgets = testCase.expectedWidgets;
|
||||
if (!expectedWidgets?.length) {
|
||||
const widget = widgets[0];
|
||||
if (!widget) return [];
|
||||
return [
|
||||
{
|
||||
expectedWidgetId: testCase.id,
|
||||
widgetIndex: 0,
|
||||
testCase,
|
||||
issues: getDeterministicEvaluationIssues(testCase, widget),
|
||||
},
|
||||
];
|
||||
}
|
||||
if (widgets.length < expectedWidgets.length) return [];
|
||||
let bestMatches: DeterministicEvaluationMatch[] = [];
|
||||
let bestIssueCount = Number.POSITIVE_INFINITY;
|
||||
for (const permutation of getWidgetIndexPermutations(widgets.length, expectedWidgets.length)) {
|
||||
const matches = expectedWidgets.map((expectedWidget, expectedIndex) => {
|
||||
const widgetIndex = permutation[expectedIndex] ?? -1;
|
||||
const expectedCase = getExpectedWidgetCase(testCase, expectedWidget);
|
||||
return {
|
||||
expectedWidgetId: expectedWidget.id,
|
||||
widgetIndex,
|
||||
testCase: expectedCase,
|
||||
issues: getDeterministicEvaluationIssues(expectedCase, widgets[widgetIndex] as HomarrCustomWidgetV2),
|
||||
};
|
||||
});
|
||||
const issueCount = matches.reduce((sum, match) => sum + match.issues.length, 0);
|
||||
if (issueCount >= bestIssueCount) continue;
|
||||
bestMatches = matches;
|
||||
bestIssueCount = issueCount;
|
||||
}
|
||||
return bestMatches;
|
||||
}
|
||||
|
||||
export function getDeterministicEvaluationSuiteIssues(
|
||||
testCase: CustomWidgetAiEvaluationCase,
|
||||
widgets: readonly HomarrCustomWidgetV2[],
|
||||
): DeterministicEvaluationIssue[] {
|
||||
if (!testCase.expectedWidgets?.length) {
|
||||
const widget = widgets[0];
|
||||
if (!widget) return [{ message: "Create the required widget." }];
|
||||
return getDeterministicEvaluationIssues(testCase, widget);
|
||||
}
|
||||
const issues: DeterministicEvaluationIssue[] = [];
|
||||
if (widgets.length !== testCase.expectedWidgets.length) {
|
||||
issues.push({
|
||||
message: `Create exactly ${testCase.expectedWidgets.length} independent widgets; received ${widgets.length}.`,
|
||||
});
|
||||
}
|
||||
const matches = getDeterministicEvaluationMatches(testCase, widgets);
|
||||
if (matches.length === 0) {
|
||||
issues.push({ message: "Every requested widget job needs its own complete manifest." });
|
||||
return issues;
|
||||
}
|
||||
for (const match of matches) {
|
||||
issues.push(
|
||||
...match.issues.map((issue) => ({
|
||||
...issue,
|
||||
message: `${match.expectedWidgetId}: ${issue.message}`,
|
||||
})),
|
||||
);
|
||||
}
|
||||
return issues;
|
||||
}
|
||||
|
||||
@@ -216,13 +463,24 @@ export function buildRepairPrompt(
|
||||
return `${originalPrompt}\n\nYour previous response did not validate. Correct only the generic contract/runtime problems below while preserving a polished design.\n\nDiagnostics:\n${diagnostics}\n\nPrevious response:\n${previousResponse}`;
|
||||
}
|
||||
|
||||
const getJudgeRuntimeContext = (widget: HomarrCustomWidgetV2) => {
|
||||
const sections = [getCustomWidgetSkillReference("runtime").content];
|
||||
const hasMutation = Object.values(widget.requests).some((request) => request.kind === "action");
|
||||
const hasProtectedSource = Object.values(widget.sources).some((source) => {
|
||||
const authType = typeof source.auth === "string" ? source.auth : source.auth.type;
|
||||
return authType !== "none";
|
||||
});
|
||||
if (hasMutation || hasProtectedSource) sections.push(getCustomWidgetSkillReference("security").content);
|
||||
return sections.join("\n\n");
|
||||
};
|
||||
|
||||
export function buildJudgePrompt(testCase: CustomWidgetAiEvaluationCase, widget: HomarrCustomWidgetV2): string {
|
||||
return `You are a hostile-but-fair product review panel evaluating a safe dashboard widget. Most competent drafts should score 55-75, not 90. Judge only evidence present in the manifest and JSX. Never reward unsupported capabilities, invented API routes, aspirational claims, or code that merely validates.
|
||||
|
||||
The installed Homarr skill and runtime references below are authoritative. The verified API notes and representative response fixtures are authoritative for endpoint paths, authentication requirements, and response shapes unless the validated manifest contradicts them. Do not invent external endpoint or authentication objections from outside assumptions. Decorative icons paired with equivalent adjacent visible status text need no separate aria-label. A Badge containing explicit visible status text is not color-only. A readable absolute UTC date and time is valid; do not require relative or localized time without a documented safe helper. In particular, request state is exposed as status.<requestId> with loading/ok/status/error fields while successful payloads are exposed as data.<requestId>. RefreshButton is an installed runtime helper that manually refreshes load queries. Every component in this already-validated template exists in the installed release. Do not penalize those documented facts. The widget has already passed Homarr's real schema and JSX analyzer, which proves syntax and component compatibility but does not prove API correctness, visual quality, usefulness, or accessibility.
|
||||
The installed Homarr skill and runtime references below are authoritative. The verified API notes and representative response fixtures are authoritative for endpoint paths, authentication requirements, and response shapes unless the validated manifest contradicts them. Do not invent external endpoint or authentication objections from outside assumptions. Judge only the scoped Request below; an endpoint mentioned in broader API notes is available, not automatically required, and an optional fixture field omitted by the scoped Request is not a missing capability. Decorative icons paired with equivalent adjacent visible status text need no separate aria-label. A Badge containing explicit visible status text is not color-only. Date.toLocaleString(value, "en-US", "UTC") is an installed safe static helper for concise absolute UTC timestamps; do not require relative time. In particular, request state is exposed as status.<requestId> with loading/ok/status/error fields while successful payloads are exposed as data.<requestId>. RefreshButton is an installed runtime helper: it refreshes load queries by default; inside a successful manual result, requestId targets and reruns that active query with unchanged parameters. A bound Pagination should declare defaultValue={1}; resetKey={inputs.query} restores that default when its dependent scalar query changes. SubFetch without trigger="manual" runs automatically and reruns when bound params change, so never demand a hidden input, debounce callback, or raw event. When the scoped Request requires manual search, do not penalize the required re-trigger after query or page changes. A manual SubFetch synchronously hides its old result and returns to its trigger when its request ID, normalized parameters, or effective definition changes; it cannot display stale results under edited inputs or fetch the new parameters before another trigger. ActionButton supplies pending UI, native success/error notification, confirmation, and declared invalidation; do not demand duplicate local action state. Safe templates forbid local declarations and helper functions, so do not penalize a repeated short literal label array used for distinct enum fields as an avoidable missing abstraction. Every component in this already-validated template exists in the installed release. Do not penalize those documented facts. The widget has already passed Homarr's real schema and JSX analyzer, which proves syntax and component compatibility but does not prove API correctness, visual quality, usefulness, or accessibility.
|
||||
|
||||
Installed Homarr Custom Widget authoring contract:
|
||||
${getCustomWidgetSkillContent()}
|
||||
Task-relevant installed Homarr runtime references:
|
||||
${getJudgeRuntimeContext(widget)}
|
||||
|
||||
Request:
|
||||
${testCase.request}
|
||||
@@ -247,7 +505,9 @@ Required review behavior:
|
||||
- Compare every requested capability with concrete manifest/JSX evidence. A missing or invented core capability is fatal and caps total at 79.
|
||||
- Judge whether the API design can actually reach the stated goal, including response paths, bindings, invalidation, and action safety.
|
||||
- Judge visual quality, not component count: hierarchy, density, whitespace, typography, restrained color, scanability, and avoidance of repetitive nested cards.
|
||||
- A purpose-specific asymmetric summary, divided hierarchy, responsive density, and restrained semantic accents can clear 75 without decorative chrome. Do not demand gradients, novelty, or a generic selected-row detail interaction.
|
||||
- Judge daily usefulness: information priority, interaction cost, refresh behavior, narrow-tile usability, and whether the widget is pleasant rather than demo-like.
|
||||
- A required manual search rerun is deliberate interaction, not daily-use friction. SubFetch owns failure and retry before its child renders; never demand an unreachable child error branch or a RefreshButton there.
|
||||
- Judge complexity discipline: penalize duplicate requests/options, unnecessary controls, excessive JSX, cleverness, and UI chrome that does not help the goal. Complexity must earn its place.
|
||||
- Recommend only interactions supported by the installed authoring contract. Do not suggest portals, modals, arbitrary event handlers, or other blocked capabilities. Prefer a responsive in-widget detail area when separation is useful.
|
||||
- A visually generic but valid widget should normally score below 75 for visualQuality. A widget that is attractive but inconvenient should score below 75 for dailyUsefulness.
|
||||
@@ -419,7 +679,7 @@ export function parseJudgeResult(raw: string): CustomWidgetJudgeResult {
|
||||
const parsed: unknown = JSON.parse(normalizedRaw);
|
||||
const result = judgeResultSchema.parse(parsed);
|
||||
const fatalProblems = result.fatalProblems.filter(
|
||||
(problem) => !/^(?:none|no fatal problems?)(?:\b|\s*[:.-])/iu.test(problem.trim()),
|
||||
(problem) => !/^(?:none|no fatal (?:problems?|issues?))(?:\b|\s*[:.-])/iu.test(problem.trim()),
|
||||
);
|
||||
const weightedTotal = Object.entries(categoryWeights).reduce(
|
||||
(sum, [category, weight]) => sum + result.categories[category as keyof typeof categoryWeights] * weight,
|
||||
@@ -445,6 +705,8 @@ async function callOpenRouter(args: {
|
||||
purpose: "generation" | "judge";
|
||||
}): Promise<string> {
|
||||
const isJudge = args.purpose === "judge";
|
||||
let configuredMaxOutputTokens = process.env.CUSTOM_WIDGET_AI_GENERATION_MAX_OUTPUT_TOKENS;
|
||||
if (isJudge) configuredMaxOutputTokens = process.env.CUSTOM_WIDGET_AI_JUDGE_MAX_OUTPUT_TOKENS;
|
||||
const response = await fetch(getAiProviderChatCompletionsUrl(args.baseUrl), {
|
||||
method: "POST",
|
||||
headers: {
|
||||
@@ -457,7 +719,7 @@ async function callOpenRouter(args: {
|
||||
model: args.model,
|
||||
messages: [{ role: "user", content: args.prompt }],
|
||||
temperature: isJudge ? 0 : 0.2,
|
||||
max_tokens: isJudge ? 8_000 : 20_000,
|
||||
max_tokens: getAiEvaluationMaxOutputTokens(args.purpose, configuredMaxOutputTokens),
|
||||
reasoning: isJudge ? { enabled: false, exclude: true } : { effort: "high", exclude: true },
|
||||
...(isJudge ? { response_format: getJudgeResponseFormat() } : {}),
|
||||
}),
|
||||
|
||||
@@ -16,23 +16,29 @@ const requestedCase = process.argv.find((value) => value.startsWith("--case="))?
|
||||
const assistantMode = process.argv.includes("--assistant");
|
||||
const configuredLoops = Number(process.env.CUSTOM_WIDGET_AI_MAX_LOOPS ?? 10);
|
||||
const maxLoops = Number.isInteger(configuredLoops) && configuredLoops > 0 ? Math.min(configuredLoops, 10) : 10;
|
||||
let selectableCases = CUSTOM_WIDGET_AI_EVALUATION_CASES;
|
||||
let selectableCases = CUSTOM_WIDGET_AI_EVALUATION_CASES.filter((testCase) => !testCase.expectedWidgets?.length);
|
||||
if (assistantMode) {
|
||||
selectableCases = CUSTOM_WIDGET_AI_EVALUATION_CASES.filter(
|
||||
(testCase) =>
|
||||
(testCase.sampleResponse !== undefined || testCase.previewResponses?.length) &&
|
||||
testCase.expectations !== undefined,
|
||||
(testCase.expectations !== undefined || testCase.expectedWidgets?.length),
|
||||
);
|
||||
}
|
||||
let selectedCases = selectableCases;
|
||||
if (requestedCase)
|
||||
selectedCases = CUSTOM_WIDGET_AI_EVALUATION_CASES.filter((testCase) => testCase.id === requestedCase);
|
||||
if (requestedCase) selectedCases = selectableCases.filter((testCase) => testCase.id === requestedCase);
|
||||
if (selectedCases.length === 0) throw new Error(`Unknown AI evaluation case '${requestedCase}'`);
|
||||
|
||||
const runId = new Date().toISOString().replaceAll(/[:.]/gu, "-");
|
||||
const outputRoot = path.resolve(process.cwd(), ".ai-evaluations", runId);
|
||||
await mkdir(outputRoot, { recursive: true });
|
||||
|
||||
function getResultScoreFloor(result: AiEvaluationResult | CustomWidgetAssistantEvaluationResult) {
|
||||
if ("judges" in result && result.judges.length > 0) {
|
||||
return Math.min(...result.judges.map((judge) => judge.total));
|
||||
}
|
||||
return result.judge?.total ?? null;
|
||||
}
|
||||
|
||||
const results: Array<AiEvaluationResult | CustomWidgetAssistantEvaluationResult> = [];
|
||||
for (const testCase of selectedCases) {
|
||||
process.stdout.write(`Evaluating ${testCase.id}...\n`);
|
||||
@@ -60,7 +66,12 @@ for (const testCase of selectedCases) {
|
||||
}
|
||||
results.push(result);
|
||||
if (result.judge) {
|
||||
process.stdout.write(` ${result.judge.total}/100 after ${result.attempts} attempt(s)\n`);
|
||||
const widgetCount = "widgets" in result ? result.widgets.length : 1;
|
||||
const scoreFloor = getResultScoreFloor(result);
|
||||
const status = result.judge.verdict === "pass" ? "passed" : "failed";
|
||||
process.stdout.write(
|
||||
` ${scoreFloor}/100 floor across ${widgetCount} widget(s), ${status} after ${result.attempts} attempt(s)\n`,
|
||||
);
|
||||
} else {
|
||||
process.stdout.write(` failed after ${result.attempts} attempt(s)\n`);
|
||||
}
|
||||
@@ -75,11 +86,14 @@ const summary = {
|
||||
results: results.map((result) => ({
|
||||
caseId: result.caseId,
|
||||
attempts: result.attempts,
|
||||
score: result.judge?.total ?? null,
|
||||
score: getResultScoreFloor(result),
|
||||
verdict: result.judge?.verdict ?? "fail",
|
||||
categories: result.judge?.categories ?? null,
|
||||
errors: result.errors,
|
||||
calledTools: "calledTools" in result ? result.calledTools : undefined,
|
||||
widgets: "widgets" in result ? result.widgets.length : 1,
|
||||
widgetScores: "judges" in result ? result.judges.map((judge) => judge.total) : undefined,
|
||||
efficiency: "efficiency" in result ? result.efficiency : undefined,
|
||||
outputDirectory: path.relative(process.cwd(), result.outputDirectory),
|
||||
})),
|
||||
};
|
||||
|
||||
@@ -4,7 +4,7 @@ import { resolve } from "node:path";
|
||||
import * as Charts from "@mantine/charts";
|
||||
import * as Core from "@mantine/core";
|
||||
import * as Dates from "@mantine/dates";
|
||||
import ts from "typescript";
|
||||
import ts from "typescript-compiler-api";
|
||||
|
||||
import {
|
||||
CUSTOM_JSX_AUTHORING_CATALOG_SCHEMA_VERSION,
|
||||
@@ -267,6 +267,7 @@ function buildComponentApi(
|
||||
bind: {
|
||||
type: bindType,
|
||||
initialProp: bindType === "boolean" ? ("defaultChecked" as const) : ("defaultValue" as const),
|
||||
resetProp: "resetKey" as const,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
@@ -587,7 +588,7 @@ const HOMARR_COMPONENT_DESCRIPTIONS: Readonly<Record<string, string>> = {
|
||||
SubData: "Reads a nested value from a named manual-query result.",
|
||||
ActionButton: "Runs a named user-triggered action.",
|
||||
ToggleSwitch: "Runs named actions for the enabled and disabled states.",
|
||||
RefreshButton: "Refreshes load-triggered widget queries.",
|
||||
RefreshButton: "Refreshes one active named query or all widget queries.",
|
||||
TablerIcon: "Resolves an installed Tabler icon by its safe name.",
|
||||
};
|
||||
|
||||
@@ -683,6 +684,7 @@ const HOMARR_COMPONENT_PROPS: Readonly<Record<string, Readonly<Record<string, Ho
|
||||
disabled: optionalProp("boolean | string"),
|
||||
},
|
||||
RefreshButton: {
|
||||
requestId: optionalProp("string", undefined, "Optional named query to refresh instead of all widget queries."),
|
||||
label: optionalProp("string"),
|
||||
color: optionalProp("string"),
|
||||
variant: optionalProp("string"),
|
||||
@@ -727,6 +729,7 @@ const FALLBACK_PROP_DESCRIPTIONS: Readonly<Record<string, string>> = {
|
||||
key: "Stable identity used by React when rendering collections.",
|
||||
requestId: "Identifier of the named Custom Widget request.",
|
||||
bind: "Name of the temporary in-memory input value.",
|
||||
resetKey: "Scalar dependency that resets a temporary input to its declared default when changed.",
|
||||
};
|
||||
|
||||
await generateCatalog();
|
||||
Reference in new issue
Block a user