From 6eff1da08f087bf1446c531e8e178f27f0a6ddc5 Mon Sep 17 00:00:00 2001 From: ajnart Date: Thu, 30 Jul 2026 11:09:17 +0200 Subject: [PATCH] test: harden custom widget AI authoring --- .../custom-widgets/agent-authoring.mdx | 19 +- .../_custom-widget-form-utils.spec.ts | 22 +- .../_custom-widget-form-utils.ts | 7 + .../_use-custom-widget-form-actions.ts | 11 +- packages/custom-widgets/package.json | 7 +- .../scripts/ai-evaluation-cases.ts | 140 +++++++- .../custom-widgets/scripts/ai-evaluation.ts | 68 ++++ .../scripts/evaluate-mcp-agent.ts | 80 +++++ packages/custom-widgets/scripts/mcp-agent.ts | 287 ++++++++++++++++ .../src/test/ai-evaluation.spec.ts | 28 ++ .../custom-widgets/src/test/mcp-agent.spec.ts | 322 ++++++++++++++++++ packages/custom-widgets/tsconfig.scripts.json | 14 + pnpm-lock.yaml | 70 ++++ pnpm-workspace.yaml | 4 + 14 files changed, 1056 insertions(+), 23 deletions(-) create mode 100644 packages/custom-widgets/scripts/evaluate-mcp-agent.ts create mode 100644 packages/custom-widgets/scripts/mcp-agent.ts create mode 100644 packages/custom-widgets/src/test/mcp-agent.spec.ts create mode 100644 packages/custom-widgets/tsconfig.scripts.json diff --git a/apps/docs/docs/management/custom-widgets/agent-authoring.mdx b/apps/docs/docs/management/custom-widgets/agent-authoring.mdx index 2aa7562d8..108d82058 100644 --- a/apps/docs/docs/management/custom-widgets/agent-authoring.mdx +++ b/apps/docs/docs/management/custom-widgets/agent-authoring.mdx @@ -94,8 +94,25 @@ Homarr maintainers can run the same self-contained prompt that the workbench cop OPENROUTER_API_KEY=... pnpm --filter @homarr/custom-widgets test:ai ``` -The suite uses DeepSeek V4 Pro to author five complex widgets and DeepSeek V4 Flash to judge only widgets that pass Homarr's real parser and canonical schema. It saves every response, validated `widget.json`, `widget.jsx`, and detailed category scores under `packages/custom-widgets/.ai-evaluations/`. +The suite uses DeepSeek V4 Pro to author Pokédex, Portainer, football, Jellyfin, and Home Assistant widgets. Before DeepSeek V4 Flash judges a result, it must pass Homarr's real clipboard parser, canonical schema, and deterministic scenario checks for the documented API routes, authentication, bindings, actions, and essential runtime components. It saves every response, validated widget manifest, and detailed category scores under `packages/custom-widgets/.ai-evaluations/`. The run has a hard budget of ten generation attempts across all scenarios. Override the models or budget with `OPENROUTER_GENERATOR_MODEL`, `OPENROUTER_JUDGE_MODEL`, and `CUSTOM_WIDGET_AI_MAX_LOOPS`. Generated artifacts are local and ignored by Git; promote a reviewed, validated result into the bundled-widget source deliberately. +To exercise the connected MCP workflow with the Vercel AI SDK agent, start Homarr and create an API key for an administrator: + +```sh +OPENROUTER_API_KEY=... \ +HOMARR_MCP_URL=http://localhost:3000/api/mcp/mcp \ +HOMARR_MCP_API_KEY=... \ +pnpm --filter @homarr/custom-widgets evaluate:mcp -- --case=pokedex +``` + +This evaluator uses DeepSeek V4 Pro with `ToolLoopAgent` and a strict structured result. It requires successful schema +and validation results, one successful preview call for every scenario query and action, and a journal read after those +calls. It can repair and retest a widget, but it does not persist by default. Add `--persist` to create a passing widget, +or `--definition= --persist` to test the edit/debug/update path. Authenticated sources use +`customWidget_configurationRequestUser`; credential-writing tools are not exposed to the model and plaintext credentials +never enter its context. The deterministic unit suite separately proves the repair, persistence, and secure +source-configuration loop without contacting a model or changing a Homarr instance. + The reference API check uses a real PokeAPI list/detail request and a read-only Portainer container query when `PORTAINER_BASE_URL` and `PORTAINER_API_KEY` are set. Use `pnpm --filter @homarr/custom-widgets test:reference-apis -- --pokeapi-only` for the public PokeAPI check alone. To exercise start, restart, and stop against a disposable container, also set `PORTAINER_TEST_CONTAINER_ID` and the explicit safeguard `PORTAINER_ALLOW_MUTATIONS=1`; the script restores the container's original running state. diff --git a/apps/nextjs/src/app/[locale]/manage/custom-widgets/_custom-widget-form-utils.spec.ts b/apps/nextjs/src/app/[locale]/manage/custom-widgets/_custom-widget-form-utils.spec.ts index 4660e20f0..0d0e9b779 100644 --- a/apps/nextjs/src/app/[locale]/manage/custom-widgets/_custom-widget-form-utils.spec.ts +++ b/apps/nextjs/src/app/[locale]/manage/custom-widgets/_custom-widget-form-utils.spec.ts @@ -3,7 +3,7 @@ import { describe, expect, it, vi } from "vitest"; import { customWidgetDefinitionSchema } from "@homarr/custom-widgets/core"; import { - applyDefinition, + applyCustomWidgetAiResponse, filterSecretsForSourceAuthentication, getChangedSecrets, getCustomWidgetPreviewOptionIssues, @@ -24,9 +24,9 @@ const definition = customWidgetDefinitionSchema.parse({ }); describe("Custom Widget workbench preview options", () => { - it("loads a complete pasted widget template into the JSX editor field", () => { + it("loads one complete AI response into every editor field, including JSX", () => { const setValues = vi.fn(); - applyDefinition( + const result = applyCustomWidgetAiResponse( { values: { name: "", @@ -40,10 +40,22 @@ describe("Custom Widget workbench preview options", () => { }, setValues, } as never, - definition, + `\`\`\`json\n${JSON.stringify(definition)}\n\`\`\``, ); - expect(setValues).toHaveBeenCalledWith(expect.objectContaining({ template: "Preview" })); + if (!result.success) throw new Error("Expected the complete AI response to be accepted"); + expect(setValues).toHaveBeenCalledOnce(); + const values = setValues.mock.calls[0]?.[0]; + expect(values).toMatchObject({ + name: "Preview options", + description: "", + iconUrl: "", + template: "Preview", + secrets: [], + }); + expect(JSON.parse(values?.sources as string)).toEqual(result.widget.sources); + expect(JSON.parse(values?.requests as string)).toEqual(result.widget.requests); + expect(JSON.parse(values?.options as string)).toEqual(result.widget.options); }); it("shows the default source first regardless of manifest key order", () => { diff --git a/apps/nextjs/src/app/[locale]/manage/custom-widgets/_custom-widget-form-utils.ts b/apps/nextjs/src/app/[locale]/manage/custom-widgets/_custom-widget-form-utils.ts index 20df769cb..eba48aa67 100644 --- a/apps/nextjs/src/app/[locale]/manage/custom-widgets/_custom-widget-form-utils.ts +++ b/apps/nextjs/src/app/[locale]/manage/custom-widgets/_custom-widget-form-utils.ts @@ -3,6 +3,7 @@ import { customWidgetDefinitionSchema, getCustomWidgetDefaultOptions, getCustomWidgetRequiredSecretKinds, + parseCustomWidgetAiResponse, validateCustomWidgetOptions, } from "@homarr/custom-widgets/core"; import type { CustomWidgetSource, HomarrCustomWidgetV2 } from "@homarr/custom-widgets/core"; @@ -56,6 +57,12 @@ export function applyDefinition(form: CustomWidgetWorkbenchForm, widget: HomarrC }); } +export function applyCustomWidgetAiResponse(form: CustomWidgetWorkbenchForm, response: string) { + const result = parseCustomWidgetAiResponse(response); + if (result.success) applyDefinition(form, result.widget); + return result; +} + export function parseSources(value: string): Array { const parsed = parseJson(value); return isRecord(parsed) diff --git a/apps/nextjs/src/app/[locale]/manage/custom-widgets/_use-custom-widget-form-actions.ts b/apps/nextjs/src/app/[locale]/manage/custom-widgets/_use-custom-widget-form-actions.ts index 2e9a1b472..429f10ecb 100644 --- a/apps/nextjs/src/app/[locale]/manage/custom-widgets/_use-custom-widget-form-actions.ts +++ b/apps/nextjs/src/app/[locale]/manage/custom-widgets/_use-custom-widget-form-actions.ts @@ -6,17 +6,13 @@ import { useRouter } from "next/navigation"; import type { UseFormReturnType } from "@mantine/form"; import { clientApi } from "@homarr/api/client"; -import { - customWidgetDefinitionSchema, - formatCustomWidgetImportIssues, - parseCustomWidgetAiResponse, -} from "@homarr/custom-widgets/core"; +import { formatCustomWidgetImportIssues } from "@homarr/custom-widgets/core"; import type { CustomWidgetFormValues } from "@homarr/custom-widgets/workbench"; import { showErrorNotification, showSuccessNotification, showWarningNotification } from "@homarr/notifications"; import { useScopedI18n } from "@homarr/translation/client"; import { - applyDefinition, + applyCustomWidgetAiResponse, buildDefinition, getChangedSecrets, getDefinitionDefaults, @@ -167,11 +163,10 @@ export function useCustomWidgetFormActions(input: FormActionsInput) { const pasteAiResponse = async () => { try { - const result = parseCustomWidgetAiResponse(await navigator.clipboard.readText()); + const result = applyCustomWidgetAiResponse(input.form, await navigator.clipboard.readText()); if (!result.success) { throw new Error(formatCustomWidgetImportIssues(result.issues)); } - applyDefinition(input.form, customWidgetDefinitionSchema.parse(result.widget)); input.setOptionsSnapshot(getDefinitionDefaults(result.widget)); input.setPreview({ data: {}, status: {}, session: null, outcome: "idle" }); showSuccessNotification({ diff --git a/packages/custom-widgets/package.json b/packages/custom-widgets/package.json index 5f3eeb3cd..c54bafd74 100644 --- a/packages/custom-widgets/package.json +++ b/packages/custom-widgets/package.json @@ -29,12 +29,13 @@ "generate:components": "pnpm generate:catalog", "generate": "pnpm generate:catalog", "evaluate:ai": "bun ./scripts/evaluate-ai-authoring.ts", + "evaluate:mcp": "tsx ./scripts/evaluate-mcp-agent.ts", "test:ai": "bun ./scripts/evaluate-ai-authoring.ts --ci", "test:reference-apis": "tsx ./scripts/verify-reference-apis.ts", "format": "oxfmt .", "lint": "oxlint", "test": "vitest run", - "typecheck": "tsc --noEmit" + "typecheck": "tsc --noEmit && tsc --noEmit -p tsconfig.scripts.json" }, "dependencies": { "@codemirror/autocomplete": "catalog:", @@ -61,10 +62,14 @@ "zod": "catalog:" }, "devDependencies": { + "@ai-sdk/mcp": "catalog:", + "@ai-sdk/provider": "catalog:", "@homarr/tsconfig": "workspace:^0.1.0", + "@openrouter/ai-sdk-provider": "catalog:", "@types/node": "catalog:", "@types/react": "catalog:", "@types/react-dom": "catalog:", + "ai": "catalog:", "jsdom": "catalog:", "react-dom": "catalog:", "typescript": "catalog:", diff --git a/packages/custom-widgets/scripts/ai-evaluation-cases.ts b/packages/custom-widgets/scripts/ai-evaluation-cases.ts index 975598ebb..73e9b87b7 100644 --- a/packages/custom-widgets/scripts/ai-evaluation-cases.ts +++ b/packages/custom-widgets/scripts/ai-evaluation-cases.ts @@ -3,6 +3,20 @@ export interface CustomWidgetAiEvaluationCase { request: string; documentationUrl: string; apiNotes: string; + acceptance: { + sourceAuth?: { type: "none" | "bearer" | "apiKeyHeader" | "apiKeyQuery"; name?: string }; + requestRules: ReadonlyArray<{ + label: string; + pathIncludes: string; + kind?: "query" | "action"; + method?: "GET" | "POST" | "PUT" | "DELETE" | "PATCH"; + trigger?: "load" | "manual"; + optionReference?: boolean; + parameterReference?: boolean; + invalidates?: boolean; + }>; + templateComponents: readonly string[]; + }; } export const CUSTOM_WIDGET_AI_EVALUATION_CASES: readonly CustomWidgetAiEvaluationCase[] = [ @@ -13,6 +27,20 @@ export const CUSTOM_WIDGET_AI_EVALUATION_CASES: readonly CustomWidgetAiEvaluatio "Create a polished Pokédex browser with searchable Pokémon, a responsive result grid, manual detail loading, sprites, types, abilities, base-stat progress bars, and clear loading, empty, and failure states.", apiNotes: "Use GET /api/v2/pokemon?limit=