test: harden custom widget AI authoring

apps/docs/docs/management/custom-widgets/agent-authoring.mdx

@@ -94,8 +94,25 @@ Homarr maintainers can run the same self-contained prompt that the workbench cop
 OPENROUTER_API_KEY=... pnpm --filter @homarr/custom-widgets test:ai
 ```
 
-The suite uses DeepSeek V4 Pro to author five complex widgets and DeepSeek V4 Flash to judge only widgets that pass Homarr's real parser and canonical schema. It saves every response, validated `widget.json`, `widget.jsx`, and detailed category scores under `packages/custom-widgets/.ai-evaluations/`.
+The suite uses DeepSeek V4 Pro to author Pokédex, Portainer, football, Jellyfin, and Home Assistant widgets. Before DeepSeek V4 Flash judges a result, it must pass Homarr's real clipboard parser, canonical schema, and deterministic scenario checks for the documented API routes, authentication, bindings, actions, and essential runtime components. It saves every response, validated widget manifest, and detailed category scores under `packages/custom-widgets/.ai-evaluations/`.
 
 The run has a hard budget of ten generation attempts across all scenarios. Override the models or budget with `OPENROUTER_GENERATOR_MODEL`, `OPENROUTER_JUDGE_MODEL`, and `CUSTOM_WIDGET_AI_MAX_LOOPS`. Generated artifacts are local and ignored by Git; promote a reviewed, validated result into the bundled-widget source deliberately.
 
+To exercise the connected MCP workflow with the Vercel AI SDK agent, start Homarr and create an API key for an administrator:
+
+```sh
+OPENROUTER_API_KEY=... \
+HOMARR_MCP_URL=http://localhost:3000/api/mcp/mcp \
+HOMARR_MCP_API_KEY=... \
+pnpm --filter @homarr/custom-widgets evaluate:mcp -- --case=pokedex
+```
+
+This evaluator uses DeepSeek V4 Pro with `ToolLoopAgent` and a strict structured result. It requires successful schema
+and validation results, one successful preview call for every scenario query and action, and a journal read after those
+calls. It can repair and retest a widget, but it does not persist by default. Add `--persist` to create a passing widget,
+or `--definition=<id> --persist` to test the edit/debug/update path. Authenticated sources use
+`customWidget_configurationRequestUser`; credential-writing tools are not exposed to the model and plaintext credentials
+never enter its context. The deterministic unit suite separately proves the repair, persistence, and secure
+source-configuration loop without contacting a model or changing a Homarr instance.
+
 The reference API check uses a real PokeAPI list/detail request and a read-only Portainer container query when `PORTAINER_BASE_URL` and `PORTAINER_API_KEY` are set. Use `pnpm --filter @homarr/custom-widgets test:reference-apis -- --pokeapi-only` for the public PokeAPI check alone. To exercise start, restart, and stop against a disposable container, also set `PORTAINER_TEST_CONTAINER_ID` and the explicit safeguard `PORTAINER_ALLOW_MUTATIONS=1`; the script restores the container's original running state.

apps/nextjs/src/app/[locale]/manage/custom-widgets/_custom-widget-form-utils.spec.ts

@@ -3,7 +3,7 @@ import { describe, expect, it, vi } from "vitest";
 import { customWidgetDefinitionSchema } from "@homarr/custom-widgets/core";
 
 import {
-  applyDefinition,
+  applyCustomWidgetAiResponse,
   filterSecretsForSourceAuthentication,
   getChangedSecrets,
   getCustomWidgetPreviewOptionIssues,
@@ -24,9 +24,9 @@ const definition = customWidgetDefinitionSchema.parse({
 });
 
 describe("Custom Widget workbench preview options", () => {
-  it("loads a complete pasted widget template into the JSX editor field", () => {
+  it("loads one complete AI response into every editor field, including JSX", () => {
     const setValues = vi.fn();
-    applyDefinition(
+    const result = applyCustomWidgetAiResponse(
       {
         values: {
           name: "",
@@ -40,10 +40,22 @@ describe("Custom Widget workbench preview options", () => {
         },
         setValues,
       } as never,
-      definition,
+      `\`\`\`json\n${JSON.stringify(definition)}\n\`\`\``,
     );
 
-    expect(setValues).toHaveBeenCalledWith(expect.objectContaining({ template: "<Text>Preview</Text>" }));
+    if (!result.success) throw new Error("Expected the complete AI response to be accepted");
+    expect(setValues).toHaveBeenCalledOnce();
+    const values = setValues.mock.calls[0]?.[0];
+    expect(values).toMatchObject({
+      name: "Preview options",
+      description: "",
+      iconUrl: "",
+      template: "<Text>Preview</Text>",
+      secrets: [],
+    });
+    expect(JSON.parse(values?.sources as string)).toEqual(result.widget.sources);
+    expect(JSON.parse(values?.requests as string)).toEqual(result.widget.requests);
+    expect(JSON.parse(values?.options as string)).toEqual(result.widget.options);
   });
 
   it("shows the default source first regardless of manifest key order", () => {

apps/nextjs/src/app/[locale]/manage/custom-widgets/_custom-widget-form-utils.ts

@@ -3,6 +3,7 @@ import {
   customWidgetDefinitionSchema,
   getCustomWidgetDefaultOptions,
   getCustomWidgetRequiredSecretKinds,
+  parseCustomWidgetAiResponse,
   validateCustomWidgetOptions,
 } from "@homarr/custom-widgets/core";
 import type { CustomWidgetSource, HomarrCustomWidgetV2 } from "@homarr/custom-widgets/core";
@@ -56,6 +57,12 @@ export function applyDefinition(form: CustomWidgetWorkbenchForm, widget: HomarrC
   });
 }
 
+export function applyCustomWidgetAiResponse(form: CustomWidgetWorkbenchForm, response: string) {
+  const result = parseCustomWidgetAiResponse(response);
+  if (result.success) applyDefinition(form, result.widget);
+  return result;
+}
+
 export function parseSources(value: string): Array<CustomWidgetSource & { id: string }> {
   const parsed = parseJson(value);
   return isRecord(parsed)

apps/nextjs/src/app/[locale]/manage/custom-widgets/_use-custom-widget-form-actions.ts

@@ -6,17 +6,13 @@ import { useRouter } from "next/navigation";
 import type { UseFormReturnType } from "@mantine/form";
 
 import { clientApi } from "@homarr/api/client";
-import {
-  customWidgetDefinitionSchema,
-  formatCustomWidgetImportIssues,
-  parseCustomWidgetAiResponse,
-} from "@homarr/custom-widgets/core";
+import { formatCustomWidgetImportIssues } from "@homarr/custom-widgets/core";
 import type { CustomWidgetFormValues } from "@homarr/custom-widgets/workbench";
 import { showErrorNotification, showSuccessNotification, showWarningNotification } from "@homarr/notifications";
 import { useScopedI18n } from "@homarr/translation/client";
 
 import {
-  applyDefinition,
+  applyCustomWidgetAiResponse,
   buildDefinition,
   getChangedSecrets,
   getDefinitionDefaults,
@@ -167,11 +163,10 @@ export function useCustomWidgetFormActions(input: FormActionsInput) {
 
   const pasteAiResponse = async () => {
     try {
-      const result = parseCustomWidgetAiResponse(await navigator.clipboard.readText());
+      const result = applyCustomWidgetAiResponse(input.form, await navigator.clipboard.readText());
       if (!result.success) {
         throw new Error(formatCustomWidgetImportIssues(result.issues));
       }
-      applyDefinition(input.form, customWidgetDefinitionSchema.parse(result.widget));
       input.setOptionsSnapshot(getDefinitionDefaults(result.widget));
       input.setPreview({ data: {}, status: {}, session: null, outcome: "idle" });
       showSuccessNotification({

packages/custom-widgets/package.json

@@ -29,12 +29,13 @@
     "generate:components": "pnpm generate:catalog",
     "generate": "pnpm generate:catalog",
     "evaluate:ai": "bun ./scripts/evaluate-ai-authoring.ts",
+    "evaluate:mcp": "tsx ./scripts/evaluate-mcp-agent.ts",
     "test:ai": "bun ./scripts/evaluate-ai-authoring.ts --ci",
     "test:reference-apis": "tsx ./scripts/verify-reference-apis.ts",
     "format": "oxfmt .",
     "lint": "oxlint",
     "test": "vitest run",
-    "typecheck": "tsc --noEmit"
+    "typecheck": "tsc --noEmit && tsc --noEmit -p tsconfig.scripts.json"
   },
   "dependencies": {
     "@codemirror/autocomplete": "catalog:",
@@ -61,10 +62,14 @@
     "zod": "catalog:"
   },
   "devDependencies": {
+    "@ai-sdk/mcp": "catalog:",
+    "@ai-sdk/provider": "catalog:",
     "@homarr/tsconfig": "workspace:^0.1.0",
+    "@openrouter/ai-sdk-provider": "catalog:",
     "@types/node": "catalog:",
     "@types/react": "catalog:",
     "@types/react-dom": "catalog:",
+    "ai": "catalog:",
     "jsdom": "catalog:",
     "react-dom": "catalog:",
     "typescript": "catalog:",

packages/custom-widgets/scripts/ai-evaluation-cases.ts

@@ -3,6 +3,20 @@ export interface CustomWidgetAiEvaluationCase {
   request: string;
   documentationUrl: string;
   apiNotes: string;
+  acceptance: {
+    sourceAuth?: { type: "none" | "bearer" | "apiKeyHeader" | "apiKeyQuery"; name?: string };
+    requestRules: ReadonlyArray<{
+      label: string;
+      pathIncludes: string;
+      kind?: "query" | "action";
+      method?: "GET" | "POST" | "PUT" | "DELETE" | "PATCH";
+      trigger?: "load" | "manual";
+      optionReference?: boolean;
+      parameterReference?: boolean;
+      invalidates?: boolean;
+    }>;
+    templateComponents: readonly string[];
+  };
 }
 
 export const CUSTOM_WIDGET_AI_EVALUATION_CASES: readonly CustomWidgetAiEvaluationCase[] = [
@@ -13,6 +27,20 @@ export const CUSTOM_WIDGET_AI_EVALUATION_CASES: readonly CustomWidgetAiEvaluatio
       "Create a polished Pokédex browser with searchable Pokémon, a responsive result grid, manual detail loading, sprites, types, abilities, base-stat progress bars, and clear loading, empty, and failure states.",
     apiNotes:
       "Use GET /api/v2/pokemon?limit=<option>&offset=<option> for the list and GET /api/v2/pokemon/{param:name} as a manual detail query. PokeAPI needs no authentication.",
+    acceptance: {
+      sourceAuth: { type: "none" },
+      requestRules: [
+        { label: "Pokémon list", pathIncludes: "/api/v2/pokemon", kind: "query", trigger: "load" },
+        {
+          label: "manual Pokémon detail",
+          pathIncludes: "/api/v2/pokemon/",
+          kind: "query",
+          trigger: "manual",
+          parameterReference: true,
+        },
+      ],
+      templateComponents: ["TextInput", "SubFetch", "Image", "Progress"],
+    },
   },
   {
     id: "portainer-containers",
@@ -21,22 +49,84 @@ export const CUSTOM_WIDGET_AI_EVALUATION_CASES: readonly CustomWidgetAiEvaluatio
       "Create an excellent Portainer container dashboard: environment option, running/stopped summary, responsive container list, health/status badges, and explicit start, stop, and restart actions with confirmation and refresh after success.",
     apiNotes:
       "Use X-API-Key auth. List containers with GET /api/endpoints/{option:endpointId}/docker/containers/json?all=true. Actions are POST /api/endpoints/{option:endpointId}/docker/containers/{param:id}/start, /stop, and /restart and invalidate the list query.",
+    acceptance: {
+      sourceAuth: { type: "apiKeyHeader", name: "X-API-Key" },
+      requestRules: [
+        {
+          label: "container list",
+          pathIncludes: "/containers/json",
+          kind: "query",
+          optionReference: true,
+        },
+        {
+          label: "start action",
+          pathIncludes: "/start",
+          kind: "action",
+          method: "POST",
+          parameterReference: true,
+          invalidates: true,
+        },
+        {
+          label: "stop action",
+          pathIncludes: "/stop",
+          kind: "action",
+          method: "POST",
+          parameterReference: true,
+          invalidates: true,
+        },
+        {
+          label: "restart action",
+          pathIncludes: "/restart",
+          kind: "action",
+          method: "POST",
+          parameterReference: true,
+          invalidates: true,
+        },
+      ],
+      templateComponents: ["ActionButton", "Badge", "RefreshButton"],
+    },
   },
   {
-    id: "tautulli-activity",
-    documentationUrl: "https://github.com/Tautulli/Tautulli/wiki/Tautulli-API-Reference",
+    id: "football-dashboard",
+    documentationUrl: "https://docs.football-data.org/general/v4/competition.html",
     request:
-      "Create a beautiful Tautulli activity widget with active stream cards, user/player details, progress, transcode/direct-play badges, bandwidth summary, and a refresh control that works in narrow and wide tiles.",
+      "Create a polished football dashboard with a competition selector, league table, upcoming fixtures, club crests, match status and kickoff time. It must be highly scannable in narrow and wide tiles, refreshable, and include complete loading, empty, and error states.",
     apiNotes:
-      "Use an apiKeyQuery source with parameter name apikey. Query GET /api/v2 with query cmd=get_activity. The response payload is under response.data and sessions is an array.",
+      "Use https://api.football-data.org/v4 with X-Auth-Token authentication. GET /competitions/{option:competition}/standings returns standings[].table with position, team, playedGames, won, draw, lost, points and goalDifference. GET /competitions/{option:competition}/matches with query status=SCHEDULED returns matches with utcDate, status, homeTeam, awayTeam and score. Use fixed choices such as PL, CL, PD, BL1, SA and FL1 for the competition option.",
+    acceptance: {
+      sourceAuth: { type: "apiKeyHeader", name: "X-Auth-Token" },
+      requestRules: [
+        {
+          label: "competition standings",
+          pathIncludes: "/standings",
+          kind: "query",
+          optionReference: true,
+        },
+        {
+          label: "competition fixtures",
+          pathIncludes: "/matches",
+          kind: "query",
+          optionReference: true,
+        },
+      ],
+      templateComponents: ["Table", "Image", "Badge", "RefreshButton"],
+    },
   },
   {
-    id: "bambubuddy-printer",
-    documentationUrl: "https://wiki.bambuddy.cool/reference/api/",
+    id: "jellyfin-activity",
+    documentationUrl: "https://api.jellyfin.org/",
     request:
-      "Create a premium BambuBuddy printer status widget with printer selector, current job, progress, remaining time, nozzle and bed temperatures, connection state, and safe pause/resume/stop controls when supported.",
+      "Create a premium Jellyfin overview with movie, series, episode and song totals plus active playback sessions. Show user, title, client, play method, progress, paused/playing state and a calm empty state when nothing is playing. It must remain useful in narrow and wide tiles.",
     apiNotes:
-      "Use X-API-Key auth and base path /api/v1. GET /printers lists printers and GET /printers/{id}/status returns state, progress, remaining_time, temperatures.nozzle, temperatures.bed, and hms_status. The official reference does not document pause, resume, or stop endpoints, so omit those actions. Configure the selected printer with choicesFrom on a widget option and use that option in the status path.",
+      "Use the user's Jellyfin server URL with private network scope and X-Emby-Token authentication. GET /Items/Counts returns MovieCount, SeriesCount, EpisodeCount and SongCount. GET /Sessions with query activeWithinSeconds=300 returns sessions; active playback has NowPlayingItem, UserName, Client, PlayState.PositionTicks, PlayState.IsPaused and TranscodingInfo or NowPlayingItem.RunTimeTicks. Do not invent playback-control actions.",
+    acceptance: {
+      sourceAuth: { type: "apiKeyHeader", name: "X-Emby-Token" },
+      requestRules: [
+        { label: "library counts", pathIncludes: "/Items/Counts", kind: "query" },
+        { label: "active sessions", pathIncludes: "/Sessions", kind: "query" },
+      ],
+      templateComponents: ["Progress", "Badge", "RefreshButton"],
+    },
   },
   {
     id: "home-assistant-control",
@@ -45,5 +135,39 @@ export const CUSTOM_WIDGET_AI_EVALUATION_CASES: readonly CustomWidgetAiEvaluatio
       "Create a refined Home Assistant room widget with temperature and humidity readings, light status, a room/entity option, and an actionable light toggle. Use calm hierarchy and responsive controls, not a pile of nested cards.",
     apiNotes:
       "Use bearer auth. GET /api/states/{option:sensorEntity} and /api/states/{option:lightEntity} load entity state. POST /api/services/light/turn_on and /turn_off accept a body with entity_id from an option and should invalidate the light query.",
+    acceptance: {
+      sourceAuth: { type: "bearer" },
+      requestRules: [
+        {
+          label: "sensor state",
+          pathIncludes: "/api/states/",
+          kind: "query",
+          optionReference: true,
+        },
+        {
+          label: "light state",
+          pathIncludes: "/api/states/",
+          kind: "query",
+          optionReference: true,
+        },
+        {
+          label: "light on action",
+          pathIncludes: "/api/services/light/turn_on",
+          kind: "action",
+          method: "POST",
+          optionReference: true,
+          invalidates: true,
+        },
+        {
+          label: "light off action",
+          pathIncludes: "/api/services/light/turn_off",
+          kind: "action",
+          method: "POST",
+          optionReference: true,
+          invalidates: true,
+        },
+      ],
+      templateComponents: ["ToggleSwitch", "Badge", "SimpleGrid"],
+    },
   },
 ] as const;

packages/custom-widgets/scripts/ai-evaluation.ts

@@ -7,6 +7,7 @@ import { buildCustomWidgetAiPrompt } from "../src/core/ai-prompt";
 import { customWidgetDefinitionSchema } from "../src/core/custom-jsx-schema";
 import type { HomarrCustomWidgetV2 } from "../src/core/custom-jsx-schema";
 import { formatCustomWidgetImportIssues, parseCustomWidgetAiResponse } from "../src/core/import";
+import { collectCustomWidgetRequestReferences } from "../src/core/request-schema";
 import type { CustomWidgetAiEvaluationCase } from "./ai-evaluation-cases";
 
 export const DEFAULT_GENERATOR_MODEL = "deepseek/deepseek-v4-pro";
@@ -158,6 +159,65 @@ export function getJudgeResponseFormat() {
   };
 }
 
+export function getScenarioAcceptanceIssues(
+  testCase: CustomWidgetAiEvaluationCase,
+  widget: HomarrCustomWidgetV2,
+): Array<{ path?: Array<string | number>; message: string }> {
+  const issues: Array<{ path?: Array<string | number>; message: string }> = [];
+  const expectedAuth = testCase.acceptance.sourceAuth;
+  if (expectedAuth) {
+    const matches = Object.values(widget.sources).some((source) => {
+      const actualType = typeof source.auth === "string" ? source.auth : source.auth.type;
+      if (actualType !== expectedAuth.type) return false;
+      return (
+        expectedAuth.name === undefined ||
+        (typeof source.auth !== "string" && source.auth.name.toLowerCase() === expectedAuth.name.toLowerCase())
+      );
+    });
+    if (!matches) {
+      issues.push({
+        path: ["sources"],
+        message: `The scenario requires ${expectedAuth.type}${expectedAuth.name ? ` '${expectedAuth.name}'` : ""} authentication.`,
+      });
+    }
+  }
+
+  const matchedRequestIds = new Set<string>();
+  for (const rule of testCase.acceptance.requestRules) {
+    const match = Object.entries(widget.requests).find(([requestId, request]) => {
+      if (matchedRequestIds.has(requestId)) return false;
+      if (!request.path.includes(rule.pathIncludes)) return false;
+      if (rule.kind && request.kind !== rule.kind) return false;
+      if (rule.method && request.method !== rule.method) return false;
+      if (rule.trigger && request.trigger !== rule.trigger) return false;
+      const references = collectCustomWidgetRequestReferences(request);
+      if (rule.optionReference && references.options.size === 0) return false;
+      if (rule.parameterReference && references.params.size === 0) return false;
+      if (
+        rule.invalidates &&
+        !request.invalidates?.some((invalidatedRequestId) => widget.requests[invalidatedRequestId]?.kind === "query")
+      ) {
+        return false;
+      }
+      return true;
+    });
+    if (match) matchedRequestIds.add(match[0]);
+    else {
+      issues.push({
+        path: ["requests"],
+        message: `Missing grounded ${rule.label} request (${rule.pathIncludes}).`,
+      });
+    }
+  }
+
+  for (const component of testCase.acceptance.templateComponents) {
+    if (!widget.template.includes(`<${component}`)) {
+      issues.push({ path: ["template"], message: `The ${testCase.id} scenario requires ${component}.` });
+    }
+  }
+  return issues;
+}
+
 export async function evaluateCustomWidgetCase(args: {
   testCase: CustomWidgetAiEvaluationCase;
   apiKey: string;
@@ -201,6 +261,14 @@ export async function evaluateCustomWidgetCase(args: {
 
     const canonical = customWidgetDefinitionSchema.parse(parsed.widget);
     await writeWidgetFiles(caseDirectory, canonical, `attempt-${attempt}`);
+    const acceptanceIssues = getScenarioAcceptanceIssues(args.testCase, canonical);
+    if (acceptanceIssues.length > 0) {
+      errors.push(
+        `Attempt ${attempt}: deterministic acceptance failed — ${acceptanceIssues.map(({ message }) => message).join("; ")}`,
+      );
+      prompt = buildRepairPrompt(originalPrompt, response, acceptanceIssues);
+      continue;
+    }
     let judge: CustomWidgetJudgeResult;
     try {
       const judgeRaw = await callOpenRouter({

packages/custom-widgets/scripts/evaluate-mcp-agent.ts

@@ -0,0 +1,80 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+
+import { createMCPClient } from "@ai-sdk/mcp";
+import { createOpenRouter } from "@openrouter/ai-sdk-provider";
+
+import { CUSTOM_WIDGET_AI_EVALUATION_CASES } from "./ai-evaluation-cases";
+import { DEFAULT_GENERATOR_MODEL } from "./ai-evaluation";
+import {
+  buildCustomWidgetMcpAgentPrompt,
+  createCustomWidgetMcpAgent,
+  getCustomWidgetMcpToolExecutions,
+  getCustomWidgetMcpWorkflowIssues,
+} from "./mcp-agent";
+
+const openRouterApiKey = process.env.OPENROUTER_API_KEY;
+const mcpUrl = process.env.HOMARR_MCP_URL;
+const mcpApiKey = process.env.HOMARR_MCP_API_KEY;
+if (!openRouterApiKey) throw new Error("OPENROUTER_API_KEY is required");
+if (!mcpUrl) throw new Error("HOMARR_MCP_URL is required");
+if (!mcpApiKey) throw new Error("HOMARR_MCP_API_KEY is required");
+
+const requestedCase = process.argv.find((value) => value.startsWith("--case="))?.slice("--case=".length) ?? "pokedex";
+const definitionId = process.argv.find((value) => value.startsWith("--definition="))?.slice("--definition=".length);
+const persist = process.argv.includes("--persist");
+const testCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === requestedCase);
+if (!testCase) throw new Error(`Unknown MCP evaluation case '${requestedCase}'`);
+
+const client = await createMCPClient({
+  transport: { type: "http", url: mcpUrl, headers: { ApiKey: mcpApiKey } },
+});
+
+try {
+  const tools = await client.tools();
+  const openrouter = createOpenRouter({
+    apiKey: openRouterApiKey,
+    appName: "Homarr Custom Widget MCP Evaluation",
+    appUrl: "https://homarr.dev",
+  });
+  const agent = createCustomWidgetMcpAgent({
+    model: openrouter(process.env.OPENROUTER_GENERATOR_MODEL ?? DEFAULT_GENERATOR_MODEL, {
+      structuredOutputs: { strict: true },
+      provider: { require_parameters: true, data_collection: "deny" },
+    }),
+    tools,
+  });
+  const result = await agent.generate({
+    prompt: buildCustomWidgetMcpAgentPrompt({ testCase, persist, definitionId }),
+    timeout: 300_000,
+  });
+  const toolExecutions = getCustomWidgetMcpToolExecutions(result.steps);
+  const toolsUsed = toolExecutions.map(({ tool }) => tool);
+  const issues = getCustomWidgetMcpWorkflowIssues({
+    output: result.output,
+    toolExecutions,
+    persist,
+    definitionId,
+    requiredQueryCount: testCase.acceptance.requestRules.filter(({ kind }) => kind !== "action").length,
+    requiredActionCount: testCase.acceptance.requestRules.filter(({ kind }) => kind === "action").length,
+  });
+  const summary = {
+    generatedAt: new Date().toISOString(),
+    model: process.env.OPENROUTER_GENERATOR_MODEL ?? DEFAULT_GENERATOR_MODEL,
+    caseId: testCase.id,
+    persist,
+    definitionId: definitionId ?? null,
+    toolsUsed,
+    failedTools: toolExecutions.filter(({ succeeded }) => !succeeded).map(({ tool }) => tool),
+    output: result.output,
+    workflowIssues: issues,
+  };
+  const outputDirectory = path.resolve(process.cwd(), ".ai-evaluations", "mcp");
+  await mkdir(outputDirectory, { recursive: true });
+  const outputFile = path.join(outputDirectory, `${testCase.id}-${Date.now()}.json`);
+  await writeFile(outputFile, JSON.stringify(summary, null, 2), "utf8");
+  process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`);
+  if (result.output.status === "fail" || issues.length > 0) process.exitCode = 1;
+} finally {
+  await client.close();
+}

packages/custom-widgets/scripts/mcp-agent.ts

@@ -0,0 +1,287 @@
+import type { LanguageModel, ToolSet } from "ai";
+import { Output, stepCountIs, ToolLoopAgent } from "ai";
+import { z } from "zod/v4";
+
+import type { CustomWidgetAiEvaluationCase } from "./ai-evaluation-cases";
+
+export const CUSTOM_WIDGET_MCP_AGENT_TOOLS = [
+  "customWidget_get",
+  "customWidget_getAuthoringPrompt",
+  "customWidget_getSkill",
+  "customWidget_schema",
+  "customWidget_validate",
+  "customWidget_previewCreate",
+  "customWidget_previewQuery",
+  "customWidget_previewAction",
+  "customWidget_previewJournal",
+  "customWidget_create",
+  "customWidget_update",
+  "customWidget_configurationRequestUser",
+] as const;
+
+export const customWidgetMcpAgentResultSchema = z.strictObject({
+  status: z.enum(["pass", "needs-user-configuration", "fail"]),
+  summary: z.string().min(1).max(1_000),
+  definitionId: z.string().min(1).nullable(),
+  previewUrl: z.string().url().nullable(),
+  iterations: z.number().int().min(1).max(20),
+  evidence: z
+    .array(
+      z.strictObject({
+        tool: z.string().min(1).max(128),
+        outcome: z.enum(["passed", "repaired", "pending", "failed"]),
+        detail: z.string().min(1).max(500),
+      }),
+    )
+    .min(1)
+    .max(30),
+  remainingIssues: z.array(z.string().min(1).max(500)).max(10),
+});
+
+export type CustomWidgetMcpAgentResult = z.infer<typeof customWidgetMcpAgentResultSchema>;
+
+export interface CustomWidgetMcpToolExecution {
+  tool: string;
+  succeeded: boolean;
+  requestId?: string;
+}
+
+interface AgentToolStep {
+  toolCalls: ReadonlyArray<{ toolCallId: string; toolName: string; input: unknown }>;
+  toolResults: ReadonlyArray<{ toolCallId: string; toolName: string; output: unknown }>;
+}
+
+const AGENT_INSTRUCTIONS = `You are the Homarr Custom Widget release evaluator. Use only the supplied Custom Widget MCP tools.
+
+Read the live authoring prompt, skill, and schema before authoring. Never invent a tool, API route, validation result, preview result, or credential. Validate every revision. Create a preview, execute every load query that can run, simulate actions instead of causing side effects, and inspect the preview journal. Repair validation, request, runtime, and template problems and repeat the failed checks.
+
+Never put plaintext credentials in a widget, response, summary, or tool argument. For an authenticated source without existing credentials, create a user configuration request with customWidget_configurationRequestUser and report needs-user-configuration. Do not call secret-writing tools.
+
+Revise unsaved definitions in your working context; do not mutate a stored template while debugging. Persist only when the user prompt explicitly says persistence is allowed and all runnable checks pass. For edits, read the current definition first and update the same definition. Return the required structured result with evidence tied to successful tool calls. A pass must have no remaining issues.`;
+
+export function selectCustomWidgetMcpTools(tools: Readonly<Record<string, unknown>>): ToolSet {
+  return Object.fromEntries(
+    CUSTOM_WIDGET_MCP_AGENT_TOOLS.flatMap((name) => (tools[name] ? [[name, tools[name]]] : [])),
+  ) as ToolSet;
+}
+
+export function createCustomWidgetMcpAgent(args: { model: LanguageModel; tools: Readonly<Record<string, unknown>> }) {
+  return new ToolLoopAgent({
+    model: args.model,
+    tools: selectCustomWidgetMcpTools(args.tools),
+    instructions: AGENT_INSTRUCTIONS,
+    stopWhen: stepCountIs(20),
+    temperature: 0,
+    output: Output.object({ schema: customWidgetMcpAgentResultSchema }),
+  });
+}
+
+export function buildCustomWidgetMcpAgentPrompt(args: {
+  testCase: CustomWidgetAiEvaluationCase;
+  persist: boolean;
+  definitionId?: string;
+}) {
+  const task = args.definitionId
+    ? `Edit and debug Custom Widget definition ${args.definitionId}.`
+    : "Create and debug a new Custom Widget.";
+  return `${task}
+
+User request:
+${args.testCase.request}
+
+Verified API notes:
+${args.testCase.apiNotes}
+
+API documentation:
+${args.testCase.documentationUrl}
+
+Persistence is ${args.persist ? "allowed after every runnable check passes" : "not allowed; stop after the validated preview"}.`;
+}
+
+export function getCustomWidgetMcpToolExecutions(steps: readonly AgentToolStep[]): CustomWidgetMcpToolExecution[] {
+  const results = new Map(
+    steps.flatMap(({ toolResults }) => toolResults).map((result) => [result.toolCallId, result] as const),
+  );
+  return steps.flatMap(({ toolCalls }) =>
+    toolCalls.map((call) => {
+      const requestId = asRecord(call.input)?.requestId;
+      return {
+        tool: call.toolName,
+        succeeded: isSuccessfulMcpToolOutput(call.toolName, results.get(call.toolCallId)?.output),
+        ...(typeof requestId === "string" ? { requestId } : {}),
+      };
+    }),
+  );
+}
+
+export function getCustomWidgetMcpWorkflowIssues(args: {
+  output: CustomWidgetMcpAgentResult;
+  toolExecutions: readonly CustomWidgetMcpToolExecution[];
+  persist: boolean;
+  definitionId?: string;
+  requiredQueryCount?: number;
+  requiredActionCount?: number;
+}) {
+  const issues: string[] = [];
+  const firstIndex = (tool: string) => args.toolExecutions.findIndex((entry) => entry.tool === tool);
+  const firstSuccessfulIndex = (tool: string) =>
+    args.toolExecutions.findIndex((entry) => entry.tool === tool && entry.succeeded);
+  const lastSuccessfulIndex = (tool: string) =>
+    args.toolExecutions.findLastIndex((entry) => entry.tool === tool && entry.succeeded);
+  const hasTool = (tool: string) => firstIndex(tool) >= 0;
+  const hasSuccessfulTool = (tool: string) => firstSuccessfulIndex(tool) >= 0;
+  const required = [
+    "customWidget_getAuthoringPrompt",
+    "customWidget_getSkill",
+    "customWidget_schema",
+    "customWidget_validate",
+  ];
+  if (args.definitionId) required.unshift("customWidget_get");
+  if (args.output.status === "pass") {
+    required.push("customWidget_previewCreate", "customWidget_previewQuery", "customWidget_previewJournal");
+    if ((args.requiredActionCount ?? 0) > 0) required.push("customWidget_previewAction");
+    if (args.persist) required.push(args.definitionId ? "customWidget_update" : "customWidget_create");
+  }
+  if (args.output.status === "needs-user-configuration") {
+    required.push("customWidget_configurationRequestUser");
+    if (!args.persist && !args.definitionId) required.push("customWidget_previewCreate");
+    if (args.persist && !args.definitionId) required.push("customWidget_create");
+  }
+  for (const tool of required) {
+    if (!hasTool(tool)) issues.push(`Required tool was not used: ${tool}`);
+    else if (!hasSuccessfulTool(tool)) issues.push(`Required tool did not complete successfully: ${tool}`);
+  }
+
+  const firstValidation = firstSuccessfulIndex("customWidget_validate");
+  const finalValidation = lastSuccessfulIndex("customWidget_validate");
+  for (const tool of ["customWidget_getAuthoringPrompt", "customWidget_getSkill", "customWidget_schema"]) {
+    if (hasSuccessfulTool(tool) && firstValidation >= 0 && firstSuccessfulIndex(tool) > firstValidation) {
+      issues.push(`${tool} must run before the first validation`);
+    }
+  }
+  if (
+    args.definitionId &&
+    hasSuccessfulTool("customWidget_get") &&
+    firstValidation >= 0 &&
+    firstSuccessfulIndex("customWidget_get") > firstValidation
+  ) {
+    issues.push("customWidget_get must run before validating an edit");
+  }
+  const previewCreate = firstSuccessfulIndex("customWidget_previewCreate");
+  if (finalValidation >= 0 && previewCreate >= 0 && finalValidation > previewCreate) {
+    issues.push("The final validation must run before customWidget_previewCreate");
+  }
+  const previewChecks = [
+    "customWidget_previewQuery",
+    ...((args.requiredActionCount ?? 0) > 0 ? ["customWidget_previewAction"] : []),
+    "customWidget_previewJournal",
+  ];
+  for (const tool of previewChecks) {
+    if (hasSuccessfulTool(tool) && previewCreate >= 0 && firstSuccessfulIndex(tool) < previewCreate) {
+      issues.push(`${tool} must run after customWidget_previewCreate`);
+    }
+  }
+  if (args.output.status === "pass") {
+    for (const [tool, requiredCount] of [
+      ["customWidget_previewQuery", args.requiredQueryCount ?? 1],
+      ["customWidget_previewAction", args.requiredActionCount ?? 0],
+    ] as const) {
+      const completed = args.toolExecutions.filter((entry) => entry.tool === tool && entry.succeeded);
+      const distinctRequestIds = new Set(completed.flatMap(({ requestId }) => (requestId ? [requestId] : [])));
+      const completedCount = distinctRequestIds.size > 0 ? distinctRequestIds.size : completed.length;
+      if (completedCount < requiredCount) {
+        issues.push(`${tool} completed ${completedCount} time(s); expected at least ${requiredCount}`);
+      }
+    }
+    const finalJournal = lastSuccessfulIndex("customWidget_previewJournal");
+    for (const tool of ["customWidget_previewQuery", "customWidget_previewAction"]) {
+      if (lastSuccessfulIndex(tool) > finalJournal) {
+        issues.push("customWidget_previewJournal must run after all preview requests");
+        break;
+      }
+    }
+  }
+
+  const persistenceTool = args.definitionId ? "customWidget_update" : "customWidget_create";
+  if (args.persist && args.output.status === "pass" && hasSuccessfulTool(persistenceTool)) {
+    const lastPreviewCheck = Math.max(...previewChecks.map(lastSuccessfulIndex));
+    if (lastSuccessfulIndex(persistenceTool) < lastPreviewCheck) {
+      issues.push(`${persistenceTool} must run after all required preview checks`);
+    }
+  }
+  if (!args.persist) {
+    for (const tool of ["customWidget_create", "customWidget_update"]) {
+      if (hasTool(tool)) issues.push(`${tool} was used even though persistence was not allowed`);
+    }
+  }
+
+  const evidenceTools = new Set(args.output.evidence.map(({ tool }) => tool));
+  for (const evidence of args.output.evidence) {
+    const tool = evidence.tool;
+    if (!hasTool(tool)) issues.push(`Structured evidence names a tool that was not used: ${tool}`);
+    else if (evidence.outcome !== "failed" && !hasSuccessfulTool(tool)) {
+      issues.push(`Structured evidence claims a successful outcome for a failed tool: ${tool}`);
+    }
+  }
+  for (const tool of required) {
+    if (hasSuccessfulTool(tool) && !evidenceTools.has(tool)) {
+      issues.push(`Structured evidence is missing the required tool: ${tool}`);
+    }
+  }
+  if (args.output.status === "pass" && args.output.remainingIssues.length > 0) {
+    issues.push("A passing result cannot contain remaining issues");
+  }
+  if (args.output.status === "pass" && args.output.previewUrl === null) {
+    issues.push("A passing result must include the tested preview URL");
+  }
+  if (args.persist && args.output.definitionId === null) {
+    issues.push("A persisted result must include the definition ID");
+  }
+  if (args.output.status === "needs-user-configuration") {
+    if (!hasSuccessfulTool("customWidget_configurationRequestUser")) {
+      issues.push("User configuration status requires customWidget_configurationRequestUser");
+    } else if (!evidenceTools.has("customWidget_configurationRequestUser")) {
+      issues.push("Structured evidence is missing customWidget_configurationRequestUser");
+    }
+    const configurationRequest = lastSuccessfulIndex("customWidget_configurationRequestUser");
+    const targetCreation = args.persist ? lastSuccessfulIndex(persistenceTool) : previewCreate;
+    if (!args.definitionId && targetCreation >= 0 && configurationRequest < targetCreation) {
+      issues.push("customWidget_configurationRequestUser must run after its definition or preview target exists");
+    }
+  }
+  return issues;
+}
+
+function isSuccessfulMcpToolOutput(tool: string, output: unknown) {
+  const record = asRecord(output);
+  if (!record || record.isError === true) return false;
+  const payload = decodeMcpTextPayload(record) ?? record;
+  if (tool === "customWidget_validate") return asRecord(payload)?.valid === true;
+  if (tool === "customWidget_previewCreate") return asRecord(payload)?.success === true;
+  if (tool === "customWidget_previewQuery" || tool === "customWidget_previewAction") {
+    return asRecord(payload)?.ok === true;
+  }
+  return output !== undefined;
+}
+
+function decodeMcpTextPayload(output: Record<string, unknown>) {
+  if (!Array.isArray(output.content)) return undefined;
+  const text = output.content
+    .flatMap((part) => {
+      const record = asRecord(part);
+      return record?.type === "text" && typeof record.text === "string" ? [record.text] : [];
+    })
+    .join("");
+  if (!text) return undefined;
+  try {
+    return JSON.parse(text) as unknown;
+  } catch {
+    return undefined;
+  }
+}
+
+function asRecord(value: unknown): Record<string, unknown> | undefined {
+  return typeof value === "object" && value !== null && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : undefined;
+}

packages/custom-widgets/src/test/ai-evaluation.spec.ts

@@ -7,10 +7,12 @@ import {
   DEFAULT_GENERATOR_MODEL,
   DEFAULT_JUDGE_MODEL,
   getJudgeResponseFormat,
+  getScenarioAcceptanceIssues,
   judgePasses,
   parseJudgeResult,
 } from "../../scripts/ai-evaluation";
 import type { CustomWidgetJudgeResult } from "../../scripts/ai-evaluation";
+import { BUNDLED_CUSTOM_WIDGETS, CUSTOM_WIDGET_STARTER, customWidgetDefinitionSchema } from "../core";
 
 const categoryNames = [
   "schemaAndBindings",
@@ -49,6 +51,13 @@ describe("AI authoring evaluation", () => {
   it("defines five distinct complex scenarios", () => {
     expect(CUSTOM_WIDGET_AI_EVALUATION_CASES).toHaveLength(5);
     expect(new Set(CUSTOM_WIDGET_AI_EVALUATION_CASES.map((entry) => entry.id)).size).toBe(5);
+    expect(CUSTOM_WIDGET_AI_EVALUATION_CASES.map(({ id }) => id)).toEqual([
+      "pokedex",
+      "portainer-containers",
+      "football-dashboard",
+      "jellyfin-activity",
+      "home-assistant-control",
+    ]);
   });
 
   it("uses the exact clipboard prompt and grounded API notes", () => {
@@ -69,6 +78,25 @@ describe("AI authoring evaluation", () => {
     expect(prompt).toContain("bad response");
   });
 
+  it("requires grounded scenario capabilities before model judging", () => {
+    const testCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === "pokedex");
+    const pokedex = BUNDLED_CUSTOM_WIDGETS.find(({ id }) => id === "seed-pokedex");
+    if (!testCase || !pokedex) throw new Error("Expected Pokédex evaluation fixtures");
+    expect(getScenarioAcceptanceIssues(testCase, customWidgetDefinitionSchema.parse(pokedex.widget))).toEqual([]);
+
+    const issues = getScenarioAcceptanceIssues(
+      testCase,
+      customWidgetDefinitionSchema.parse({ ...CUSTOM_WIDGET_STARTER, template: "<Text>Pokémon</Text>" }),
+    );
+    expect(issues.map(({ message }) => message)).toEqual(
+      expect.arrayContaining([
+        expect.stringContaining("Pokémon list"),
+        expect.stringContaining("manual Pokémon detail"),
+        expect.stringContaining("SubFetch"),
+      ]),
+    );
+  });
+
   it("uses the requested DeepSeek models and strict structured judge output", () => {
     expect(DEFAULT_GENERATOR_MODEL).toBe("deepseek/deepseek-v4-pro");
     expect(DEFAULT_JUDGE_MODEL).toBe("deepseek/deepseek-v4-flash");

packages/custom-widgets/src/test/mcp-agent.spec.ts

@@ -0,0 +1,322 @@
+import type { LanguageModelV3GenerateResult } from "@ai-sdk/provider";
+import { tool } from "ai";
+import { MockLanguageModelV3 } from "ai/test";
+import { describe, expect, it, vi } from "vitest";
+import { z } from "zod/v4";
+
+import {
+  createCustomWidgetMcpAgent,
+  getCustomWidgetMcpToolExecutions,
+  getCustomWidgetMcpWorkflowIssues,
+} from "../../scripts/mcp-agent";
+
+const usage = {
+  inputTokens: { total: 10, noCache: 10, cacheRead: undefined, cacheWrite: undefined },
+  outputTokens: { total: 10, text: 10, reasoning: undefined },
+};
+
+const toolCall = (
+  toolName: string,
+  index: number,
+  input: Record<string, unknown> = {},
+): LanguageModelV3GenerateResult => ({
+  content: [{ type: "tool-call", toolCallId: `call-${index}`, toolName, input: JSON.stringify(input) }],
+  finishReason: { unified: "tool-calls", raw: undefined },
+  usage,
+  warnings: [],
+});
+
+const finalResult = (value: unknown): LanguageModelV3GenerateResult => ({
+  content: [{ type: "text", text: JSON.stringify(value) }],
+  finishReason: { unified: "stop", raw: undefined },
+  usage,
+  warnings: [],
+});
+
+const successfulExecutions = (tools: readonly string[]) =>
+  tools.map((toolName) => ({ tool: toolName, succeeded: true }));
+const mcpOutput = (value: unknown) => ({ content: [{ type: "text" as const, text: JSON.stringify(value) }] });
+
+describe("Custom Widget MCP agent", () => {
+  it("runs a repair, preview, and persistence workflow before structured success", async () => {
+    const calls: string[] = [];
+    let validationCount = 0;
+    const makeTool = (name: string) =>
+      tool({
+        description: name,
+        inputSchema: z.object({ requestId: z.string().optional() }),
+        execute: vi.fn(() => {
+          calls.push(name);
+          if (name === "customWidget_validate") {
+            validationCount += 1;
+            return mcpOutput({ valid: validationCount > 1 });
+          }
+          if (name === "customWidget_previewCreate") return mcpOutput({ success: true });
+          if (name === "customWidget_previewQuery" || name === "customWidget_previewAction") {
+            return mcpOutput({ ok: true });
+          }
+          return mcpOutput({});
+        }),
+      });
+    const sequence = [
+      "customWidget_getAuthoringPrompt",
+      "customWidget_getSkill",
+      "customWidget_schema",
+      "customWidget_validate",
+      "customWidget_validate",
+      "customWidget_previewCreate",
+      "customWidget_previewQuery",
+      "customWidget_previewQuery",
+      "customWidget_previewAction",
+      "customWidget_previewJournal",
+      "customWidget_create",
+    ];
+    const output = {
+      status: "pass" as const,
+      summary: "The repaired widget passed validation and preview checks and was persisted.",
+      definitionId: "widget-1",
+      previewUrl: "http://localhost:3000/manage/custom-widgets/preview/session-1",
+      iterations: 2,
+      evidence: [...new Set(sequence)].map((name) => ({
+        tool: name,
+        outcome: name === "customWidget_validate" ? ("repaired" as const) : ("passed" as const),
+        detail: `${name} completed.`,
+      })),
+      remainingIssues: [],
+    };
+    let queryIndex = 0;
+    const responses = [
+      ...sequence.map((toolName, index) => {
+        if (toolName === "customWidget_previewQuery") {
+          queryIndex += 1;
+          return toolCall(toolName, index, { requestId: `query-${queryIndex}` });
+        }
+        if (toolName === "customWidget_previewAction") return toolCall(toolName, index, { requestId: "action-1" });
+        return toolCall(toolName, index);
+      }),
+      finalResult(output),
+    ];
+    let responseIndex = 0;
+    const model = new MockLanguageModelV3({
+      doGenerate: async () => responses[responseIndex++] ?? finalResult(output),
+    });
+    const tools = Object.fromEntries(sequence.map((name) => [name, makeTool(name)]));
+
+    const result = await createCustomWidgetMcpAgent({ model, tools }).generate({ prompt: "Build a widget" });
+    expect(result.output).toEqual(output);
+    expect(calls).toEqual(sequence);
+    expect(model.doGenerateCalls.at(-1)?.responseFormat).toMatchObject({ type: "json" });
+    const toolExecutions = getCustomWidgetMcpToolExecutions(result.steps);
+    expect(toolExecutions.filter(({ tool: toolName }) => toolName === "customWidget_validate")).toEqual([
+      { tool: "customWidget_validate", succeeded: false },
+      { tool: "customWidget_validate", succeeded: true },
+    ]);
+    expect(
+      toolExecutions
+        .filter(({ tool: toolName }) => toolName === "customWidget_previewQuery")
+        .map(({ requestId }) => requestId),
+    ).toEqual(["query-1", "query-2"]);
+    expect(
+      getCustomWidgetMcpWorkflowIssues({
+        output: result.output,
+        toolExecutions,
+        persist: true,
+        requiredQueryCount: 2,
+        requiredActionCount: 1,
+      }),
+    ).toEqual([]);
+  });
+
+  it("uses the current definition and updates it in the edit workflow", async () => {
+    const calls: string[] = [];
+    const sequence = [
+      "customWidget_getAuthoringPrompt",
+      "customWidget_getSkill",
+      "customWidget_schema",
+      "customWidget_get",
+      "customWidget_validate",
+      "customWidget_previewCreate",
+      "customWidget_previewQuery",
+      "customWidget_previewJournal",
+      "customWidget_update",
+    ];
+    const output = {
+      status: "pass" as const,
+      summary: "The existing widget passed its edited preview and was updated.",
+      definitionId: "widget-1",
+      previewUrl: "http://localhost:3000/manage/custom-widgets/preview/session-2",
+      iterations: 1,
+      evidence: sequence.map((name) => ({ tool: name, outcome: "passed" as const, detail: `${name} completed.` })),
+      remainingIssues: [],
+    };
+    const responses = [
+      ...sequence.map((toolName, index) =>
+        toolCall(toolName, index, toolName === "customWidget_previewQuery" ? { requestId: "query-1" } : {}),
+      ),
+      finalResult(output),
+    ];
+    let responseIndex = 0;
+    const model = new MockLanguageModelV3({
+      doGenerate: async () => responses[responseIndex++] ?? finalResult(output),
+    });
+    const tools = Object.fromEntries(
+      sequence.map((name) => [
+        name,
+        tool({
+          description: name,
+          inputSchema: z.object({ requestId: z.string().optional() }),
+          execute: () => {
+            calls.push(name);
+            if (name === "customWidget_validate") return mcpOutput({ valid: true });
+            if (name === "customWidget_previewCreate") return mcpOutput({ success: true });
+            if (name === "customWidget_previewQuery") return mcpOutput({ ok: true });
+            return mcpOutput({});
+          },
+        }),
+      ]),
+    );
+
+    const result = await createCustomWidgetMcpAgent({ model, tools }).generate({ prompt: "Edit widget-1" });
+    expect(result.output).toEqual(output);
+    expect(
+      getCustomWidgetMcpWorkflowIssues({
+        output: result.output,
+        toolExecutions: getCustomWidgetMcpToolExecutions(result.steps),
+        persist: true,
+        definitionId: "widget-1",
+      }),
+    ).toEqual([]);
+  });
+
+  it("rejects a passing summary that skipped real MCP checks", () => {
+    expect(
+      getCustomWidgetMcpWorkflowIssues({
+        output: {
+          status: "pass",
+          summary: "Looks good.",
+          definitionId: null,
+          previewUrl: null,
+          iterations: 1,
+          evidence: [{ tool: "customWidget_validate", outcome: "passed", detail: "Valid." }],
+          remainingIssues: ["Preview was not tested."],
+        },
+        toolExecutions: successfulExecutions(["customWidget_validate"]),
+        persist: false,
+      }),
+    ).toEqual(
+      expect.arrayContaining([
+        expect.stringContaining("customWidget_previewCreate"),
+        "A passing result cannot contain remaining issues",
+      ]),
+    );
+  });
+
+  it("rejects fabricated evidence and persistence before the final preview checks", () => {
+    const toolsUsed = [
+      "customWidget_getAuthoringPrompt",
+      "customWidget_getSkill",
+      "customWidget_schema",
+      "customWidget_validate",
+      "customWidget_create",
+      "customWidget_previewCreate",
+      "customWidget_previewQuery",
+      "customWidget_previewJournal",
+    ];
+
+    expect(
+      getCustomWidgetMcpWorkflowIssues({
+        output: {
+          status: "pass",
+          summary: "The widget was tested.",
+          definitionId: "widget-1",
+          previewUrl: "http://localhost:3000/manage/custom-widgets/preview/session-3",
+          iterations: 1,
+          evidence: [
+            ...toolsUsed.map((name) => ({ tool: name, outcome: "passed" as const, detail: `${name} completed.` })),
+            { tool: "customWidget_previewAction", outcome: "passed", detail: "The model claimed an action passed." },
+          ],
+          remainingIssues: [],
+        },
+        toolExecutions: successfulExecutions(toolsUsed),
+        persist: true,
+      }),
+    ).toEqual(
+      expect.arrayContaining([
+        "customWidget_create must run after all required preview checks",
+        "Structured evidence names a tool that was not used: customWidget_previewAction",
+      ]),
+    );
+  });
+
+  it("allows secure preview configuration to pause before authenticated requests can run", () => {
+    const toolsUsed = [
+      "customWidget_getAuthoringPrompt",
+      "customWidget_getSkill",
+      "customWidget_schema",
+      "customWidget_validate",
+      "customWidget_previewCreate",
+      "customWidget_configurationRequestUser",
+    ];
+
+    expect(
+      getCustomWidgetMcpWorkflowIssues({
+        output: {
+          status: "needs-user-configuration",
+          summary: "The definition is valid and the authenticated preview is waiting for secure user input.",
+          definitionId: null,
+          previewUrl: "http://localhost:3000/manage/custom-widgets/preview/session-4",
+          iterations: 1,
+          evidence: [...new Set(toolsUsed)].map((name) => ({
+            tool: name,
+            outcome: "passed" as const,
+            detail: `${name} completed.`,
+          })),
+          remainingIssues: [],
+        },
+        toolExecutions: successfulExecutions(toolsUsed),
+        persist: false,
+      }),
+    ).toEqual([]);
+  });
+
+  it("never exposes a credential-writing tool to the model", async () => {
+    const forbiddenTool = tool({
+      description: "Write a plaintext secret",
+      inputSchema: z.object({ secret: z.string() }),
+      execute: vi.fn(),
+    });
+    const safeTool = tool({
+      description: "Request secure user configuration",
+      inputSchema: z.object({}),
+      execute: vi.fn(),
+    });
+    const model = new MockLanguageModelV3({
+      doGenerate: async () =>
+        finalResult({
+          status: "fail",
+          summary: "No source values were supplied.",
+          definitionId: null,
+          previewUrl: null,
+          iterations: 1,
+          evidence: [
+            { tool: "customWidget_configurationRequestUser", outcome: "pending", detail: "User input needed." },
+          ],
+          remainingIssues: ["Source configuration is required."],
+        }),
+    });
+
+    await createCustomWidgetMcpAgent({
+      model,
+      tools: {
+        customWidget_sourceConfigure: forbiddenTool,
+        customWidget_secretSet: forbiddenTool,
+        customWidget_configurationRequestUser: safeTool,
+      },
+    }).generate({ prompt: "Create an authenticated widget" });
+
+    const exposedTools = model.doGenerateCalls[0]?.tools?.map(({ name }) => name);
+    expect(exposedTools).toContain("customWidget_configurationRequestUser");
+    expect(exposedTools).not.toContain("customWidget_sourceConfigure");
+    expect(exposedTools).not.toContain("customWidget_secretSet");
+  });
+});

packages/custom-widgets/tsconfig.scripts.json

@@ -0,0 +1,14 @@
+{
+  "extends": "@homarr/tsconfig/base.json",
+  "compilerOptions": {
+    "tsBuildInfoFile": "node_modules/.cache/scripts-tsbuildinfo.json"
+  },
+  "include": [
+    "scripts/ai-evaluation-cases.ts",
+    "scripts/ai-evaluation.ts",
+    "scripts/evaluate-ai-authoring.ts",
+    "scripts/evaluate-mcp-agent.ts",
+    "scripts/mcp-agent.ts"
+  ],
+  "exclude": ["node_modules"]
+}

pnpm-lock.yaml

@@ -6,6 +6,12 @@ settings:
 
 catalogs:
   default:
+    '@ai-sdk/mcp':
+      specifier: 1.0.0
+      version: 1.0.0
+    '@ai-sdk/provider':
+      specifier: 3.0.2
+      version: 3.0.2
     '@auth/core':
       specifier: ^0.41.2
       version: 0.41.2
@@ -120,6 +126,9 @@ catalogs:
     '@modelcontextprotocol/sdk':
       specifier: 1.29.0
       version: 1.29.0
+    '@openrouter/ai-sdk-provider':
+      specifier: 2.9.1
+      version: 2.9.1
     '@paralleldrive/cuid2':
       specifier: ^3.3.0
       version: 3.3.0
@@ -324,6 +333,9 @@ catalogs:
     adm-zip:
       specifier: 0.5.18
       version: 0.5.18
+    ai:
+      specifier: 6.0.33
+      version: 6.0.33
     babel-plugin-react-compiler:
       specifier: ^1.0.0
       version: 1.0.0
@@ -1858,9 +1870,18 @@ importers:
         specifier: 'catalog:'
         version: 4.4.3
     devDependencies:
+      '@ai-sdk/mcp':
+        specifier: 'catalog:'
+        version: 1.0.0(zod@4.4.3)
+      '@ai-sdk/provider':
+        specifier: 'catalog:'
+        version: 3.0.2
       '@homarr/tsconfig':
         specifier: workspace:^0.1.0
         version: link:../../tooling/typescript
+      '@openrouter/ai-sdk-provider':
+        specifier: 'catalog:'
+        version: 2.9.1(ai@6.0.33(zod@4.4.3))(zod@4.4.3)
       '@types/node':
         specifier: 'catalog:'
         version: 24.13.3
@@ -1870,6 +1891,9 @@ importers:
       '@types/react-dom':
         specifier: 'catalog:'
         version: 19.2.3(@types/react@19.2.17)
+      ai:
+        specifier: 'catalog:'
+        version: 6.0.33(zod@4.4.3)
       jsdom:
         specifier: 'catalog:'
         version: 29.1.1(@noble/hashes@2.0.1)
@@ -2990,12 +3014,28 @@ packages:
     peerDependencies:
       zod: ^3.25.76 || ^4.1.8
 
+  '@ai-sdk/mcp@1.0.0':
+    resolution: {integrity: sha512-0lKYIfUgCvzu/ey5kGrbY8JMgZlRpMopZG8vs0BGzVLnWhWqCx/b0hYW3fpN3IU0+snTNrvJyYVLQ+3gmuKpqg==}
+    engines: {node: '>=18'}
+    peerDependencies:
+      zod: ^3.25.76 || ^4.1.8
+
+  '@ai-sdk/provider-utils@4.0.0':
+    resolution: {integrity: sha512-HyCyOls9I3a3e38+gtvOJOEjuw9KRcvbBnCL5GBuSmJvS9Jh9v3fz7pRC6ha1EUo/ZH1zwvLWYXBMtic8MTguA==}
+    engines: {node: '>=18'}
+    peerDependencies:
+      zod: ^3.25.76 || ^4.1.8
+
   '@ai-sdk/provider-utils@4.0.5':
     resolution: {integrity: sha512-Ow/X/SEkeExTTc1x+nYLB9ZHK2WUId8+9TlkamAx7Tl9vxU+cKzWx2dwjgMHeCN6twrgwkLrrtqckQeO4mxgVA==}
     engines: {node: '>=18'}
     peerDependencies:
       zod: ^3.25.76 || ^4.1.8
 
+  '@ai-sdk/provider@3.0.0':
+    resolution: {integrity: sha512-m9ka3ptkPQbaHHZHqDXDF9C9B5/Mav0KTdky1k2HZ3/nrW2t1AgObxIVPyGDWQNS9FXT/FS6PIoSjpcP/No8rQ==}
+    engines: {node: '>=18'}
+
   '@ai-sdk/provider@3.0.2':
     resolution: {integrity: sha512-HrEmNt/BH/hkQ7zpi2o6N3k1ZR1QTb7z85WYhYygiTxOQuaml4CMtHCWRbric5WPU+RNsYI7r1EpyVQMKO1pYw==}
     engines: {node: '>=18'}
@@ -6293,6 +6333,13 @@ packages:
   '@open-draft/until@2.1.0':
     resolution: {integrity: sha512-U69T3ItWHvLwGg5eJ0n3I62nWuE6ilHlmz7zM0npLBRvPRd7e6NYmg54vvRtP5mZG7kZqZCFVdsTWo7BPtBujg==}
 
+  '@openrouter/ai-sdk-provider@2.9.1':
+    resolution: {integrity: sha512-okgq07Vdkro4CB5INbfhwa0e6VR1HS7sidNcfHN/MeXLJvX1JmQCff/vem6tcxwT9r1avyFrXSlfv9B28D/Pag==}
+    engines: {node: '>=18'}
+    peerDependencies:
+      ai: ^6.0.0
+      zod: ^3.25.0 || ^4.0.0
+
   '@opentelemetry/api@1.9.0':
     resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==}
     engines: {node: '>=8.0.0'}
@@ -15872,6 +15919,20 @@ snapshots:
       '@vercel/oidc': 3.1.0
       zod: 4.4.3
 
+  '@ai-sdk/mcp@1.0.0(zod@4.4.3)':
+    dependencies:
+      '@ai-sdk/provider': 3.0.0
+      '@ai-sdk/provider-utils': 4.0.0(zod@4.4.3)
+      pkce-challenge: 5.0.1
+      zod: 4.4.3
+
+  '@ai-sdk/provider-utils@4.0.0(zod@4.4.3)':
+    dependencies:
+      '@ai-sdk/provider': 3.0.0
+      '@standard-schema/spec': 1.1.0
+      eventsource-parser: 3.1.0
+      zod: 4.4.3
+
   '@ai-sdk/provider-utils@4.0.5(zod@4.4.3)':
     dependencies:
       '@ai-sdk/provider': 3.0.2
@@ -15879,6 +15940,10 @@ snapshots:
       eventsource-parser: 3.1.0
       zod: 4.4.3
 
+  '@ai-sdk/provider@3.0.0':
+    dependencies:
+      json-schema: 0.4.0
+
   '@ai-sdk/provider@3.0.2':
     dependencies:
       json-schema: 0.4.0
@@ -20135,6 +20200,11 @@ snapshots:
   '@open-draft/until@2.1.0':
     optional: true
 
+  '@openrouter/ai-sdk-provider@2.9.1(ai@6.0.33(zod@4.4.3))(zod@4.4.3)':
+    dependencies:
+      ai: 6.0.33(zod@4.4.3)
+      zod: 4.4.3
+
   '@opentelemetry/api@1.9.0': {}
 
   '@opentelemetry/semantic-conventions@1.41.1': {}

pnpm-workspace.yaml

@@ -3,6 +3,9 @@ packages:
   - packages/*
   - tooling/*
 catalog:
+  "@ai-sdk/mcp": 1.0.0
+  "@ai-sdk/provider": 3.0.2
+  "@openrouter/ai-sdk-provider": 2.9.1
   "@codemirror/autocomplete": ^6.20.3
   "@codemirror/commands": ^6.10.3
   "@codemirror/lang-css": ^6.3.1
@@ -114,6 +117,7 @@ catalog:
   "@xterm/addon-fit": 0.11.0
   "@xterm/xterm": ^6.0.0
   adm-zip: 0.5.18
+  ai: 6.0.33
   acorn: ^8.16.0
   acorn-jsx: ^5.3.2
   jszip: ^3.10.1