test custom widget assistant authoring

packages/custom-widgets/src/test/ai-assistant-evaluation.spec.ts

@@ -0,0 +1,189 @@
+import { describe, expect, it } from "vitest";
+
+import { CUSTOM_WIDGET_AI_EVALUATION_CASES } from "../../scripts/ai-evaluation-cases";
+import {
+  createAssistantEvaluationState,
+  customWidgetAssistantEvaluationToolDefinitions,
+  executeAssistantEvaluationTool,
+  getAssistantEvaluationPreviewResponse,
+  getAssistantEvaluationRecoveryToolName,
+  getRequiredAssistantEvaluationRequestParams,
+  getAssistantEvaluationLifecycleIssues,
+} from "../../scripts/ai-assistant-evaluation";
+
+const widget = {
+  $schema: "homarr-custom-widget-v2",
+  name: "Homelab health",
+  description: "Compact health overview",
+  sources: {
+    default: {
+      baseUrl: "https://status.example.test",
+      networkScope: "public",
+      auth: "none",
+    },
+  },
+  requests: {
+    health: {
+      source: "default",
+      kind: "query",
+      method: "GET",
+      path: "/v1/health",
+      trigger: "load",
+      auth: "inherit",
+      permission: "view",
+      cacheSeconds: 30,
+    },
+  },
+  options: {},
+  templateLines: [
+    '<Stack gap="sm">',
+    '  <Group justify="space-between"><Text fw={700}>Homelab health</Text><RefreshButton /></Group>',
+    "  {status.health?.loading ? <Skeleton h={64} /> : null}",
+    '  {status.health?.ok === false ? <Alert color="red">Health data is unavailable.</Alert> : null}',
+    '  {data.health ? <Stack><Text fw={700}>{data.health.status}</Text><Text>{data.health.openIncidents} incidents · {data.health.latencyMs} ms · {data.health.checkedAt}</Text>{(data.health.services ?? []).map(service => <Text key={service.id}>{service.name}: {service.status} · {service.latencyMs} ms</Text>)}</Stack> : <Text c="dimmed">No health data</Text>}',
+    "</Stack>",
+  ],
+};
+
+describe("Custom Widget assistant live evaluation harness", () => {
+  it("exposes the complete documentation and authoring lifecycle", () => {
+    expect(customWidgetAssistantEvaluationToolDefinitions.map(({ function: definition }) => definition.name)).toEqual(
+      expect.arrayContaining([
+        "customWidget_getSkill",
+        "customWidget_schema",
+        "customWidget_getComponentCatalog",
+        "customWidget_validate",
+        "customWidget_previewCreate",
+        "customWidget_previewQuery",
+        "customWidget_createFromPreview",
+      ]),
+    );
+  });
+
+  it("recovers a stalled model at the next required evidence checkpoint", () => {
+    const state = createAssistantEvaluationState();
+    expect(getAssistantEvaluationRecoveryToolName(state)).toBe("customWidget_getSkill");
+    state.calledTools.push("customWidget_getSkill", "customWidget_schema", "customWidget_getComponentCatalog");
+    expect(getAssistantEvaluationRecoveryToolName(state)).toBe("customWidget_validate");
+  });
+
+  it("rejects skipped evidence and persists the exact fixture-tested definition", () => {
+    const testCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === "fake-service-health");
+    if (!testCase) throw new Error("Fake service-health case is missing");
+    const state = createAssistantEvaluationState();
+
+    expect(
+      executeAssistantEvaluationTool(testCase, state, "customWidget_previewCreate", { definition: widget }),
+    ).toEqual({ error: "Validate this exact complete definition before creating its preview." });
+    executeAssistantEvaluationTool(testCase, state, "customWidget_getSkill", {});
+    executeAssistantEvaluationTool(testCase, state, "customWidget_schema", {});
+    executeAssistantEvaluationTool(testCase, state, "customWidget_getComponentCatalog", {});
+    expect(executeAssistantEvaluationTool(testCase, state, "customWidget_validate", { widget })).toMatchObject({
+      valid: true,
+      summary: { requestIds: ["health"] },
+    });
+    expect(
+      executeAssistantEvaluationTool(testCase, state, "customWidget_previewCreate", { definition: widget }),
+    ).toMatchObject({
+      success: true,
+      previewSession: { id: "preview-1" },
+      queries: [{ requestId: "health" }],
+    });
+    expect(
+      executeAssistantEvaluationTool(testCase, state, "customWidget_createFromPreview", {
+        previewSessionId: "preview-1",
+      }),
+    ).toEqual({ error: "Test every preview query before creation: health" });
+    expect(
+      executeAssistantEvaluationTool(testCase, state, "customWidget_previewQuery", {
+        sessionId: "preview-1",
+        requestId: "health",
+        params: {},
+      }),
+    ).toMatchObject({ ok: true, status: 200, data: testCase.sampleResponse });
+    expect(
+      executeAssistantEvaluationTool(testCase, state, "customWidget_createFromPreview", {
+        previewSessionId: "preview-1",
+      }),
+    ).toMatchObject({ id: "created-fake-service-health" });
+    expect(state.createdWidget).toMatchObject({ name: widget.name, template: widget.templateLines.join("\n") });
+    expect(getAssistantEvaluationLifecycleIssues(state)).toEqual([]);
+  });
+
+  it("rejects excess parallel component lookups while keeping validation available", () => {
+    const testCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === "fake-service-health");
+    if (!testCase) throw new Error("Fake service-health case is missing");
+    const state = createAssistantEvaluationState();
+
+    for (let index = 0; index < 8; index += 1) {
+      expect(
+        executeAssistantEvaluationTool(testCase, state, "customWidget_getComponent", { name: "Text" }),
+      ).toMatchObject({ name: "Text" });
+    }
+    expect(executeAssistantEvaluationTool(testCase, state, "customWidget_getComponent", { name: "Stack" })).toEqual({
+      error:
+        "The targeted component-document budget is exhausted. Continue with validation using the documentation already loaded.",
+    });
+    expect(executeAssistantEvaluationTool(testCase, state, "customWidget_validate", { widget })).toMatchObject({
+      valid: true,
+    });
+  });
+
+  it("uses four targeted component documents after loading a complete example", () => {
+    const testCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === "fake-service-health");
+    if (!testCase) throw new Error("Fake service-health case is missing");
+    const state = createAssistantEvaluationState();
+    executeAssistantEvaluationTool(testCase, state, "customWidget_getExample", { name: "service-dashboard" });
+
+    for (let index = 0; index < 4; index += 1) {
+      expect(
+        executeAssistantEvaluationTool(testCase, state, "customWidget_getComponent", { name: "Text" }),
+      ).toMatchObject({ name: "Text" });
+    }
+    expect(executeAssistantEvaluationTool(testCase, state, "customWidget_getComponent", { name: "Stack" })).toEqual({
+      error:
+        "The targeted component-document budget is exhausted. Continue with validation using the documentation already loaded.",
+    });
+  });
+
+  it("enforces explicit multi-iteration preview requests before persistence", () => {
+    const baseCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === "fake-service-health");
+    if (!baseCase) throw new Error("Fake service-health case is missing");
+    const testCase = { ...baseCase, minimumPreviewCycles: 2 };
+    const state = createAssistantEvaluationState();
+
+    executeAssistantEvaluationTool(testCase, state, "customWidget_getSkill", {});
+    executeAssistantEvaluationTool(testCase, state, "customWidget_schema", {});
+    executeAssistantEvaluationTool(testCase, state, "customWidget_getComponentCatalog", {});
+    executeAssistantEvaluationTool(testCase, state, "customWidget_validate", { widget });
+    executeAssistantEvaluationTool(testCase, state, "customWidget_previewCreate", { definition: widget });
+    executeAssistantEvaluationTool(testCase, state, "customWidget_previewQuery", {
+      sessionId: "preview-1",
+      requestId: "health",
+      params: {},
+    });
+    expect(
+      executeAssistantEvaluationTool(testCase, state, "customWidget_createFromPreview", {
+        previewSessionId: "preview-1",
+      }),
+    ).toMatchObject({ error: expect.stringContaining("at least 2 validated preview-and-query cycles") });
+    expect(getAssistantEvaluationRecoveryToolName(state, 2)).toBe("customWidget_validate");
+  });
+
+  it("routes multi-request fixtures and requires manual preview parameters", () => {
+    const testCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === "pokedex");
+    if (!testCase) throw new Error("Pokédex case is missing");
+    const detailRequest = {
+      source: "default",
+      kind: "query" as const,
+      method: "GET" as const,
+      path: "/api/v2/pokemon/{param:name}",
+      trigger: "manual" as const,
+      auth: "inherit" as const,
+      permission: "view" as const,
+    };
+
+    expect(getAssistantEvaluationPreviewResponse(testCase, detailRequest)).toMatchObject({ name: "pikachu" });
+    expect(getRequiredAssistantEvaluationRequestParams(detailRequest)).toEqual(["name"]);
+  });
+});

packages/custom-widgets/src/test/ai-evaluation.spec.ts

@@ -3,9 +3,12 @@ import { describe, expect, it } from "vitest";
 import { CUSTOM_WIDGET_AI_EVALUATION_CASES } from "../../scripts/ai-evaluation-cases";
 import {
   buildEvaluationPrompt,
+  buildJudgePrompt,
   buildRepairPrompt,
   DEFAULT_GENERATOR_MODEL,
   DEFAULT_JUDGE_MODEL,
+  getDeterministicEvaluationIssues,
+  getEvaluationResponseFixtureText,
   getJudgeResponseFormat,
   judgePasses,
   parseJudgeResult,
@@ -46,9 +49,106 @@ const makeJudgeResult = (score: number) => ({
 });
 
 describe("AI authoring evaluation", () => {
-  it("defines five distinct complex scenarios", () => {
-    expect(CUSTOM_WIDGET_AI_EVALUATION_CASES).toHaveLength(5);
-    expect(new Set(CUSTOM_WIDGET_AI_EVALUATION_CASES.map((entry) => entry.id)).size).toBe(5);
+  it("defines distinct complex and public-API scenarios", () => {
+    expect(CUSTOM_WIDGET_AI_EVALUATION_CASES).toHaveLength(9);
+    expect(new Set(CUSTOM_WIDGET_AI_EVALUATION_CASES.map((entry) => entry.id)).size).toBe(9);
+    expect(CUSTOM_WIDGET_AI_EVALUATION_CASES.map(({ id }) => id)).toEqual(
+      expect.arrayContaining([
+        "pokedex",
+        "fake-service-health",
+        "coinmarketcap-keyless",
+        "bored-activity",
+        "agify-name",
+      ]),
+    );
+  });
+
+  it("grounds fixture-backed scenarios in verified routes and authentication", () => {
+    const testCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === "agify-name");
+    if (!testCase) throw new Error("Agify evaluation case is missing");
+    const validWidget = {
+      $schema: "homarr-custom-widget-v2" as const,
+      name: "Agify",
+      sources: {
+        default: {
+          baseUrl: "https://api.agify.io",
+          networkScope: "public" as const,
+          auth: { type: "apiKeyQuery" as const, name: "apikey" },
+        },
+      },
+      requests: {
+        prediction: {
+          source: "default",
+          kind: "query" as const,
+          method: "GET" as const,
+          path: "/",
+          trigger: "manual" as const,
+          query: { name: { $param: "name" }, country_id: { $param: "country" } },
+          auth: "inherit" as const,
+          permission: "view" as const,
+        },
+      },
+      options: {},
+      template:
+        '<Stack><TextInput bind="name" /><SubFetch requestId="prediction" params={{ name: inputs.name, country: "US" }}>{(result) => <Text>{result.age} from {result.count}</Text>}</SubFetch></Stack>',
+    };
+    expect(getDeterministicEvaluationIssues(testCase, validWidget)).toEqual([]);
+    expect(
+      getDeterministicEvaluationIssues(testCase, {
+        ...validWidget,
+        sources: { default: { ...validWidget.sources.default, auth: "none" as const } },
+      }),
+    ).toContainEqual(expect.objectContaining({ path: ["sources", "default", "auth"] }));
+  });
+
+  it("checks the public API response contract instead of only the endpoint", () => {
+    const testCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === "coinmarketcap-keyless");
+    if (!testCase) throw new Error("CoinMarketCap evaluation case is missing");
+    const widget = {
+      $schema: "homarr-custom-widget-v2" as const,
+      name: "Crypto",
+      sources: {
+        default: {
+          baseUrl: "https://pro-api.coinmarketcap.com",
+          networkScope: "public" as const,
+          auth: "none" as const,
+        },
+      },
+      requests: {
+        prices: {
+          source: "default",
+          kind: "query" as const,
+          method: "GET" as const,
+          path: "/public-api/v3/cryptocurrency/quotes/latest",
+          trigger: "load" as const,
+          query: { id: "1,1027,5426", convert: "USD" },
+          auth: "inherit" as const,
+          permission: "view" as const,
+        },
+      },
+      options: {},
+      template:
+        "<Stack><RefreshButton />{data.prices?.data?.map(item => <Text key={item.symbol}>{item.quote?.[0]?.percent_change_24h} {item.quote?.[0]?.market_cap} {item.quote?.[0]?.volume_24h} {item.last_updated}</Text>)}</Stack>",
+    };
+
+    expect(getDeterministicEvaluationIssues(testCase, widget)).toEqual([]);
+    expect(
+      getDeterministicEvaluationIssues(testCase, {
+        ...widget,
+        requests: { prices: { ...widget.requests.prices, query: { id: "1,1027", convert: "USD" } } },
+      }),
+    ).toContainEqual(expect.objectContaining({ path: ["requests"] }));
+  });
+
+  it("includes every path-specific Pokédex fixture in generator and judge context", () => {
+    const testCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === "pokedex");
+    if (!testCase) throw new Error("Pokédex evaluation case is missing");
+    const fixtureText = getEvaluationResponseFixtureText(testCase);
+
+    expect(fixtureText).toContain("/api/v2/pokemon/{param:name}");
+    expect(fixtureText).toContain('"official-artwork"');
+    expect(fixtureText).toContain('"results"');
+    expect(testCase.minimumPreviewCycles).toBe(3);
   });
 
   it("uses the exact clipboard prompt and grounded API notes", () => {
@@ -70,8 +170,8 @@ describe("AI authoring evaluation", () => {
   });
 
   it("uses the requested DeepSeek models and strict structured judge output", () => {
-    expect(DEFAULT_GENERATOR_MODEL).toBe("deepseek/deepseek-v4-pro");
-    expect(DEFAULT_JUDGE_MODEL).toBe("deepseek/deepseek-v4-flash");
+    expect(DEFAULT_GENERATOR_MODEL).toBe("~deepseek/deepseek-v4-flash-latest");
+    expect(DEFAULT_JUDGE_MODEL).toBe("~deepseek/deepseek-v4-flash-latest");
     const format = getJudgeResponseFormat();
     expect(format.type).toBe("json_schema");
     expect(format.json_schema.strict).toBe(true);
@@ -89,6 +189,28 @@ describe("AI authoring evaluation", () => {
     expect(judgePasses({ ...makeJudgeResult(95), fatalProblems: ["A requested core action is missing."] })).toBe(false);
   });
 
+  it("gives the harsh judge the authoritative request-state runtime contract", () => {
+    const testCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === "fake-service-health");
+    if (!testCase) throw new Error("Fake service-health case is missing");
+    const widget = {
+      $schema: "homarr-custom-widget-v2" as const,
+      name: "Health",
+      sources: {
+        default: { baseUrl: "https://status.example.test", networkScope: "public" as const, auth: "none" as const },
+      },
+      requests: {},
+      options: {},
+      template: "<Text>Health</Text>",
+    };
+    const prompt = buildJudgePrompt(testCase, widget);
+
+    expect(prompt).toContain("status.<requestId> with loading/ok/status/error fields");
+    expect(prompt).toContain("RefreshButton is an installed runtime helper");
+    expect(prompt).toContain(JSON.stringify(testCase.sampleResponse, null, 2));
+    expect(prompt).toContain("# Bundled file: references/runtime.md");
+    expect(prompt).toContain("does not prove API correctness, visual quality, usefulness, or accessibility");
+  });
+
   it("computes the weighted score and refuses inflated advisory verdicts", () => {
     const parsed = parseJudgeResult(
       JSON.stringify({
@@ -105,4 +227,18 @@ describe("AI authoring evaluation", () => {
     expect(parsed.total).toBe(91);
     expect(parsed.verdict).toBe("fail");
   });
+
+  it("accepts a provider response that wraps structured review JSON in a code fence", () => {
+    const parsed = parseJudgeResult(`\`\`\`json\n${JSON.stringify(makeJudgeResult(90))}\n\`\`\``);
+    expect(parsed.total).toBe(90);
+    expect(parsed.verdict).toBe("pass");
+  });
+
+  it("normalizes a prose 'none' entry out of an otherwise empty fatal-problem list", () => {
+    const parsed = parseJudgeResult(
+      JSON.stringify({ ...makeJudgeResult(90), fatalProblems: ["None: all requested capabilities are present."] }),
+    );
+    expect(parsed.fatalProblems).toEqual([]);
+    expect(parsed.verdict).toBe("pass");
+  });
 });

packages/custom-widgets/src/test/ai-prompt.spec.ts

@@ -47,6 +47,13 @@ describe("AI prompt", () => {
     expect(CUSTOM_WIDGET_MCP_AUTHORING_PROMPT).toContain("customWidget_previewQuery once for every query");
     expect(CUSTOM_WIDGET_MCP_AUTHORING_PROMPT).toContain("customWidget_createFromPreview");
     expect(CUSTOM_WIDGET_MCP_AUTHORING_PROMPT).toContain("customWidget_getSharedProps");
+    expect(CUSTOM_WIDGET_MCP_AUTHORING_PROMPT).toContain("at most eight named component documents");
+    expect(CUSTOM_WIDGET_MCP_AUTHORING_PROMPT).toContain("at most four after loading a complete example");
+    expect(CUSTOM_WIDGET_MCP_AUTHORING_PROMPT).toContain("Treat preview data as the binding contract");
+    expect(CUSTOM_WIDGET_MCP_AUTHORING_PROMPT).toContain("freshness context");
+    expect(CUSTOM_WIDGET_MCP_AUTHORING_PROMPT).toContain("Make initial states actionable");
+    expect(CUSTOM_WIDGET_MCP_AUTHORING_PROMPT).toContain("compact narrow-tile rows");
+    expect(CUSTOM_WIDGET_MCP_AUTHORING_PROMPT).toContain("Label standalone icons");
     expect(CUSTOM_WIDGET_MCP_AUTHORING_PROMPT).toContain("Prefer templateLines");
     expect(CUSTOM_WIDGET_MCP_AUTHORING_PROMPT.indexOf("customWidget_previewQuery")).toBeLessThan(
       CUSTOM_WIDGET_MCP_AUTHORING_PROMPT.indexOf("customWidget_create"),

packages/custom-widgets/src/test/authoring-resources.spec.ts

@@ -31,6 +31,10 @@ describe("authoring resources", () => {
       expect(bundledContent).toContain(content.trim());
     }
     expect(bundledContent).toContain(skill.skillMd.trim());
+    expect(bundledContent).toContain("at most eight named component documents");
+    expect(bundledContent).toContain("at most four after loading a complete example");
+    expect(bundledContent).toContain("successful preview response as the binding contract");
+    expect(bundledContent).toContain("Do not use an unlabeled decorative icon as an empty state");
     expect(Buffer.byteLength(JSON.stringify(skill), "utf8")).toBeLessThan(12_000);
   });