test: harden custom widget AI authoring
This commit is contained in:
1 parent
2f0c0a07dd
commit
6eff1da08f
14 files changed
+1056
-23
No files matched your search
@@ -7,10 +7,12 @@ import {
|
||||
DEFAULT_GENERATOR_MODEL,
|
||||
DEFAULT_JUDGE_MODEL,
|
||||
getJudgeResponseFormat,
|
||||
getScenarioAcceptanceIssues,
|
||||
judgePasses,
|
||||
parseJudgeResult,
|
||||
} from "../../scripts/ai-evaluation";
|
||||
import type { CustomWidgetJudgeResult } from "../../scripts/ai-evaluation";
|
||||
import { BUNDLED_CUSTOM_WIDGETS, CUSTOM_WIDGET_STARTER, customWidgetDefinitionSchema } from "../core";
|
||||
|
||||
const categoryNames = [
|
||||
"schemaAndBindings",
|
||||
@@ -49,6 +51,13 @@ describe("AI authoring evaluation", () => {
|
||||
it("defines five distinct complex scenarios", () => {
|
||||
expect(CUSTOM_WIDGET_AI_EVALUATION_CASES).toHaveLength(5);
|
||||
expect(new Set(CUSTOM_WIDGET_AI_EVALUATION_CASES.map((entry) => entry.id)).size).toBe(5);
|
||||
expect(CUSTOM_WIDGET_AI_EVALUATION_CASES.map(({ id }) => id)).toEqual([
|
||||
"pokedex",
|
||||
"portainer-containers",
|
||||
"football-dashboard",
|
||||
"jellyfin-activity",
|
||||
"home-assistant-control",
|
||||
]);
|
||||
});
|
||||
|
||||
it("uses the exact clipboard prompt and grounded API notes", () => {
|
||||
@@ -69,6 +78,25 @@ describe("AI authoring evaluation", () => {
|
||||
expect(prompt).toContain("bad response");
|
||||
});
|
||||
|
||||
it("requires grounded scenario capabilities before model judging", () => {
|
||||
const testCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === "pokedex");
|
||||
const pokedex = BUNDLED_CUSTOM_WIDGETS.find(({ id }) => id === "seed-pokedex");
|
||||
if (!testCase || !pokedex) throw new Error("Expected Pokédex evaluation fixtures");
|
||||
expect(getScenarioAcceptanceIssues(testCase, customWidgetDefinitionSchema.parse(pokedex.widget))).toEqual([]);
|
||||
|
||||
const issues = getScenarioAcceptanceIssues(
|
||||
testCase,
|
||||
customWidgetDefinitionSchema.parse({ ...CUSTOM_WIDGET_STARTER, template: "<Text>Pokémon</Text>" }),
|
||||
);
|
||||
expect(issues.map(({ message }) => message)).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.stringContaining("Pokémon list"),
|
||||
expect.stringContaining("manual Pokémon detail"),
|
||||
expect.stringContaining("SubFetch"),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("uses the requested DeepSeek models and strict structured judge output", () => {
|
||||
expect(DEFAULT_GENERATOR_MODEL).toBe("deepseek/deepseek-v4-pro");
|
||||
expect(DEFAULT_JUDGE_MODEL).toBe("deepseek/deepseek-v4-flash");
|
||||
|
||||
@@ -0,0 +1,322 @@
|
||||
import type { LanguageModelV3GenerateResult } from "@ai-sdk/provider";
|
||||
import { tool } from "ai";
|
||||
import { MockLanguageModelV3 } from "ai/test";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { z } from "zod/v4";
|
||||
|
||||
import {
|
||||
createCustomWidgetMcpAgent,
|
||||
getCustomWidgetMcpToolExecutions,
|
||||
getCustomWidgetMcpWorkflowIssues,
|
||||
} from "../../scripts/mcp-agent";
|
||||
|
||||
const usage = {
|
||||
inputTokens: { total: 10, noCache: 10, cacheRead: undefined, cacheWrite: undefined },
|
||||
outputTokens: { total: 10, text: 10, reasoning: undefined },
|
||||
};
|
||||
|
||||
const toolCall = (
|
||||
toolName: string,
|
||||
index: number,
|
||||
input: Record<string, unknown> = {},
|
||||
): LanguageModelV3GenerateResult => ({
|
||||
content: [{ type: "tool-call", toolCallId: `call-${index}`, toolName, input: JSON.stringify(input) }],
|
||||
finishReason: { unified: "tool-calls", raw: undefined },
|
||||
usage,
|
||||
warnings: [],
|
||||
});
|
||||
|
||||
const finalResult = (value: unknown): LanguageModelV3GenerateResult => ({
|
||||
content: [{ type: "text", text: JSON.stringify(value) }],
|
||||
finishReason: { unified: "stop", raw: undefined },
|
||||
usage,
|
||||
warnings: [],
|
||||
});
|
||||
|
||||
const successfulExecutions = (tools: readonly string[]) =>
|
||||
tools.map((toolName) => ({ tool: toolName, succeeded: true }));
|
||||
const mcpOutput = (value: unknown) => ({ content: [{ type: "text" as const, text: JSON.stringify(value) }] });
|
||||
|
||||
describe("Custom Widget MCP agent", () => {
|
||||
it("runs a repair, preview, and persistence workflow before structured success", async () => {
|
||||
const calls: string[] = [];
|
||||
let validationCount = 0;
|
||||
const makeTool = (name: string) =>
|
||||
tool({
|
||||
description: name,
|
||||
inputSchema: z.object({ requestId: z.string().optional() }),
|
||||
execute: vi.fn(() => {
|
||||
calls.push(name);
|
||||
if (name === "customWidget_validate") {
|
||||
validationCount += 1;
|
||||
return mcpOutput({ valid: validationCount > 1 });
|
||||
}
|
||||
if (name === "customWidget_previewCreate") return mcpOutput({ success: true });
|
||||
if (name === "customWidget_previewQuery" || name === "customWidget_previewAction") {
|
||||
return mcpOutput({ ok: true });
|
||||
}
|
||||
return mcpOutput({});
|
||||
}),
|
||||
});
|
||||
const sequence = [
|
||||
"customWidget_getAuthoringPrompt",
|
||||
"customWidget_getSkill",
|
||||
"customWidget_schema",
|
||||
"customWidget_validate",
|
||||
"customWidget_validate",
|
||||
"customWidget_previewCreate",
|
||||
"customWidget_previewQuery",
|
||||
"customWidget_previewQuery",
|
||||
"customWidget_previewAction",
|
||||
"customWidget_previewJournal",
|
||||
"customWidget_create",
|
||||
];
|
||||
const output = {
|
||||
status: "pass" as const,
|
||||
summary: "The repaired widget passed validation and preview checks and was persisted.",
|
||||
definitionId: "widget-1",
|
||||
previewUrl: "http://localhost:3000/manage/custom-widgets/preview/session-1",
|
||||
iterations: 2,
|
||||
evidence: [...new Set(sequence)].map((name) => ({
|
||||
tool: name,
|
||||
outcome: name === "customWidget_validate" ? ("repaired" as const) : ("passed" as const),
|
||||
detail: `${name} completed.`,
|
||||
})),
|
||||
remainingIssues: [],
|
||||
};
|
||||
let queryIndex = 0;
|
||||
const responses = [
|
||||
...sequence.map((toolName, index) => {
|
||||
if (toolName === "customWidget_previewQuery") {
|
||||
queryIndex += 1;
|
||||
return toolCall(toolName, index, { requestId: `query-${queryIndex}` });
|
||||
}
|
||||
if (toolName === "customWidget_previewAction") return toolCall(toolName, index, { requestId: "action-1" });
|
||||
return toolCall(toolName, index);
|
||||
}),
|
||||
finalResult(output),
|
||||
];
|
||||
let responseIndex = 0;
|
||||
const model = new MockLanguageModelV3({
|
||||
doGenerate: async () => responses[responseIndex++] ?? finalResult(output),
|
||||
});
|
||||
const tools = Object.fromEntries(sequence.map((name) => [name, makeTool(name)]));
|
||||
|
||||
const result = await createCustomWidgetMcpAgent({ model, tools }).generate({ prompt: "Build a widget" });
|
||||
expect(result.output).toEqual(output);
|
||||
expect(calls).toEqual(sequence);
|
||||
expect(model.doGenerateCalls.at(-1)?.responseFormat).toMatchObject({ type: "json" });
|
||||
const toolExecutions = getCustomWidgetMcpToolExecutions(result.steps);
|
||||
expect(toolExecutions.filter(({ tool: toolName }) => toolName === "customWidget_validate")).toEqual([
|
||||
{ tool: "customWidget_validate", succeeded: false },
|
||||
{ tool: "customWidget_validate", succeeded: true },
|
||||
]);
|
||||
expect(
|
||||
toolExecutions
|
||||
.filter(({ tool: toolName }) => toolName === "customWidget_previewQuery")
|
||||
.map(({ requestId }) => requestId),
|
||||
).toEqual(["query-1", "query-2"]);
|
||||
expect(
|
||||
getCustomWidgetMcpWorkflowIssues({
|
||||
output: result.output,
|
||||
toolExecutions,
|
||||
persist: true,
|
||||
requiredQueryCount: 2,
|
||||
requiredActionCount: 1,
|
||||
}),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("uses the current definition and updates it in the edit workflow", async () => {
|
||||
const calls: string[] = [];
|
||||
const sequence = [
|
||||
"customWidget_getAuthoringPrompt",
|
||||
"customWidget_getSkill",
|
||||
"customWidget_schema",
|
||||
"customWidget_get",
|
||||
"customWidget_validate",
|
||||
"customWidget_previewCreate",
|
||||
"customWidget_previewQuery",
|
||||
"customWidget_previewJournal",
|
||||
"customWidget_update",
|
||||
];
|
||||
const output = {
|
||||
status: "pass" as const,
|
||||
summary: "The existing widget passed its edited preview and was updated.",
|
||||
definitionId: "widget-1",
|
||||
previewUrl: "http://localhost:3000/manage/custom-widgets/preview/session-2",
|
||||
iterations: 1,
|
||||
evidence: sequence.map((name) => ({ tool: name, outcome: "passed" as const, detail: `${name} completed.` })),
|
||||
remainingIssues: [],
|
||||
};
|
||||
const responses = [
|
||||
...sequence.map((toolName, index) =>
|
||||
toolCall(toolName, index, toolName === "customWidget_previewQuery" ? { requestId: "query-1" } : {}),
|
||||
),
|
||||
finalResult(output),
|
||||
];
|
||||
let responseIndex = 0;
|
||||
const model = new MockLanguageModelV3({
|
||||
doGenerate: async () => responses[responseIndex++] ?? finalResult(output),
|
||||
});
|
||||
const tools = Object.fromEntries(
|
||||
sequence.map((name) => [
|
||||
name,
|
||||
tool({
|
||||
description: name,
|
||||
inputSchema: z.object({ requestId: z.string().optional() }),
|
||||
execute: () => {
|
||||
calls.push(name);
|
||||
if (name === "customWidget_validate") return mcpOutput({ valid: true });
|
||||
if (name === "customWidget_previewCreate") return mcpOutput({ success: true });
|
||||
if (name === "customWidget_previewQuery") return mcpOutput({ ok: true });
|
||||
return mcpOutput({});
|
||||
},
|
||||
}),
|
||||
]),
|
||||
);
|
||||
|
||||
const result = await createCustomWidgetMcpAgent({ model, tools }).generate({ prompt: "Edit widget-1" });
|
||||
expect(result.output).toEqual(output);
|
||||
expect(
|
||||
getCustomWidgetMcpWorkflowIssues({
|
||||
output: result.output,
|
||||
toolExecutions: getCustomWidgetMcpToolExecutions(result.steps),
|
||||
persist: true,
|
||||
definitionId: "widget-1",
|
||||
}),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("rejects a passing summary that skipped real MCP checks", () => {
|
||||
expect(
|
||||
getCustomWidgetMcpWorkflowIssues({
|
||||
output: {
|
||||
status: "pass",
|
||||
summary: "Looks good.",
|
||||
definitionId: null,
|
||||
previewUrl: null,
|
||||
iterations: 1,
|
||||
evidence: [{ tool: "customWidget_validate", outcome: "passed", detail: "Valid." }],
|
||||
remainingIssues: ["Preview was not tested."],
|
||||
},
|
||||
toolExecutions: successfulExecutions(["customWidget_validate"]),
|
||||
persist: false,
|
||||
}),
|
||||
).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.stringContaining("customWidget_previewCreate"),
|
||||
"A passing result cannot contain remaining issues",
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects fabricated evidence and persistence before the final preview checks", () => {
|
||||
const toolsUsed = [
|
||||
"customWidget_getAuthoringPrompt",
|
||||
"customWidget_getSkill",
|
||||
"customWidget_schema",
|
||||
"customWidget_validate",
|
||||
"customWidget_create",
|
||||
"customWidget_previewCreate",
|
||||
"customWidget_previewQuery",
|
||||
"customWidget_previewJournal",
|
||||
];
|
||||
|
||||
expect(
|
||||
getCustomWidgetMcpWorkflowIssues({
|
||||
output: {
|
||||
status: "pass",
|
||||
summary: "The widget was tested.",
|
||||
definitionId: "widget-1",
|
||||
previewUrl: "http://localhost:3000/manage/custom-widgets/preview/session-3",
|
||||
iterations: 1,
|
||||
evidence: [
|
||||
...toolsUsed.map((name) => ({ tool: name, outcome: "passed" as const, detail: `${name} completed.` })),
|
||||
{ tool: "customWidget_previewAction", outcome: "passed", detail: "The model claimed an action passed." },
|
||||
],
|
||||
remainingIssues: [],
|
||||
},
|
||||
toolExecutions: successfulExecutions(toolsUsed),
|
||||
persist: true,
|
||||
}),
|
||||
).toEqual(
|
||||
expect.arrayContaining([
|
||||
"customWidget_create must run after all required preview checks",
|
||||
"Structured evidence names a tool that was not used: customWidget_previewAction",
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("allows secure preview configuration to pause before authenticated requests can run", () => {
|
||||
const toolsUsed = [
|
||||
"customWidget_getAuthoringPrompt",
|
||||
"customWidget_getSkill",
|
||||
"customWidget_schema",
|
||||
"customWidget_validate",
|
||||
"customWidget_previewCreate",
|
||||
"customWidget_configurationRequestUser",
|
||||
];
|
||||
|
||||
expect(
|
||||
getCustomWidgetMcpWorkflowIssues({
|
||||
output: {
|
||||
status: "needs-user-configuration",
|
||||
summary: "The definition is valid and the authenticated preview is waiting for secure user input.",
|
||||
definitionId: null,
|
||||
previewUrl: "http://localhost:3000/manage/custom-widgets/preview/session-4",
|
||||
iterations: 1,
|
||||
evidence: [...new Set(toolsUsed)].map((name) => ({
|
||||
tool: name,
|
||||
outcome: "passed" as const,
|
||||
detail: `${name} completed.`,
|
||||
})),
|
||||
remainingIssues: [],
|
||||
},
|
||||
toolExecutions: successfulExecutions(toolsUsed),
|
||||
persist: false,
|
||||
}),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("never exposes a credential-writing tool to the model", async () => {
|
||||
const forbiddenTool = tool({
|
||||
description: "Write a plaintext secret",
|
||||
inputSchema: z.object({ secret: z.string() }),
|
||||
execute: vi.fn(),
|
||||
});
|
||||
const safeTool = tool({
|
||||
description: "Request secure user configuration",
|
||||
inputSchema: z.object({}),
|
||||
execute: vi.fn(),
|
||||
});
|
||||
const model = new MockLanguageModelV3({
|
||||
doGenerate: async () =>
|
||||
finalResult({
|
||||
status: "fail",
|
||||
summary: "No source values were supplied.",
|
||||
definitionId: null,
|
||||
previewUrl: null,
|
||||
iterations: 1,
|
||||
evidence: [
|
||||
{ tool: "customWidget_configurationRequestUser", outcome: "pending", detail: "User input needed." },
|
||||
],
|
||||
remainingIssues: ["Source configuration is required."],
|
||||
}),
|
||||
});
|
||||
|
||||
await createCustomWidgetMcpAgent({
|
||||
model,
|
||||
tools: {
|
||||
customWidget_sourceConfigure: forbiddenTool,
|
||||
customWidget_secretSet: forbiddenTool,
|
||||
customWidget_configurationRequestUser: safeTool,
|
||||
},
|
||||
}).generate({ prompt: "Create an authenticated widget" });
|
||||
|
||||
const exposedTools = model.doGenerateCalls[0]?.tools?.map(({ name }) => name);
|
||||
expect(exposedTools).toContain("customWidget_configurationRequestUser");
|
||||
expect(exposedTools).not.toContain("customWidget_sourceConfigure");
|
||||
expect(exposedTools).not.toContain("customWidget_secretSet");
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user