test: strengthen custom widget AI evaluation
This commit is contained in:
1 parent
45dd20f4f7
commit
5cd40b5c53
7 files changed
+246
-47
No files matched your search
@@ -13,6 +13,7 @@ import {
|
||||
} from "../../scripts/ai-evaluation";
|
||||
import type { CustomWidgetJudgeResult } from "../../scripts/ai-evaluation";
|
||||
import { BUNDLED_CUSTOM_WIDGETS, CUSTOM_WIDGET_STARTER, customWidgetDefinitionSchema } from "../core";
|
||||
import { PORTAINER_REFERENCE_WIDGET } from "./fixtures/reference-widgets";
|
||||
|
||||
const categoryNames = [
|
||||
"schemaAndBindings",
|
||||
@@ -76,6 +77,8 @@ describe("AI authoring evaluation", () => {
|
||||
]);
|
||||
expect(prompt).toContain("requests.list.path: Required");
|
||||
expect(prompt).toContain("bad response");
|
||||
expect(prompt).toContain("visual, and UX problem");
|
||||
expect(prompt).toContain("redesign weak areas");
|
||||
});
|
||||
|
||||
it("requires grounded scenario capabilities before model judging", () => {
|
||||
@@ -97,6 +100,39 @@ describe("AI authoring evaluation", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("requires the exact option bindings and invalidation target for scenario actions", () => {
|
||||
const testCase = CUSTOM_WIDGET_AI_EVALUATION_CASES.find(({ id }) => id === "portainer-containers");
|
||||
if (!testCase) throw new Error("Expected the Portainer evaluation case");
|
||||
const portainer = customWidgetDefinitionSchema.parse(PORTAINER_REFERENCE_WIDGET);
|
||||
expect(getScenarioAcceptanceIssues(testCase, portainer)).toEqual([]);
|
||||
|
||||
const wrongBinding = customWidgetDefinitionSchema.parse({
|
||||
...portainer,
|
||||
requests: {
|
||||
...portainer.requests,
|
||||
start: {
|
||||
...portainer.requests.start,
|
||||
path: portainer.requests.start?.path.replace("{option:endpointId}", "{option:showAll}"),
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(getScenarioAcceptanceIssues(testCase, wrongBinding).map(({ message }) => message)).toContain(
|
||||
"Missing grounded start action request (/start).",
|
||||
);
|
||||
|
||||
const wrongInvalidation = customWidgetDefinitionSchema.parse({
|
||||
...portainer,
|
||||
requests: {
|
||||
...portainer.requests,
|
||||
health: { path: "/api/status" },
|
||||
start: { ...portainer.requests.start, invalidates: ["health"] },
|
||||
},
|
||||
});
|
||||
expect(getScenarioAcceptanceIssues(testCase, wrongInvalidation).map(({ message }) => message)).toContain(
|
||||
"Missing grounded start action request (/start).",
|
||||
);
|
||||
});
|
||||
|
||||
it("uses the requested DeepSeek models and strict structured judge output", () => {
|
||||
expect(DEFAULT_GENERATOR_MODEL).toBe("deepseek/deepseek-v4-pro");
|
||||
expect(DEFAULT_JUDGE_MODEL).toBe("deepseek/deepseek-v4-flash");
|
||||
|
||||
@@ -339,12 +339,143 @@ describe("Custom Widget MCP agent", () => {
|
||||
})),
|
||||
remainingIssues: [],
|
||||
},
|
||||
toolExecutions: successfulExecutions(toolsUsed),
|
||||
toolExecutions: successfulExecutions(toolsUsed).map((entry) =>
|
||||
entry.tool === "customWidget_configurationRequestUser"
|
||||
? { ...entry, configurationStatus: "pending" as const }
|
||||
: entry,
|
||||
),
|
||||
persist: false,
|
||||
}),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("resumes preview checks after secure user configuration reports completion", async () => {
|
||||
let configurationCalls = 0;
|
||||
const sequence = [
|
||||
"customWidget_getAuthoringPrompt",
|
||||
"customWidget_getSkill",
|
||||
"customWidget_schema",
|
||||
"customWidget_validate",
|
||||
"customWidget_previewCreate",
|
||||
"customWidget_configurationRequestUser",
|
||||
"customWidget_configurationRequestUser",
|
||||
"customWidget_previewQuery",
|
||||
"customWidget_previewJournal",
|
||||
];
|
||||
const output = {
|
||||
status: "pass" as const,
|
||||
summary: "The user completed secure source configuration and the authenticated preview passed.",
|
||||
definitionId: null,
|
||||
previewUrl: "http://localhost:3000/manage/custom-widgets/preview/session-configured",
|
||||
iterations: 2,
|
||||
evidence: [...new Set(sequence)].map((name) => ({
|
||||
tool: name,
|
||||
outcome: "passed" as const,
|
||||
detail: `${name} completed.`,
|
||||
})),
|
||||
remainingIssues: [],
|
||||
};
|
||||
const responses = [
|
||||
...sequence.map((toolName, index) => {
|
||||
if (toolName === "customWidget_configurationRequestUser") {
|
||||
return toolCall(
|
||||
toolName,
|
||||
index,
|
||||
configurationCalls++ === 0
|
||||
? { previewSessionId: "preview-1", sourceId: "default" }
|
||||
: { requestId: "configuration-1" },
|
||||
);
|
||||
}
|
||||
return toolCall(toolName, index, toolName === "customWidget_previewQuery" ? { requestId: "query-1" } : {});
|
||||
}),
|
||||
finalResult(output),
|
||||
];
|
||||
let responseIndex = 0;
|
||||
let configurationResults = 0;
|
||||
const model = new MockLanguageModelV3({
|
||||
doGenerate: async () => responses[responseIndex++] ?? finalResult(output),
|
||||
});
|
||||
const tools = Object.fromEntries(
|
||||
sequence.map((name) => [
|
||||
name,
|
||||
tool({
|
||||
description: name,
|
||||
inputSchema: z.object({
|
||||
requestId: z.string().optional(),
|
||||
previewSessionId: z.string().optional(),
|
||||
sourceId: z.string().optional(),
|
||||
}),
|
||||
execute: () => {
|
||||
if (name === "customWidget_validate") return mcpOutput({ valid: true });
|
||||
if (name === "customWidget_previewCreate") return mcpOutput({ success: true });
|
||||
if (name === "customWidget_previewQuery") return mcpOutput({ ok: true });
|
||||
if (name === "customWidget_configurationRequestUser") {
|
||||
configurationResults += 1;
|
||||
return mcpOutput({
|
||||
requestId: "configuration-1",
|
||||
status: configurationResults === 1 ? "pending" : "completed",
|
||||
});
|
||||
}
|
||||
return mcpOutput({});
|
||||
},
|
||||
}),
|
||||
]),
|
||||
);
|
||||
|
||||
const result = await createCustomWidgetMcpAgent({ model, tools }).generate({ prompt: "Build a Jellyfin widget" });
|
||||
const toolExecutions = getCustomWidgetMcpToolExecutions(result.steps);
|
||||
expect(
|
||||
toolExecutions
|
||||
.filter(({ tool: toolName }) => toolName === "customWidget_configurationRequestUser")
|
||||
.map(({ configurationStatus }) => configurationStatus),
|
||||
).toEqual(["pending", "completed"]);
|
||||
expect(
|
||||
getCustomWidgetMcpWorkflowIssues({
|
||||
output: result.output,
|
||||
toolExecutions,
|
||||
persist: false,
|
||||
}),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("rejects success while secure user configuration is still pending", () => {
|
||||
const toolsUsed = [
|
||||
"customWidget_getAuthoringPrompt",
|
||||
"customWidget_getSkill",
|
||||
"customWidget_schema",
|
||||
"customWidget_validate",
|
||||
"customWidget_previewCreate",
|
||||
"customWidget_configurationRequestUser",
|
||||
"customWidget_previewQuery",
|
||||
"customWidget_previewJournal",
|
||||
];
|
||||
const toolExecutions = successfulExecutions(toolsUsed).map((entry) =>
|
||||
entry.tool === "customWidget_configurationRequestUser"
|
||||
? { ...entry, configurationStatus: "pending" as const }
|
||||
: entry,
|
||||
);
|
||||
|
||||
expect(
|
||||
getCustomWidgetMcpWorkflowIssues({
|
||||
output: {
|
||||
status: "pass",
|
||||
summary: "The agent claimed the authenticated preview passed before configuration completed.",
|
||||
definitionId: null,
|
||||
previewUrl: "http://localhost:3000/manage/custom-widgets/preview/session-pending",
|
||||
iterations: 1,
|
||||
evidence: toolsUsed.map((name) => ({
|
||||
tool: name,
|
||||
outcome: "passed" as const,
|
||||
detail: `${name} completed.`,
|
||||
})),
|
||||
remainingIssues: [],
|
||||
},
|
||||
toolExecutions,
|
||||
persist: false,
|
||||
}),
|
||||
).toContain("A passing result requires the user configuration request to report completed");
|
||||
});
|
||||
|
||||
it("never exposes a credential-writing tool to the model", async () => {
|
||||
const forbiddenTool = tool({
|
||||
description: "Write a plaintext secret",
|
||||
|
||||
Reference in new issue
Block a user