test: address custom widget review findings

This commit is contained in:
ajnart committed 2026-07-30 11:34:24 +02:00
1 parent 6eff1da08f
commit 45dd20f4f7
6 files changed
+96 -16

No files matched your search

+2
View File
@@ -414,6 +414,7 @@ describe("Custom JSX v2 workbench", () => {
await page.reload(); await page.reload();
const migratedCard = page const migratedCard = page
.getByRole("main")
.getByText(legacyWidgetName, { exact: true }) .getByText(legacyWidgetName, { exact: true })
.locator("xpath=ancestor::*[contains(@class, 'mantine-Card-root')]"); .locator("xpath=ancestor::*[contains(@class, 'mantine-Card-root')]");
await expect(migratedCard.getByText("Migration required", { exact: true })).not.toBeVisible(); await expect(migratedCard.getByText("Migration required", { exact: true })).not.toBeVisible();
@@ -441,6 +442,7 @@ describe("Custom JSX v2 workbench", () => {
await page.reload(); await page.reload();
const resurfacedLegacyCard = page const resurfacedLegacyCard = page
.getByRole("main")
.getByText(legacyWidgetName, { exact: true }) .getByText(legacyWidgetName, { exact: true })
.locator("xpath=ancestor::*[contains(@class, 'mantine-Card-root')]"); .locator("xpath=ancestor::*[contains(@class, 'mantine-Card-root')]");
await expect(resurfacedLegacyCard.getByText("Migration required", { exact: true })).toBeVisible(); await expect(resurfacedLegacyCard.getByText("Migration required", { exact: true })).toBeVisible();
@@ -47,6 +47,7 @@ const program = ts.createProgram({ rootNames: [...config.fileNames, import.meta.
const checker = program.getTypeChecker(); const checker = program.getTypeChecker();
const source = program.getSourceFile(import.meta.filename); const source = program.getSourceFile(import.meta.filename);
if (!source) throw new Error("Unable to inspect the component catalog generator"); if (!source) throw new Error("Unable to inspect the component catalog generator");
const sourceNode: ts.Node = source;
const moduleSymbols = new Map<string, ts.Symbol>(); const moduleSymbols = new Map<string, ts.Symbol>();
for (const declaration of source.statements) { for (const declaration of source.statements) {
@@ -163,7 +164,7 @@ function inspectMantineComponent(component: CatalogSourceComponent): InspectedCo
const rootName = component.name.split(".")[0]; const rootName = component.name.split(".")[0];
let symbol = let symbol =
moduleSymbol && checker.getExportsOfModule(moduleSymbol).find((candidate) => candidate.name === rootName); moduleSymbol && checker.getExportsOfModule(moduleSymbol).find((candidate) => candidate.name === rootName);
let location = symbol?.valueDeclaration ?? symbol?.declarations?.[0] ?? source; let location: ts.Node = symbol?.valueDeclaration ?? symbol?.declarations?.[0] ?? sourceNode;
let type = symbol && checker.getTypeOfSymbolAtLocation(symbol, location); let type = symbol && checker.getTypeOfSymbolAtLocation(symbol, location);
for (const segment of component.name.split(".").slice(1)) { for (const segment of component.name.split(".").slice(1)) {
symbol = type?.getProperty(segment); symbol = type?.getProperty(segment);
@@ -311,7 +312,7 @@ function mergeGlobalProp(name: string, candidates: UninternedPropDescriptor[]):
const fallback = createFallbackProp(name, "global"); const fallback = createFallbackProp(name, "global");
const type = mostFrequent(candidates.map((candidate) => candidate.type)) ?? fallback.type; const type = mostFrequent(candidates.map((candidate) => candidate.type)) ?? fallback.type;
const descriptions = candidates.flatMap(({ description }) => (description ? [description] : [])); const descriptions = candidates.flatMap(({ description }) => (description ? [description] : []));
const literalValues = uniqueLiterals(candidates.flatMap(({ literalValues }) => literalValues ?? [])); const literalValues = uniqueLiterals(candidates.flatMap((candidate) => candidate.literalValues ?? []));
return { return {
name, name,
type, type,
+9 -6
View File
@@ -23,7 +23,7 @@ export const customWidgetMcpAgentResultSchema = z.strictObject({
status: z.enum(["pass", "needs-user-configuration", "fail"]), status: z.enum(["pass", "needs-user-configuration", "fail"]),
summary: z.string().min(1).max(1_000), summary: z.string().min(1).max(1_000),
definitionId: z.string().min(1).nullable(), definitionId: z.string().min(1).nullable(),
previewUrl: z.string().url().nullable(), previewUrl: z.url().nullable(),
iterations: z.number().int().min(1).max(20), iterations: z.number().int().min(1).max(20),
evidence: z evidence: z
.array( .array(
@@ -168,7 +168,8 @@ export function getCustomWidgetMcpWorkflowIssues(args: {
issues.push("customWidget_get must run before validating an edit"); issues.push("customWidget_get must run before validating an edit");
} }
const previewCreate = firstSuccessfulIndex("customWidget_previewCreate"); const previewCreate = firstSuccessfulIndex("customWidget_previewCreate");
if (finalValidation >= 0 && previewCreate >= 0 && finalValidation > previewCreate) { const finalPreviewCreate = lastSuccessfulIndex("customWidget_previewCreate");
if (finalValidation >= 0 && finalPreviewCreate >= 0 && finalValidation > finalPreviewCreate) {
issues.push("The final validation must run before customWidget_previewCreate"); issues.push("The final validation must run before customWidget_previewCreate");
} }
const previewChecks = [ const previewChecks = [
@@ -194,10 +195,12 @@ export function getCustomWidgetMcpWorkflowIssues(args: {
} }
} }
const finalJournal = lastSuccessfulIndex("customWidget_previewJournal"); const finalJournal = lastSuccessfulIndex("customWidget_previewJournal");
for (const tool of ["customWidget_previewQuery", "customWidget_previewAction"]) { if (finalJournal >= 0) {
if (lastSuccessfulIndex(tool) > finalJournal) { for (const tool of ["customWidget_previewQuery", "customWidget_previewAction"]) {
issues.push("customWidget_previewJournal must run after all preview requests"); if (lastSuccessfulIndex(tool) > finalJournal) {
break; issues.push("customWidget_previewJournal must run after all preview requests");
break;
}
} }
} }
} }
@@ -69,4 +69,18 @@ describe("AI prompt", () => {
expect(prompt).toContain('"X-Auth": "[REDACTED]"'); expect(prompt).toContain('"X-Auth": "[REDACTED]"');
expect(prompt).toContain('"X-Feature-Key": "dashboard-layout"'); expect(prompt).toContain('"X-Feature-Key": "dashboard-layout"');
}); });
it("redacts credentials embedded directly in the free-form request", () => {
const prompt = buildCustomWidgetAiPrompt(
undefined,
null,
null,
"Use Authorization: Bearer sk-secret-123456 and token=ghp_abcdefghijklmnopqrstuvwxyz123456",
);
expect(prompt).not.toContain("sk-secret-123456");
expect(prompt).not.toContain("ghp_abcdefghijklmnopqrstuvwxyz123456");
expect(prompt).toContain("Authorization: Bearer [REDACTED]");
expect(prompt).toContain("token=[REDACTED]");
});
}); });
@@ -248,6 +248,72 @@ describe("Custom Widget MCP agent", () => {
); );
}); });
it("accepts a repaired preview followed by final validation and a fresh preview", () => {
const toolsUsed = [
"customWidget_getAuthoringPrompt",
"customWidget_getSkill",
"customWidget_schema",
"customWidget_validate",
"customWidget_previewCreate",
"customWidget_validate",
"customWidget_previewCreate",
"customWidget_previewQuery",
"customWidget_previewJournal",
];
expect(
getCustomWidgetMcpWorkflowIssues({
output: {
status: "pass",
summary: "The repaired widget passed a fresh preview.",
definitionId: null,
previewUrl: "http://localhost:3000/manage/custom-widgets/preview/session-repaired",
iterations: 2,
evidence: [...new Set(toolsUsed)].map((name) => ({
tool: name,
outcome: name === "customWidget_validate" ? ("repaired" as const) : ("passed" as const),
detail: `${name} completed.`,
})),
remainingIssues: [],
},
toolExecutions: successfulExecutions(toolsUsed),
persist: false,
}),
).toEqual([]);
});
it("does not report journal ordering when the journal was never run", () => {
const toolsUsed = [
"customWidget_getAuthoringPrompt",
"customWidget_getSkill",
"customWidget_schema",
"customWidget_validate",
"customWidget_previewCreate",
"customWidget_previewQuery",
];
const issues = getCustomWidgetMcpWorkflowIssues({
output: {
status: "pass",
summary: "The agent incorrectly skipped the journal.",
definitionId: null,
previewUrl: "http://localhost:3000/manage/custom-widgets/preview/session-no-journal",
iterations: 1,
evidence: toolsUsed.map((name) => ({
tool: name,
outcome: "passed" as const,
detail: `${name} completed.`,
})),
remainingIssues: [],
},
toolExecutions: successfulExecutions(toolsUsed),
persist: false,
});
expect(issues).toContain("Required tool was not used: customWidget_previewJournal");
expect(issues).not.toContain("customWidget_previewJournal must run after all preview requests");
});
it("allows secure preview configuration to pause before authenticated requests can run", () => { it("allows secure preview configuration to pause before authenticated requests can run", () => {
const toolsUsed = [ const toolsUsed = [
"customWidget_getAuthoringPrompt", "customWidget_getAuthoringPrompt",
@@ -3,12 +3,6 @@
"compilerOptions": { "compilerOptions": {
"tsBuildInfoFile": "node_modules/.cache/scripts-tsbuildinfo.json" "tsBuildInfoFile": "node_modules/.cache/scripts-tsbuildinfo.json"
}, },
"include": [ "include": ["scripts/**/*.ts"],
"scripts/ai-evaluation-cases.ts", "exclude": ["node_modules", "scripts/verify-reference-apis.ts"]
"scripts/ai-evaluation.ts",
"scripts/evaluate-ai-authoring.ts",
"scripts/evaluate-mcp-agent.ts",
"scripts/mcp-agent.ts"
],
"exclude": ["node_modules"]
} }