From 45dd20f4f736b757ad19501134901b8a26ebaecc Mon Sep 17 00:00:00 2001 From: ajnart Date: Thu, 30 Jul 2026 11:34:24 +0200 Subject: [PATCH] test: address custom widget review findings --- e2e/custom-widgets-custom-jsx-v2.spec.ts | 2 + .../scripts/generate-component-metadata.ts | 5 +- packages/custom-widgets/scripts/mcp-agent.ts | 15 +++-- .../custom-widgets/src/test/ai-prompt.spec.ts | 14 ++++ .../custom-widgets/src/test/mcp-agent.spec.ts | 66 +++++++++++++++++++ packages/custom-widgets/tsconfig.scripts.json | 10 +-- 6 files changed, 96 insertions(+), 16 deletions(-) diff --git a/e2e/custom-widgets-custom-jsx-v2.spec.ts b/e2e/custom-widgets-custom-jsx-v2.spec.ts index d3930033f..efc93ff6d 100644 --- a/e2e/custom-widgets-custom-jsx-v2.spec.ts +++ b/e2e/custom-widgets-custom-jsx-v2.spec.ts @@ -414,6 +414,7 @@ describe("Custom JSX v2 workbench", () => { await page.reload(); const migratedCard = page + .getByRole("main") .getByText(legacyWidgetName, { exact: true }) .locator("xpath=ancestor::*[contains(@class, 'mantine-Card-root')]"); await expect(migratedCard.getByText("Migration required", { exact: true })).not.toBeVisible(); @@ -441,6 +442,7 @@ describe("Custom JSX v2 workbench", () => { await page.reload(); const resurfacedLegacyCard = page + .getByRole("main") .getByText(legacyWidgetName, { exact: true }) .locator("xpath=ancestor::*[contains(@class, 'mantine-Card-root')]"); await expect(resurfacedLegacyCard.getByText("Migration required", { exact: true })).toBeVisible(); diff --git a/packages/custom-widgets/scripts/generate-component-metadata.ts b/packages/custom-widgets/scripts/generate-component-metadata.ts index f5f089d76..6109e463f 100644 --- a/packages/custom-widgets/scripts/generate-component-metadata.ts +++ b/packages/custom-widgets/scripts/generate-component-metadata.ts @@ -47,6 +47,7 @@ const program = ts.createProgram({ rootNames: [...config.fileNames, import.meta. const checker = program.getTypeChecker(); const source = program.getSourceFile(import.meta.filename); if (!source) throw new Error("Unable to inspect the component catalog generator"); +const sourceNode: ts.Node = source; const moduleSymbols = new Map(); for (const declaration of source.statements) { @@ -163,7 +164,7 @@ function inspectMantineComponent(component: CatalogSourceComponent): InspectedCo const rootName = component.name.split(".")[0]; let symbol = moduleSymbol && checker.getExportsOfModule(moduleSymbol).find((candidate) => candidate.name === rootName); - let location = symbol?.valueDeclaration ?? symbol?.declarations?.[0] ?? source; + let location: ts.Node = symbol?.valueDeclaration ?? symbol?.declarations?.[0] ?? sourceNode; let type = symbol && checker.getTypeOfSymbolAtLocation(symbol, location); for (const segment of component.name.split(".").slice(1)) { symbol = type?.getProperty(segment); @@ -311,7 +312,7 @@ function mergeGlobalProp(name: string, candidates: UninternedPropDescriptor[]): const fallback = createFallbackProp(name, "global"); const type = mostFrequent(candidates.map((candidate) => candidate.type)) ?? fallback.type; const descriptions = candidates.flatMap(({ description }) => (description ? [description] : [])); - const literalValues = uniqueLiterals(candidates.flatMap(({ literalValues }) => literalValues ?? [])); + const literalValues = uniqueLiterals(candidates.flatMap((candidate) => candidate.literalValues ?? [])); return { name, type, diff --git a/packages/custom-widgets/scripts/mcp-agent.ts b/packages/custom-widgets/scripts/mcp-agent.ts index d8cbe5b6f..218f16f4c 100644 --- a/packages/custom-widgets/scripts/mcp-agent.ts +++ b/packages/custom-widgets/scripts/mcp-agent.ts @@ -23,7 +23,7 @@ export const customWidgetMcpAgentResultSchema = z.strictObject({ status: z.enum(["pass", "needs-user-configuration", "fail"]), summary: z.string().min(1).max(1_000), definitionId: z.string().min(1).nullable(), - previewUrl: z.string().url().nullable(), + previewUrl: z.url().nullable(), iterations: z.number().int().min(1).max(20), evidence: z .array( @@ -168,7 +168,8 @@ export function getCustomWidgetMcpWorkflowIssues(args: { issues.push("customWidget_get must run before validating an edit"); } const previewCreate = firstSuccessfulIndex("customWidget_previewCreate"); - if (finalValidation >= 0 && previewCreate >= 0 && finalValidation > previewCreate) { + const finalPreviewCreate = lastSuccessfulIndex("customWidget_previewCreate"); + if (finalValidation >= 0 && finalPreviewCreate >= 0 && finalValidation > finalPreviewCreate) { issues.push("The final validation must run before customWidget_previewCreate"); } const previewChecks = [ @@ -194,10 +195,12 @@ export function getCustomWidgetMcpWorkflowIssues(args: { } } const finalJournal = lastSuccessfulIndex("customWidget_previewJournal"); - for (const tool of ["customWidget_previewQuery", "customWidget_previewAction"]) { - if (lastSuccessfulIndex(tool) > finalJournal) { - issues.push("customWidget_previewJournal must run after all preview requests"); - break; + if (finalJournal >= 0) { + for (const tool of ["customWidget_previewQuery", "customWidget_previewAction"]) { + if (lastSuccessfulIndex(tool) > finalJournal) { + issues.push("customWidget_previewJournal must run after all preview requests"); + break; + } } } } diff --git a/packages/custom-widgets/src/test/ai-prompt.spec.ts b/packages/custom-widgets/src/test/ai-prompt.spec.ts index ba7e5a429..a949ea592 100644 --- a/packages/custom-widgets/src/test/ai-prompt.spec.ts +++ b/packages/custom-widgets/src/test/ai-prompt.spec.ts @@ -69,4 +69,18 @@ describe("AI prompt", () => { expect(prompt).toContain('"X-Auth": "[REDACTED]"'); expect(prompt).toContain('"X-Feature-Key": "dashboard-layout"'); }); + + it("redacts credentials embedded directly in the free-form request", () => { + const prompt = buildCustomWidgetAiPrompt( + undefined, + null, + null, + "Use Authorization: Bearer sk-secret-123456 and token=ghp_abcdefghijklmnopqrstuvwxyz123456", + ); + + expect(prompt).not.toContain("sk-secret-123456"); + expect(prompt).not.toContain("ghp_abcdefghijklmnopqrstuvwxyz123456"); + expect(prompt).toContain("Authorization: Bearer [REDACTED]"); + expect(prompt).toContain("token=[REDACTED]"); + }); }); diff --git a/packages/custom-widgets/src/test/mcp-agent.spec.ts b/packages/custom-widgets/src/test/mcp-agent.spec.ts index e443af060..30c7d5257 100644 --- a/packages/custom-widgets/src/test/mcp-agent.spec.ts +++ b/packages/custom-widgets/src/test/mcp-agent.spec.ts @@ -248,6 +248,72 @@ describe("Custom Widget MCP agent", () => { ); }); + it("accepts a repaired preview followed by final validation and a fresh preview", () => { + const toolsUsed = [ + "customWidget_getAuthoringPrompt", + "customWidget_getSkill", + "customWidget_schema", + "customWidget_validate", + "customWidget_previewCreate", + "customWidget_validate", + "customWidget_previewCreate", + "customWidget_previewQuery", + "customWidget_previewJournal", + ]; + + expect( + getCustomWidgetMcpWorkflowIssues({ + output: { + status: "pass", + summary: "The repaired widget passed a fresh preview.", + definitionId: null, + previewUrl: "http://localhost:3000/manage/custom-widgets/preview/session-repaired", + iterations: 2, + evidence: [...new Set(toolsUsed)].map((name) => ({ + tool: name, + outcome: name === "customWidget_validate" ? ("repaired" as const) : ("passed" as const), + detail: `${name} completed.`, + })), + remainingIssues: [], + }, + toolExecutions: successfulExecutions(toolsUsed), + persist: false, + }), + ).toEqual([]); + }); + + it("does not report journal ordering when the journal was never run", () => { + const toolsUsed = [ + "customWidget_getAuthoringPrompt", + "customWidget_getSkill", + "customWidget_schema", + "customWidget_validate", + "customWidget_previewCreate", + "customWidget_previewQuery", + ]; + + const issues = getCustomWidgetMcpWorkflowIssues({ + output: { + status: "pass", + summary: "The agent incorrectly skipped the journal.", + definitionId: null, + previewUrl: "http://localhost:3000/manage/custom-widgets/preview/session-no-journal", + iterations: 1, + evidence: toolsUsed.map((name) => ({ + tool: name, + outcome: "passed" as const, + detail: `${name} completed.`, + })), + remainingIssues: [], + }, + toolExecutions: successfulExecutions(toolsUsed), + persist: false, + }); + + expect(issues).toContain("Required tool was not used: customWidget_previewJournal"); + expect(issues).not.toContain("customWidget_previewJournal must run after all preview requests"); + }); + it("allows secure preview configuration to pause before authenticated requests can run", () => { const toolsUsed = [ "customWidget_getAuthoringPrompt", diff --git a/packages/custom-widgets/tsconfig.scripts.json b/packages/custom-widgets/tsconfig.scripts.json index 6b6feb8e9..eadc5a4c3 100644 --- a/packages/custom-widgets/tsconfig.scripts.json +++ b/packages/custom-widgets/tsconfig.scripts.json @@ -3,12 +3,6 @@ "compilerOptions": { "tsBuildInfoFile": "node_modules/.cache/scripts-tsbuildinfo.json" }, - "include": [ - "scripts/ai-evaluation-cases.ts", - "scripts/ai-evaluation.ts", - "scripts/evaluate-ai-authoring.ts", - "scripts/evaluate-mcp-agent.ts", - "scripts/mcp-agent.ts" - ], - "exclude": ["node_modules"] + "include": ["scripts/**/*.ts"], + "exclude": ["node_modules", "scripts/verify-reference-apis.ts"] }