test: address custom widget review findings
This commit is contained in:
1 parent
6eff1da08f
commit
45dd20f4f7
6 files changed
+96
-16
No files matched your search
@@ -414,6 +414,7 @@ describe("Custom JSX v2 workbench", () => {
|
|||||||
|
|
||||||
await page.reload();
|
await page.reload();
|
||||||
const migratedCard = page
|
const migratedCard = page
|
||||||
|
.getByRole("main")
|
||||||
.getByText(legacyWidgetName, { exact: true })
|
.getByText(legacyWidgetName, { exact: true })
|
||||||
.locator("xpath=ancestor::*[contains(@class, 'mantine-Card-root')]");
|
.locator("xpath=ancestor::*[contains(@class, 'mantine-Card-root')]");
|
||||||
await expect(migratedCard.getByText("Migration required", { exact: true })).not.toBeVisible();
|
await expect(migratedCard.getByText("Migration required", { exact: true })).not.toBeVisible();
|
||||||
@@ -441,6 +442,7 @@ describe("Custom JSX v2 workbench", () => {
|
|||||||
|
|
||||||
await page.reload();
|
await page.reload();
|
||||||
const resurfacedLegacyCard = page
|
const resurfacedLegacyCard = page
|
||||||
|
.getByRole("main")
|
||||||
.getByText(legacyWidgetName, { exact: true })
|
.getByText(legacyWidgetName, { exact: true })
|
||||||
.locator("xpath=ancestor::*[contains(@class, 'mantine-Card-root')]");
|
.locator("xpath=ancestor::*[contains(@class, 'mantine-Card-root')]");
|
||||||
await expect(resurfacedLegacyCard.getByText("Migration required", { exact: true })).toBeVisible();
|
await expect(resurfacedLegacyCard.getByText("Migration required", { exact: true })).toBeVisible();
|
||||||
|
|||||||
@@ -47,6 +47,7 @@ const program = ts.createProgram({ rootNames: [...config.fileNames, import.meta.
|
|||||||
const checker = program.getTypeChecker();
|
const checker = program.getTypeChecker();
|
||||||
const source = program.getSourceFile(import.meta.filename);
|
const source = program.getSourceFile(import.meta.filename);
|
||||||
if (!source) throw new Error("Unable to inspect the component catalog generator");
|
if (!source) throw new Error("Unable to inspect the component catalog generator");
|
||||||
|
const sourceNode: ts.Node = source;
|
||||||
|
|
||||||
const moduleSymbols = new Map<string, ts.Symbol>();
|
const moduleSymbols = new Map<string, ts.Symbol>();
|
||||||
for (const declaration of source.statements) {
|
for (const declaration of source.statements) {
|
||||||
@@ -163,7 +164,7 @@ function inspectMantineComponent(component: CatalogSourceComponent): InspectedCo
|
|||||||
const rootName = component.name.split(".")[0];
|
const rootName = component.name.split(".")[0];
|
||||||
let symbol =
|
let symbol =
|
||||||
moduleSymbol && checker.getExportsOfModule(moduleSymbol).find((candidate) => candidate.name === rootName);
|
moduleSymbol && checker.getExportsOfModule(moduleSymbol).find((candidate) => candidate.name === rootName);
|
||||||
let location = symbol?.valueDeclaration ?? symbol?.declarations?.[0] ?? source;
|
let location: ts.Node = symbol?.valueDeclaration ?? symbol?.declarations?.[0] ?? sourceNode;
|
||||||
let type = symbol && checker.getTypeOfSymbolAtLocation(symbol, location);
|
let type = symbol && checker.getTypeOfSymbolAtLocation(symbol, location);
|
||||||
for (const segment of component.name.split(".").slice(1)) {
|
for (const segment of component.name.split(".").slice(1)) {
|
||||||
symbol = type?.getProperty(segment);
|
symbol = type?.getProperty(segment);
|
||||||
@@ -311,7 +312,7 @@ function mergeGlobalProp(name: string, candidates: UninternedPropDescriptor[]):
|
|||||||
const fallback = createFallbackProp(name, "global");
|
const fallback = createFallbackProp(name, "global");
|
||||||
const type = mostFrequent(candidates.map((candidate) => candidate.type)) ?? fallback.type;
|
const type = mostFrequent(candidates.map((candidate) => candidate.type)) ?? fallback.type;
|
||||||
const descriptions = candidates.flatMap(({ description }) => (description ? [description] : []));
|
const descriptions = candidates.flatMap(({ description }) => (description ? [description] : []));
|
||||||
const literalValues = uniqueLiterals(candidates.flatMap(({ literalValues }) => literalValues ?? []));
|
const literalValues = uniqueLiterals(candidates.flatMap((candidate) => candidate.literalValues ?? []));
|
||||||
return {
|
return {
|
||||||
name,
|
name,
|
||||||
type,
|
type,
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ export const customWidgetMcpAgentResultSchema = z.strictObject({
|
|||||||
status: z.enum(["pass", "needs-user-configuration", "fail"]),
|
status: z.enum(["pass", "needs-user-configuration", "fail"]),
|
||||||
summary: z.string().min(1).max(1_000),
|
summary: z.string().min(1).max(1_000),
|
||||||
definitionId: z.string().min(1).nullable(),
|
definitionId: z.string().min(1).nullable(),
|
||||||
previewUrl: z.string().url().nullable(),
|
previewUrl: z.url().nullable(),
|
||||||
iterations: z.number().int().min(1).max(20),
|
iterations: z.number().int().min(1).max(20),
|
||||||
evidence: z
|
evidence: z
|
||||||
.array(
|
.array(
|
||||||
@@ -168,7 +168,8 @@ export function getCustomWidgetMcpWorkflowIssues(args: {
|
|||||||
issues.push("customWidget_get must run before validating an edit");
|
issues.push("customWidget_get must run before validating an edit");
|
||||||
}
|
}
|
||||||
const previewCreate = firstSuccessfulIndex("customWidget_previewCreate");
|
const previewCreate = firstSuccessfulIndex("customWidget_previewCreate");
|
||||||
if (finalValidation >= 0 && previewCreate >= 0 && finalValidation > previewCreate) {
|
const finalPreviewCreate = lastSuccessfulIndex("customWidget_previewCreate");
|
||||||
|
if (finalValidation >= 0 && finalPreviewCreate >= 0 && finalValidation > finalPreviewCreate) {
|
||||||
issues.push("The final validation must run before customWidget_previewCreate");
|
issues.push("The final validation must run before customWidget_previewCreate");
|
||||||
}
|
}
|
||||||
const previewChecks = [
|
const previewChecks = [
|
||||||
@@ -194,10 +195,12 @@ export function getCustomWidgetMcpWorkflowIssues(args: {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
const finalJournal = lastSuccessfulIndex("customWidget_previewJournal");
|
const finalJournal = lastSuccessfulIndex("customWidget_previewJournal");
|
||||||
for (const tool of ["customWidget_previewQuery", "customWidget_previewAction"]) {
|
if (finalJournal >= 0) {
|
||||||
if (lastSuccessfulIndex(tool) > finalJournal) {
|
for (const tool of ["customWidget_previewQuery", "customWidget_previewAction"]) {
|
||||||
issues.push("customWidget_previewJournal must run after all preview requests");
|
if (lastSuccessfulIndex(tool) > finalJournal) {
|
||||||
break;
|
issues.push("customWidget_previewJournal must run after all preview requests");
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -69,4 +69,18 @@ describe("AI prompt", () => {
|
|||||||
expect(prompt).toContain('"X-Auth": "[REDACTED]"');
|
expect(prompt).toContain('"X-Auth": "[REDACTED]"');
|
||||||
expect(prompt).toContain('"X-Feature-Key": "dashboard-layout"');
|
expect(prompt).toContain('"X-Feature-Key": "dashboard-layout"');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("redacts credentials embedded directly in the free-form request", () => {
|
||||||
|
const prompt = buildCustomWidgetAiPrompt(
|
||||||
|
undefined,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
"Use Authorization: Bearer sk-secret-123456 and token=ghp_abcdefghijklmnopqrstuvwxyz123456",
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(prompt).not.toContain("sk-secret-123456");
|
||||||
|
expect(prompt).not.toContain("ghp_abcdefghijklmnopqrstuvwxyz123456");
|
||||||
|
expect(prompt).toContain("Authorization: Bearer [REDACTED]");
|
||||||
|
expect(prompt).toContain("token=[REDACTED]");
|
||||||
|
});
|
||||||
});
|
});
|
||||||
@@ -248,6 +248,72 @@ describe("Custom Widget MCP agent", () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("accepts a repaired preview followed by final validation and a fresh preview", () => {
|
||||||
|
const toolsUsed = [
|
||||||
|
"customWidget_getAuthoringPrompt",
|
||||||
|
"customWidget_getSkill",
|
||||||
|
"customWidget_schema",
|
||||||
|
"customWidget_validate",
|
||||||
|
"customWidget_previewCreate",
|
||||||
|
"customWidget_validate",
|
||||||
|
"customWidget_previewCreate",
|
||||||
|
"customWidget_previewQuery",
|
||||||
|
"customWidget_previewJournal",
|
||||||
|
];
|
||||||
|
|
||||||
|
expect(
|
||||||
|
getCustomWidgetMcpWorkflowIssues({
|
||||||
|
output: {
|
||||||
|
status: "pass",
|
||||||
|
summary: "The repaired widget passed a fresh preview.",
|
||||||
|
definitionId: null,
|
||||||
|
previewUrl: "http://localhost:3000/manage/custom-widgets/preview/session-repaired",
|
||||||
|
iterations: 2,
|
||||||
|
evidence: [...new Set(toolsUsed)].map((name) => ({
|
||||||
|
tool: name,
|
||||||
|
outcome: name === "customWidget_validate" ? ("repaired" as const) : ("passed" as const),
|
||||||
|
detail: `${name} completed.`,
|
||||||
|
})),
|
||||||
|
remainingIssues: [],
|
||||||
|
},
|
||||||
|
toolExecutions: successfulExecutions(toolsUsed),
|
||||||
|
persist: false,
|
||||||
|
}),
|
||||||
|
).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not report journal ordering when the journal was never run", () => {
|
||||||
|
const toolsUsed = [
|
||||||
|
"customWidget_getAuthoringPrompt",
|
||||||
|
"customWidget_getSkill",
|
||||||
|
"customWidget_schema",
|
||||||
|
"customWidget_validate",
|
||||||
|
"customWidget_previewCreate",
|
||||||
|
"customWidget_previewQuery",
|
||||||
|
];
|
||||||
|
|
||||||
|
const issues = getCustomWidgetMcpWorkflowIssues({
|
||||||
|
output: {
|
||||||
|
status: "pass",
|
||||||
|
summary: "The agent incorrectly skipped the journal.",
|
||||||
|
definitionId: null,
|
||||||
|
previewUrl: "http://localhost:3000/manage/custom-widgets/preview/session-no-journal",
|
||||||
|
iterations: 1,
|
||||||
|
evidence: toolsUsed.map((name) => ({
|
||||||
|
tool: name,
|
||||||
|
outcome: "passed" as const,
|
||||||
|
detail: `${name} completed.`,
|
||||||
|
})),
|
||||||
|
remainingIssues: [],
|
||||||
|
},
|
||||||
|
toolExecutions: successfulExecutions(toolsUsed),
|
||||||
|
persist: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(issues).toContain("Required tool was not used: customWidget_previewJournal");
|
||||||
|
expect(issues).not.toContain("customWidget_previewJournal must run after all preview requests");
|
||||||
|
});
|
||||||
|
|
||||||
it("allows secure preview configuration to pause before authenticated requests can run", () => {
|
it("allows secure preview configuration to pause before authenticated requests can run", () => {
|
||||||
const toolsUsed = [
|
const toolsUsed = [
|
||||||
"customWidget_getAuthoringPrompt",
|
"customWidget_getAuthoringPrompt",
|
||||||
|
|||||||
@@ -3,12 +3,6 @@
|
|||||||
"compilerOptions": {
|
"compilerOptions": {
|
||||||
"tsBuildInfoFile": "node_modules/.cache/scripts-tsbuildinfo.json"
|
"tsBuildInfoFile": "node_modules/.cache/scripts-tsbuildinfo.json"
|
||||||
},
|
},
|
||||||
"include": [
|
"include": ["scripts/**/*.ts"],
|
||||||
"scripts/ai-evaluation-cases.ts",
|
"exclude": ["node_modules", "scripts/verify-reference-apis.ts"]
|
||||||
"scripts/ai-evaluation.ts",
|
|
||||||
"scripts/evaluate-ai-authoring.ts",
|
|
||||||
"scripts/evaluate-mcp-agent.ts",
|
|
||||||
"scripts/mcp-agent.ts"
|
|
||||||
],
|
|
||||||
"exclude": ["node_modules"]
|
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user