diff --git a/packages/custom-widgets/scripts/ai-evaluation.ts b/packages/custom-widgets/scripts/ai-evaluation.ts index 02caea39f..7e0a67773 100644 --- a/packages/custom-widgets/scripts/ai-evaluation.ts +++ b/packages/custom-widgets/scripts/ai-evaluation.ts @@ -212,15 +212,21 @@ export function getScenarioAcceptanceIssues( const requestIdsByLabel = new Map([...candidateMatches].map(([label, [requestId]]) => [label, requestId] as const)); for (const rule of testCase.acceptance.requestRules) { const match = candidateMatches.get(rule.label); - const invalidatedRequestId = rule.invalidatesRequest ? requestIdsByLabel.get(rule.invalidatesRequest) : undefined; - if ( - !match || - (rule.invalidatesRequest && (!invalidatedRequestId || !match[1].invalidates?.includes(invalidatedRequestId))) - ) { + if (!match) { issues.push({ path: ["requests"], message: `Missing grounded ${rule.label} request (${rule.pathIncludes}).`, }); + continue; + } + if (rule.invalidatesRequest) { + const invalidatedRequestId = requestIdsByLabel.get(rule.invalidatesRequest); + if (!invalidatedRequestId || !match[1].invalidates?.includes(invalidatedRequestId)) { + issues.push({ + path: ["requests"], + message: `The ${rule.label} request must invalidate the ${rule.invalidatesRequest} request.`, + }); + } } } diff --git a/packages/custom-widgets/src/test/ai-evaluation.spec.ts b/packages/custom-widgets/src/test/ai-evaluation.spec.ts index 2bc720c75..0fac16112 100644 --- a/packages/custom-widgets/src/test/ai-evaluation.spec.ts +++ b/packages/custom-widgets/src/test/ai-evaluation.spec.ts @@ -141,7 +141,7 @@ describe("AI authoring evaluation", () => { }, }); expect(getScenarioAcceptanceIssues(testCase, wrongInvalidation).map(({ message }) => message)).toContain( - "Missing grounded start action request (/start).", + "The start action request must invalidate the container list request.", ); });