test custom widget assistant authoring

packages/custom-widgets/scripts/evaluate-ai-authoring.ts

@@ -2,6 +2,8 @@ import { mkdir, writeFile } from "node:fs/promises";
 import path from "node:path";
 
 import { CUSTOM_WIDGET_AI_EVALUATION_CASES } from "./ai-evaluation-cases";
+import { evaluateCustomWidgetAssistantCase } from "./ai-assistant-evaluation";
+import type { CustomWidgetAssistantEvaluationResult } from "./ai-assistant-evaluation";
 import { DEFAULT_GENERATOR_MODEL, DEFAULT_JUDGE_MODEL, evaluateCustomWidgetCase } from "./ai-evaluation";
 import type { AiEvaluationResult } from "./ai-evaluation";
 
@@ -9,11 +11,19 @@ const apiKey = process.env.OPENROUTER_API_KEY;
 if (!apiKey) throw new Error("OPENROUTER_API_KEY is required for the live Custom Widget AI evaluation");
 
 const requestedCase = process.argv.find((value) => value.startsWith("--case="))?.slice("--case=".length);
+const assistantMode = process.argv.includes("--assistant");
 const configuredLoops = Number(process.env.CUSTOM_WIDGET_AI_MAX_LOOPS ?? 10);
 const totalLoopBudget = Number.isInteger(configuredLoops) && configuredLoops > 0 ? Math.min(configuredLoops, 10) : 10;
+const selectableCases = assistantMode
+  ? CUSTOM_WIDGET_AI_EVALUATION_CASES.filter(
+      (testCase) =>
+        (testCase.sampleResponse !== undefined || testCase.previewResponses?.length) &&
+        testCase.expectations !== undefined,
+    )
+  : CUSTOM_WIDGET_AI_EVALUATION_CASES;
 const selectedCases = requestedCase
   ? CUSTOM_WIDGET_AI_EVALUATION_CASES.filter((testCase) => testCase.id === requestedCase)
-  : CUSTOM_WIDGET_AI_EVALUATION_CASES;
+  : selectableCases;
 if (selectedCases.length === 0) throw new Error(`Unknown AI evaluation case '${requestedCase}'`);
 if (totalLoopBudget < selectedCases.length) {
   throw new Error(`CUSTOM_WIDGET_AI_MAX_LOOPS must be at least ${selectedCases.length} for this run`);
@@ -23,20 +33,29 @@ const runId = new Date().toISOString().replaceAll(/[:.]/gu, "-");
 const outputRoot = path.resolve(process.cwd(), ".ai-evaluations", runId);
 await mkdir(outputRoot, { recursive: true });
 
-const results: AiEvaluationResult[] = [];
+const results: Array<AiEvaluationResult | CustomWidgetAssistantEvaluationResult> = [];
 let remainingLoopBudget = totalLoopBudget;
 for (const testCase of selectedCases) {
   const remainingCases = selectedCases.length - results.length;
   const maxLoops = Math.max(1, Math.floor(remainingLoopBudget / remainingCases));
   process.stdout.write(`Evaluating ${testCase.id}...\n`);
-  const result = await evaluateCustomWidgetCase({
-    testCase,
-    apiKey,
-    outputRoot,
-    maxLoops,
-    generatorModel: process.env.OPENROUTER_GENERATOR_MODEL ?? DEFAULT_GENERATOR_MODEL,
-    judgeModel: process.env.OPENROUTER_JUDGE_MODEL ?? DEFAULT_JUDGE_MODEL,
-  });
+  const result = assistantMode
+    ? await evaluateCustomWidgetAssistantCase({
+        testCase,
+        apiKey,
+        outputRoot,
+        maxLoops,
+        generatorModel: process.env.OPENROUTER_GENERATOR_MODEL ?? DEFAULT_GENERATOR_MODEL,
+        judgeModel: process.env.OPENROUTER_JUDGE_MODEL ?? DEFAULT_JUDGE_MODEL,
+      })
+    : await evaluateCustomWidgetCase({
+        testCase,
+        apiKey,
+        outputRoot,
+        maxLoops,
+        generatorModel: process.env.OPENROUTER_GENERATOR_MODEL ?? DEFAULT_GENERATOR_MODEL,
+        judgeModel: process.env.OPENROUTER_JUDGE_MODEL ?? DEFAULT_JUDGE_MODEL,
+      });
   results.push(result);
   remainingLoopBudget -= result.attempts;
   process.stdout.write(
@@ -48,6 +67,7 @@ for (const testCase of selectedCases) {
 
 const summary = {
   generatedAt: new Date().toISOString(),
+  mode: assistantMode ? "assistant-tool-loop" : "manifest",
   generatorModel: process.env.OPENROUTER_GENERATOR_MODEL ?? DEFAULT_GENERATOR_MODEL,
   judgeModel: process.env.OPENROUTER_JUDGE_MODEL ?? DEFAULT_JUDGE_MODEL,
   results: results.map((result) => ({
@@ -57,6 +77,7 @@ const summary = {
     verdict: result.judge?.verdict ?? "fail",
     categories: result.judge?.categories ?? null,
     errors: result.errors,
+    calledTools: "calledTools" in result ? result.calledTools : undefined,
     outputDirectory: path.relative(process.cwd(), result.outputDirectory),
   })),
 };