fix: harden assistant authoring and UI

This commit is contained in:
ajnart authored and Thomas Camlong committed 2026-08-09 18:09:11 +02:00
1 parent fa0e55cc93
commit edc7b4d55b
51 files changed
+4118 -916

No files matched your search

@@ -1,7 +1,12 @@
import { describe, expect, test } from "vitest";
import type { UIMessage } from "ai";
import { getForcedAssistantToolName, withAssistantToolPolicy } from "./assistant-tool-policy";
import {
customWidgetAssistantInstructions,
getForcedAssistantToolName,
getRequiredAssistantToolNames,
withAssistantToolPolicy,
} from "./assistant-tool-policy";
const assistantMessage = (...parts: UIMessage["parts"]): UIMessage => ({
id: "assistant-message",
@@ -23,6 +28,27 @@ describe("withAssistantToolPolicy", () => {
});
});
describe("customWidgetAssistantInstructions", () => {
test("requires the complete skill and real preview-query data before creation", () => {
expect(customWidgetAssistantInstructions).toContain("first call customWidget_getSkill");
expect(customWidgetAssistantInstructions).toContain("customWidget_schema");
expect(customWidgetAssistantInstructions).toContain("customWidget_getComponentCatalog");
expect(customWidgetAssistantInstructions).toContain("customWidget_getComponent");
expect(customWidgetAssistantInstructions).toContain("customWidget_getSharedProps once");
expect(customWidgetAssistantInstructions).toContain("customWidget_getExample");
expect(customWidgetAssistantInstructions).toContain("templateLines");
expect(customWidgetAssistantInstructions).toContain("customWidget_previewQuery for every query");
expect(customWidgetAssistantInstructions).toContain("Call customWidget_create only after");
expect(customWidgetAssistantInstructions).toContain("managementPath");
});
test("routes meaningful follow-up choices through ask_user instead of prose", () => {
expect(customWidgetAssistantInstructions).toContain("call ask_user with explicit choices");
expect(customWidgetAssistantInstructions).toContain("Never end a custom-widget response with a prose question");
expect(customWidgetAssistantInstructions).toContain("purely rhetorical questions do not require ask_user");
});
});
describe("getForcedAssistantToolName", () => {
test("continues directly from the reviewed app form to app creation", () => {
expect(
@@ -152,3 +178,137 @@ describe("getForcedAssistantToolName", () => {
).toBeUndefined();
});
});
describe("getRequiredAssistantToolNames", () => {
test("requires a structured placement decision after custom widget creation", () => {
expect(
getRequiredAssistantToolNames([
assistantMessage({
type: "dynamic-tool",
toolName: "customWidget_create",
toolCallId: "create-widget-1",
input: { name: "PSG fixtures" },
state: "output-available",
output: {
id: "widget-1",
managementPath: "/manage/custom-widgets/edit/widget-1",
},
}),
]),
).toEqual(["configure_widget", "ask_user"]);
});
test("does not force a follow-up after failed custom widget creation", () => {
expect(
getRequiredAssistantToolNames([
assistantMessage({
type: "dynamic-tool",
toolName: "customWidget_create",
toolCallId: "create-widget-1",
input: { name: "PSG fixtures" },
state: "output-available",
output: { error: "The custom widget input was invalid." },
}),
]),
).toEqual([]);
});
test("requires the structured follow-up immediately after creation in the same agent run", () => {
const completedSteps = [
{
toolResults: [
{
toolName: "customWidget_previewQuery",
output: { status: 200, data: { events: [] } },
},
],
},
{
toolResults: [
{
toolName: "customWidget_create",
output: {
id: "widget-1",
managementPath: "/manage/custom-widgets/edit/widget-1",
nextAction: {
type: "place-custom-widget",
widgetKind: "customApi",
options: { definitionId: "widget-1" },
},
},
},
],
},
];
expect(getRequiredAssistantToolNames([], completedSteps)).toEqual(["configure_widget", "ask_user"]);
});
test("requires the structured follow-up after an approved creation executes before step zero", () => {
const responseMessages = [
{
role: "tool",
content: [
{
type: "tool-result",
toolCallId: "create-widget-1",
toolName: "customWidget_create",
output: {
type: "json",
value: {
id: "widget-1",
managementPath: "/manage/custom-widgets/edit/widget-1",
nextAction: {
type: "place-custom-widget",
widgetKind: "customApi",
options: { definitionId: "widget-1" },
},
},
},
},
],
},
];
expect(getRequiredAssistantToolNames([], [], responseMessages)).toEqual(["configure_widget", "ask_user"]);
});
test("does not keep forcing the creation follow-up after a later step has completed", () => {
const completedSteps = [
{
toolResults: [
{
toolName: "customWidget_create",
output: {
id: "widget-1",
managementPath: "/manage/custom-widgets/edit/widget-1",
},
},
],
},
{
toolResults: [{ toolName: "board_all", output: { boards: [] } }],
},
];
expect(getRequiredAssistantToolNames([], completedSteps)).toEqual([]);
});
test("does not require a follow-up for an in-request creation error", () => {
expect(
getRequiredAssistantToolNames(
[],
[
{
toolResults: [
{
toolName: "customWidget_create",
output: { error: "The custom widget input was invalid." },
},
],
},
],
),
).toEqual([]);
});
});