refactor(assistant): generate MCP tool staging
This commit is contained in:
1 parent
0fbe1e3fa3
commit
05ae366fe1
17 files changed
+212
-380
No files matched your search
@@ -27,7 +27,7 @@ import {
|
||||
selectSequentialCustomWidgetToolCalls,
|
||||
} from "../src/core/assistant-tool-step";
|
||||
import type { HomarrCustomWidgetV2 } from "../src/core/custom-jsx-schema";
|
||||
import { CUSTOM_WIDGET_ASSISTANT_POLICY, CUSTOM_WIDGET_LAZY_TOOL_DISCOVERY_INSTRUCTION } from "../src/core/ai-prompt";
|
||||
import { CUSTOM_WIDGET_ASSISTANT_POLICY, CUSTOM_WIDGET_TOOL_STAGING_INSTRUCTION } from "../src/core/ai-prompt";
|
||||
import { getCustomWidgetJsonSchema } from "../src/core/schema";
|
||||
import { addCustomJsxDiagnosticSourceExcerpts, validateCustomJsxTemplate } from "../src/jsx";
|
||||
import type { CustomWidgetAiEvaluationCase } from "./ai-evaluation-cases";
|
||||
@@ -76,7 +76,6 @@ type OpenRouterMessage =
|
||||
|
||||
const assistantEvaluationContextMaxCharacters = 48_000;
|
||||
const reloadableEvaluationToolNames = new Set([
|
||||
"homarr_findTools",
|
||||
"customWidget_schema",
|
||||
"customWidget_getComponentCatalog",
|
||||
"customWidget_findComponents",
|
||||
@@ -210,7 +209,6 @@ interface AssistantEvaluationToolCall {
|
||||
export interface AssistantAttemptState {
|
||||
calledTools: string[];
|
||||
toolCalls: AssistantEvaluationToolCall[];
|
||||
activatedTools: Set<string>;
|
||||
validatedTemplates: Set<string>;
|
||||
previews: Map<string, PreviewState>;
|
||||
completedPreviewSignatures: Set<string>;
|
||||
@@ -253,15 +251,6 @@ const isRecord = (value: unknown): value is Record<string, unknown> =>
|
||||
typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
|
||||
export const customWidgetAssistantEvaluationToolDefinitions: ToolDefinition[] = [
|
||||
{
|
||||
type: "function",
|
||||
function: {
|
||||
name: "homarr_findTools",
|
||||
description:
|
||||
"Find Homarr tools by capability without loading every tool schema. Matching tools become callable on the next step.",
|
||||
parameters: objectSchema({ query: { type: "string", minLength: 2, maxLength: 240 } }, ["query"]),
|
||||
},
|
||||
},
|
||||
{
|
||||
type: "function",
|
||||
function: {
|
||||
@@ -453,12 +442,7 @@ export const customWidgetAssistantEvaluationToolDefinitions: ToolDefinition[] =
|
||||
},
|
||||
];
|
||||
|
||||
const initiallyActiveAssistantEvaluationTools = new Set([
|
||||
"homarr_findTools",
|
||||
"web_search",
|
||||
"customWidget_getSkill",
|
||||
"customWidget_validateTemplate",
|
||||
]);
|
||||
const initiallyActiveAssistantEvaluationTools = new Set(["web_search", "customWidget_getSkill"]);
|
||||
const maxFocusedComponentSearchesPerPhase = 4;
|
||||
const customWidgetContextToolBudgets: Readonly<Record<string, number>> = {
|
||||
customWidget_findComponents: maxFocusedComponentSearchesPerPhase,
|
||||
@@ -468,46 +452,6 @@ const customWidgetContextToolBudgets: Readonly<Record<string, number>> = {
|
||||
customWidget_getExample: 1,
|
||||
};
|
||||
|
||||
const normalizeToolSearchText = (value: string) =>
|
||||
value
|
||||
.replaceAll(/([a-z\d])([A-Z])/gu, "$1 $2")
|
||||
.toLowerCase()
|
||||
.replaceAll(/[^a-z\d]+/gu, " ")
|
||||
.trim();
|
||||
|
||||
const findAssistantEvaluationTools = (query: string) => {
|
||||
const terms = [
|
||||
...new Set(
|
||||
normalizeToolSearchText(query)
|
||||
.split(" ")
|
||||
.filter((term) => term.length > 2),
|
||||
),
|
||||
];
|
||||
const normalizedQuery = normalizeToolSearchText(query);
|
||||
const requestsCompleteCatalog =
|
||||
normalizedQuery.includes("all components") ||
|
||||
normalizedQuery.includes("full component catalog") ||
|
||||
normalizedQuery.includes("complete component catalog");
|
||||
return customWidgetAssistantEvaluationToolDefinitions
|
||||
.filter(({ function: definition }) => !initiallyActiveAssistantEvaluationTools.has(definition.name))
|
||||
.filter(({ function: definition }) => {
|
||||
if (definition.name === "customWidget_getComponentCatalog") return requestsCompleteCatalog;
|
||||
if (definition.name === "customWidget_schema") return false;
|
||||
return true;
|
||||
})
|
||||
.map((tool) => {
|
||||
const searchable = normalizeToolSearchText(`${tool.function.name} ${tool.function.description}`);
|
||||
const score = terms.reduce((total, term) => total + (searchable.includes(term) ? 1 : 0), 0);
|
||||
return { tool, score };
|
||||
})
|
||||
.filter(({ score }) => score > 0)
|
||||
.toSorted(
|
||||
(left, right) => right.score - left.score || left.tool.function.name.localeCompare(right.tool.function.name),
|
||||
)
|
||||
.slice(0, 8)
|
||||
.map(({ tool }) => tool);
|
||||
};
|
||||
|
||||
export const getActiveAssistantEvaluationToolDefinitions = (state: AssistantAttemptState) => {
|
||||
const availableNames = customWidgetAssistantEvaluationToolDefinitions.map(
|
||||
({ function: definition }) => definition.name,
|
||||
@@ -516,7 +460,7 @@ export const getActiveAssistantEvaluationToolDefinitions = (state: AssistantAtte
|
||||
toolResults: [{ toolName: toolCall.name, output: toolCall.output }],
|
||||
}));
|
||||
const phaseToolNames = getCustomWidgetPhaseToolNames(availableNames, steps);
|
||||
const activeToolNames = new Set(phaseToolNames ?? state.activatedTools);
|
||||
const activeToolNames = new Set(phaseToolNames ?? initiallyActiveAssistantEvaluationTools);
|
||||
return customWidgetAssistantEvaluationToolDefinitions.filter(({ function: definition }) =>
|
||||
activeToolNames.has(definition.name),
|
||||
);
|
||||
@@ -693,22 +637,6 @@ const executeAssistantEvaluationToolCore = (
|
||||
if (contextBudget !== undefined && getContextToolCallsInCurrentPhase(state, name) >= contextBudget) {
|
||||
return getContextPhaseCompleteOutput(name);
|
||||
}
|
||||
if (name === "homarr_findTools") {
|
||||
const query = typeof input.query === "string" ? input.query : "";
|
||||
const matches = findAssistantEvaluationTools(query);
|
||||
for (const match of matches) state.activatedTools.add(match.function.name);
|
||||
return {
|
||||
query,
|
||||
matches: matches.map(({ function: definition }) => ({
|
||||
name: definition.name,
|
||||
description: definition.description,
|
||||
})),
|
||||
nextStep:
|
||||
matches.length > 0
|
||||
? "Use the matching tools now. Search again only for a different capability."
|
||||
: "No matching Homarr tool was found. Refine the capability.",
|
||||
};
|
||||
}
|
||||
if (name === "web_search") {
|
||||
return {
|
||||
query: typeof input.query === "string" ? input.query : "",
|
||||
@@ -1067,7 +995,7 @@ export function getAssistantEvaluationEfficiencyIssues(
|
||||
if (skillCall && skillCall.outputCharacters > 10_000) {
|
||||
issues.push(`The compact skill entrypoint used ${skillCall.outputCharacters} output characters.`);
|
||||
}
|
||||
const otherReusableContextTools = new Set(["homarr_findTools", "web_search"]);
|
||||
const otherReusableContextTools = new Set(["web_search"]);
|
||||
const seenContextCalls = new Set<string>();
|
||||
for (const toolCall of state.toolCalls) {
|
||||
if (toolCall.phaseLimited) continue;
|
||||
@@ -1154,7 +1082,6 @@ export function createAssistantEvaluationState(): AssistantAttemptState {
|
||||
return {
|
||||
calledTools: [],
|
||||
toolCalls: [],
|
||||
activatedTools: new Set(initiallyActiveAssistantEvaluationTools),
|
||||
validatedTemplates: new Set(),
|
||||
previews: new Map(),
|
||||
completedPreviewSignatures: new Set(),
|
||||
@@ -1260,7 +1187,7 @@ async function runAssistantAttempt(args: {
|
||||
const messages: OpenRouterMessage[] = [
|
||||
{
|
||||
role: "system",
|
||||
content: `${CUSTOM_WIDGET_LAZY_TOOL_DISCOVERY_INSTRUCTION}\n\n${CUSTOM_WIDGET_ASSISTANT_POLICY}\n\nThis is an unassisted tool-use evaluation. No tool will be forced for you. Complete and persist all ${getExpectedWidgetCount(args.testCase)} requested widget jobs before returning prose.`,
|
||||
content: `${CUSTOM_WIDGET_TOOL_STAGING_INSTRUCTION}\n\n${CUSTOM_WIDGET_ASSISTANT_POLICY}\n\nThis is an unassisted tool-use evaluation. No tool will be forced for you. Complete and persist all ${getExpectedWidgetCount(args.testCase)} requested widget jobs before returning prose.`,
|
||||
},
|
||||
{ role: "user", content: buildAssistantPrompt(args.testCase, args.feedback) },
|
||||
];
|
||||
|
||||
@@ -110,8 +110,8 @@ Use customWidget_validateTemplate for focused JSX repair without resending the m
|
||||
|
||||
export const CUSTOM_WIDGET_AUTHORING_PROMPT = AUTHORING_PROMPT;
|
||||
|
||||
export const CUSTOM_WIDGET_LAZY_TOOL_DISCOVERY_INSTRUCTION =
|
||||
"Use homarr_findTools for task-needed component, reference, lifecycle, configuration, or placement tools; external MCP keeps live schema and Assistant context compact.";
|
||||
export const CUSTOM_WIDGET_TOOL_STAGING_INSTRUCTION =
|
||||
"Custom Widget tools are staged by the authoring lifecycle. Use only visible task-needed tools; successful phases expose the next typed tools without loading the full catalog.";
|
||||
|
||||
export const CUSTOM_WIDGET_ASSISTANT_POLICY = `Custom Widget work:
|
||||
- Use customWidget tools; never substitute prose.
|
||||
|
||||
@@ -31,6 +31,15 @@ const customWidgetDraftPhaseToolNames = new Set([
|
||||
"customWidget_getSharedProps",
|
||||
"customWidget_validateTemplate",
|
||||
]);
|
||||
const customWidgetContextPhaseToolNames = new Set([
|
||||
"customWidget_getReference",
|
||||
"customWidget_findComponents",
|
||||
"customWidget_getComponents",
|
||||
"customWidget_getComponent",
|
||||
"customWidget_getSharedProps",
|
||||
"customWidget_getExample",
|
||||
"customWidget_validateTemplate",
|
||||
]);
|
||||
const maxFocusedComponentSearchesPerPhase = 4;
|
||||
|
||||
export interface CustomWidgetToolStep {
|
||||
@@ -182,5 +191,11 @@ export const getCustomWidgetPhaseToolNames = <TToolName extends string>(
|
||||
if (focusedSearches >= maxFocusedComponentSearchesPerPhase) {
|
||||
return availableToolNames.filter((toolName) => customWidgetDraftPhaseToolNames.has(toolName));
|
||||
}
|
||||
const skillLoaded = steps.some((step) =>
|
||||
step.toolResults.some((result) => result.toolName === "customWidget_getSkill"),
|
||||
);
|
||||
if (skillLoaded) {
|
||||
return availableToolNames.filter((toolName) => customWidgetContextPhaseToolNames.has(toolName));
|
||||
}
|
||||
return null;
|
||||
};
|
||||
@@ -336,50 +336,38 @@ describe("Custom Widget assistant live evaluation harness", () => {
|
||||
expect(names).not.toContain("customWidget_validate");
|
||||
});
|
||||
|
||||
it("starts with only the authoring bootstrap and discovers later lifecycle tools on demand", () => {
|
||||
it("starts with compact authoring context tools and stages later lifecycle tools by phase", () => {
|
||||
const testCase = getCase("seerr-media-workflows");
|
||||
const state = createAssistantEvaluationState();
|
||||
const getActiveNames = () =>
|
||||
getActiveAssistantEvaluationToolDefinitions(state).map(({ function: definition }) => definition.name);
|
||||
|
||||
expect(getActiveNames()).toEqual(["web_search", "customWidget_getSkill"]);
|
||||
executeAssistantEvaluationTool(testCase, state, "customWidget_getSkill", {});
|
||||
expect(getActiveNames()).toEqual([
|
||||
"homarr_findTools",
|
||||
"web_search",
|
||||
"customWidget_getSkill",
|
||||
"customWidget_getReference",
|
||||
"customWidget_findComponents",
|
||||
"customWidget_getComponents",
|
||||
"customWidget_getComponent",
|
||||
"customWidget_getSharedProps",
|
||||
"customWidget_getExample",
|
||||
"customWidget_validateTemplate",
|
||||
]);
|
||||
expect(
|
||||
executeAssistantEvaluationTool(testCase, state, "homarr_findTools", {
|
||||
query: "preview create revise template query action persist journal",
|
||||
}),
|
||||
).toMatchObject({
|
||||
matches: expect.arrayContaining([
|
||||
expect.objectContaining({ name: "customWidget_previewCreate" }),
|
||||
expect.objectContaining({ name: "customWidget_previewReviseTemplate" }),
|
||||
expect.objectContaining({ name: "customWidget_previewQuery" }),
|
||||
expect.objectContaining({ name: "customWidget_previewAction" }),
|
||||
expect.objectContaining({ name: "customWidget_createFromPreview" }),
|
||||
]),
|
||||
});
|
||||
expect(getActiveNames()).toEqual(
|
||||
expect.arrayContaining([
|
||||
"customWidget_previewCreate",
|
||||
"customWidget_previewReviseTemplate",
|
||||
"customWidget_previewQuery",
|
||||
"customWidget_previewAction",
|
||||
"customWidget_createFromPreview",
|
||||
]),
|
||||
expect(getActiveNames()).not.toEqual(
|
||||
expect.arrayContaining(["customWidget_previewCreate", "customWidget_createFromPreview"]),
|
||||
);
|
||||
expect(
|
||||
executeAssistantEvaluationTool(testCase, state, "homarr_findTools", {
|
||||
query: "customWidget_findComponents focused component search and selected component documentation batch",
|
||||
}),
|
||||
).toMatchObject({
|
||||
matches: expect.arrayContaining([
|
||||
expect.objectContaining({ name: "customWidget_findComponents" }),
|
||||
expect.objectContaining({ name: "customWidget_getComponents" }),
|
||||
]),
|
||||
|
||||
executeAssistantEvaluationTool(testCase, state, "customWidget_getComponents", {
|
||||
names: ["Stack", "TextInput", "Button"],
|
||||
});
|
||||
expect(getActiveNames()).toEqual(["customWidget_getReference", "customWidget_validateTemplate"]);
|
||||
|
||||
validateTemplate(testCase, state, mediaResearchWidget);
|
||||
expect(getActiveNames()).toEqual([
|
||||
"customWidget_validateTemplate",
|
||||
"customWidget_previewCreate",
|
||||
"customWidget_previewReviseTemplate",
|
||||
]);
|
||||
});
|
||||
|
||||
it("compacts obsolete documentation and validation calls but retains the current preview", () => {
|
||||
@@ -855,9 +843,7 @@ describe("Custom Widget assistant live evaluation harness", () => {
|
||||
const testCase = getCase("fake-service-health");
|
||||
const state = createAssistantEvaluationState();
|
||||
|
||||
executeAssistantEvaluationTool(testCase, state, "homarr_findTools", {
|
||||
query: "focused component search and selected component documentation batch",
|
||||
});
|
||||
executeAssistantEvaluationTool(testCase, state, "customWidget_getSkill", {});
|
||||
executeAssistantEvaluationTool(testCase, state, "customWidget_findComponents", {
|
||||
query: "health status layout",
|
||||
});
|
||||
|
||||
@@ -5,7 +5,7 @@ import {
|
||||
buildCustomWidgetMcpPrompt,
|
||||
CUSTOM_WIDGET_ASSISTANT_POLICY,
|
||||
CUSTOM_WIDGET_FINAL_OUTPUT_INSTRUCTION,
|
||||
CUSTOM_WIDGET_LAZY_TOOL_DISCOVERY_INSTRUCTION,
|
||||
CUSTOM_WIDGET_TOOL_STAGING_INSTRUCTION,
|
||||
CUSTOM_WIDGET_MCP_AUTHORING_PROMPT,
|
||||
} from "../core/ai-prompt";
|
||||
|
||||
@@ -54,8 +54,8 @@ describe("AI prompt", () => {
|
||||
expect(CUSTOM_WIDGET_ASSISTANT_POLICY).toContain("change the active phase");
|
||||
expect(CUSTOM_WIDGET_ASSISTANT_POLICY).toContain("customWidget_getComponents");
|
||||
expect(CUSTOM_WIDGET_ASSISTANT_POLICY).toContain("Do not load the full catalog");
|
||||
expect(CUSTOM_WIDGET_LAZY_TOOL_DISCOVERY_INSTRUCTION).toContain("homarr_findTools");
|
||||
expect(CUSTOM_WIDGET_LAZY_TOOL_DISCOVERY_INSTRUCTION).toContain("task-needed");
|
||||
expect(CUSTOM_WIDGET_TOOL_STAGING_INSTRUCTION).toContain("staged by the authoring lifecycle");
|
||||
expect(CUSTOM_WIDGET_TOOL_STAGING_INSTRUCTION).toContain("task-needed");
|
||||
expect(CUSTOM_WIDGET_ASSISTANT_POLICY).toContain("Reuse loaded context");
|
||||
expect(CUSTOM_WIDGET_ASSISTANT_POLICY).toContain("no arbitrary documentation or creativity cap");
|
||||
expect(CUSTOM_WIDGET_ASSISTANT_POLICY).toContain("one focused component search per widget job");
|
||||
|
||||
Reference in new issue
Block a user