refactor(assistant): generate MCP tool staging
This commit is contained in:
ajnart committed
1 parent
0fbe1e3fa3
commit
05ae366fe1
17 files changed
+212
-380
No files matched your search
@@ -201,8 +201,8 @@ Load the installed skill before authoring. `customWidget_getSkill` returns its c
|
||||
reference index. Load a named schema, runtime, or security reference with `customWidget_getReference` only when the
|
||||
design needs it. Read the live JSON Schema or component resources when their detail is required by the design.
|
||||
|
||||
1. Call `customWidget_getSkill`. In Homarr Assistant, use `homarr_findTools` to activate later authoring tools only when
|
||||
they are needed.
|
||||
1. Call `customWidget_getSkill`. Homarr Assistant stages the next typed authoring tools automatically as the widget
|
||||
moves through context, validation, preview, evidence, and persistence phases.
|
||||
2. Load the live schema and applicable named references on demand. Prefer `customWidget_findComponents` for a small
|
||||
release-matched result set; reserve the full catalog for broad exploration. Fetch only relevant installed component
|
||||
documentation with `customWidget_getComponents` (or `customWidget_getComponent` for one repair) and, when useful,
|
||||
|
||||
@@ -22,7 +22,7 @@ describe("createCustomWidgetToolStepGate", () => {
|
||||
gate.begin(0);
|
||||
expect(gate.claim("customWidget_getSkill")).toBe(true);
|
||||
expect(gate.claim("web_search")).toBe(true);
|
||||
expect(gate.claim("homarr_findTools")).toBe(true);
|
||||
expect(gate.claim("homarr_enableToolGroups")).toBe(true);
|
||||
expect(gate.claim("customWidget_getReference")).toBe(true);
|
||||
expect(gate.claim("customWidget_getComponent")).toBe(true);
|
||||
expect(gate.claim("customWidget_getComponent")).toBe(false);
|
||||
@@ -30,7 +30,7 @@ describe("createCustomWidgetToolStepGate", () => {
|
||||
|
||||
gate.begin(1);
|
||||
expect(gate.claim("customWidget_validateTemplate")).toBe(true);
|
||||
expect(gate.claim("homarr_findTools")).toBe(false);
|
||||
expect(gate.claim("homarr_enableToolGroups")).toBe(false);
|
||||
expect(gate.claim("customWidget_previewCreate")).toBe(false);
|
||||
|
||||
gate.begin(2);
|
||||
|
||||
@@ -1,10 +1,7 @@
|
||||
import { describe, expect, test } from "vitest";
|
||||
import type { ModelMessage, UIMessage } from "ai";
|
||||
|
||||
import {
|
||||
compactAssistantStepMessages,
|
||||
convertAssistantMessagesToModelMessages,
|
||||
} from "./assistant-message-conversion";
|
||||
import { compactAssistantStepMessages, convertAssistantMessagesToModelMessages } from "./assistant-message-conversion";
|
||||
|
||||
describe("convertAssistantMessagesToModelMessages", () => {
|
||||
test("drops an interrupted tool call while preserving completed tool results", async () => {
|
||||
@@ -205,8 +202,8 @@ describe("convertAssistantMessagesToModelMessages", () => {
|
||||
{
|
||||
type: "tool-call",
|
||||
toolCallId: "regular-tool",
|
||||
toolName: "homarr_findTools",
|
||||
input: { query: "boards" },
|
||||
toolName: "homarr_enableToolGroups",
|
||||
input: { groups: ["board"] },
|
||||
},
|
||||
],
|
||||
},
|
||||
@@ -228,8 +225,8 @@ describe("convertAssistantMessagesToModelMessages", () => {
|
||||
{
|
||||
type: "tool-result",
|
||||
toolCallId: "regular-tool",
|
||||
toolName: "homarr_findTools",
|
||||
output: { type: "json", value: { matches: [] } },
|
||||
toolName: "homarr_enableToolGroups",
|
||||
output: { type: "json", value: { enabledGroups: ["board"] } },
|
||||
},
|
||||
],
|
||||
},
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
import { describe, expect, test } from "vitest";
|
||||
import { z } from "zod/v4";
|
||||
|
||||
import type { McpTool } from "@homarr/api/mcp";
|
||||
|
||||
import { filterAssistantMcpToolMatches, findAssistantMcpTools } from "./assistant-tool-discovery";
|
||||
|
||||
const createTool = (name: string, description: string, pathInRouter = name.split("_")): McpTool => ({
|
||||
name,
|
||||
description,
|
||||
pathInRouter,
|
||||
inputSchema: { type: "object", properties: {} },
|
||||
inputValidator: z.object({}),
|
||||
});
|
||||
|
||||
describe("Assistant tool discovery", () => {
|
||||
const tools = [
|
||||
createTool("customWidget_previewCreate", "Validate a complete Custom Widget and create its preview."),
|
||||
createTool("customWidget_previewQuery", "Test a query from a Custom Widget preview."),
|
||||
createTool("customWidget_createFromPreview", "Persist the exact tested Custom Widget preview."),
|
||||
createTool("mediaRequests_search", "Search Seerr media requests."),
|
||||
createTool("board_getAllBoards", "List Homarr boards."),
|
||||
];
|
||||
|
||||
test("finds only the small relevant lifecycle subset", () => {
|
||||
expect(findAssistantMcpTools(tools, "custom widget preview and persist", 3).map(({ name }) => name)).toEqual([
|
||||
"customWidget_createFromPreview",
|
||||
"customWidget_previewCreate",
|
||||
"customWidget_previewQuery",
|
||||
]);
|
||||
});
|
||||
|
||||
test("uses service aliases without loading unrelated schemas", () => {
|
||||
expect(findAssistantMcpTools(tools, "Seerr movie requests").map(({ name }) => name)).toEqual([
|
||||
"mediaRequests_search",
|
||||
]);
|
||||
});
|
||||
|
||||
test("returns no tools when a query contains only ignored terms", () => {
|
||||
expect(findAssistantMcpTools(tools, "a an and the to")).toEqual([]);
|
||||
});
|
||||
|
||||
test("keeps heavyweight authoring resources hidden unless explicitly requested", () => {
|
||||
const matches = [
|
||||
createTool("customWidget_getComponentCatalog", "Get the complete component catalog."),
|
||||
createTool("customWidget_schema", "Get the complete JSON Schema."),
|
||||
createTool("customWidget_findComponents", "Find focused components."),
|
||||
];
|
||||
|
||||
expect(filterAssistantMcpToolMatches(matches, "custom widget", true).map(({ name }) => name)).toEqual([
|
||||
"customWidget_findComponents",
|
||||
]);
|
||||
expect(filterAssistantMcpToolMatches(matches, "full component catalog and live schema", true).map(({ name }) =>
|
||||
name,
|
||||
)).toEqual(["customWidget_getComponentCatalog", "customWidget_findComponents"]);
|
||||
expect(filterAssistantMcpToolMatches(matches, "custom widget", false)).toEqual(matches);
|
||||
});
|
||||
});
|
||||
@@ -1,110 +0,0 @@
|
||||
import type { UIMessage } from "ai";
|
||||
|
||||
import type { McpTool } from "@homarr/api/mcp";
|
||||
|
||||
const ignoredSearchTerms = new Set([
|
||||
"a",
|
||||
"an",
|
||||
"and",
|
||||
"can",
|
||||
"create",
|
||||
"for",
|
||||
"from",
|
||||
"get",
|
||||
"in",
|
||||
"list",
|
||||
"make",
|
||||
"my",
|
||||
"of",
|
||||
"on",
|
||||
"please",
|
||||
"the",
|
||||
"to",
|
||||
"want",
|
||||
"with",
|
||||
]);
|
||||
|
||||
const searchTermAliases: Record<string, readonly string[]> = {
|
||||
account: ["user"],
|
||||
dashboard: ["board"],
|
||||
k8s: ["kubernetes"],
|
||||
movie: ["media", "radarr", "request"],
|
||||
seerr: ["overseerr", "jellyseerr", "media", "request"],
|
||||
show: ["media", "sonarr", "request"],
|
||||
streaming: ["stream", "media", "server"],
|
||||
};
|
||||
|
||||
const normalizeSearchText = (value: string) =>
|
||||
value
|
||||
.replaceAll(/([a-z\d])([A-Z])/gu, "$1 $2")
|
||||
.normalize("NFKD")
|
||||
.toLowerCase()
|
||||
.replaceAll(/[^a-z\d]+/gu, " ")
|
||||
.trim();
|
||||
|
||||
const getSearchTerms = (query: string) => {
|
||||
const terms = normalizeSearchText(query)
|
||||
.split(" ")
|
||||
.filter((term) => term.length > 1 && !ignoredSearchTerms.has(term));
|
||||
return [...new Set(terms.flatMap((term) => [term, ...(searchTermAliases[term] ?? [])]))];
|
||||
};
|
||||
|
||||
const scoreTool = (tool: McpTool, query: string, terms: readonly string[]) => {
|
||||
const normalizedName = normalizeSearchText(tool.name);
|
||||
const normalizedPath = normalizeSearchText(tool.pathInRouter.join(" "));
|
||||
const normalizedDescription = normalizeSearchText(tool.description);
|
||||
const searchable = `${normalizedName} ${normalizedPath} ${normalizedDescription}`;
|
||||
let score = 0;
|
||||
if (normalizedName.includes(query)) score += 40;
|
||||
if (normalizedDescription.includes(query)) score += 20;
|
||||
for (const term of terms) {
|
||||
if (normalizedName.split(" ").includes(term)) score += 12;
|
||||
else if (normalizedName.includes(term)) score += 7;
|
||||
if (normalizedPath.split(" ").includes(term)) score += 6;
|
||||
if (normalizedDescription.split(" ").includes(term)) score += 4;
|
||||
else if (searchable.includes(term)) score += 1;
|
||||
}
|
||||
return score;
|
||||
};
|
||||
|
||||
export const findAssistantMcpTools = (tools: readonly McpTool[], query: string, limit = 8) => {
|
||||
const normalizedQuery = normalizeSearchText(query);
|
||||
const terms = getSearchTerms(query);
|
||||
if (!normalizedQuery || terms.length === 0) return [];
|
||||
|
||||
return tools
|
||||
.map((tool) => ({ tool, score: scoreTool(tool, normalizedQuery, terms) }))
|
||||
.filter(({ score }) => score > 0)
|
||||
.toSorted((left, right) => right.score - left.score || left.tool.name.localeCompare(right.tool.name))
|
||||
.slice(0, limit)
|
||||
.map(({ tool }) => tool);
|
||||
};
|
||||
|
||||
export const filterAssistantMcpToolMatches = <TTool extends Pick<McpTool, "name">>(
|
||||
tools: readonly TTool[],
|
||||
query: string,
|
||||
customWidgetAuthoringActive: boolean,
|
||||
) => {
|
||||
if (!customWidgetAuthoringActive) return [...tools];
|
||||
const normalizedQuery = normalizeSearchText(query);
|
||||
const requestsCompleteCatalog =
|
||||
normalizedQuery.includes("all components") ||
|
||||
["full component catalog", "complete component catalog"].some((phrase) => normalizedQuery.includes(phrase));
|
||||
return tools.filter((tool) => {
|
||||
if (tool.name === "customWidget_getComponentCatalog") return requestsCompleteCatalog;
|
||||
if (tool.name === "customWidget_schema") return false;
|
||||
return true;
|
||||
});
|
||||
};
|
||||
|
||||
export const getLatestAssistantUserText = (messages: UIMessage[]) => {
|
||||
const latestUserMessage = messages.toReversed().find((message) => message.role === "user");
|
||||
if (!latestUserMessage) return "";
|
||||
return latestUserMessage.parts
|
||||
.flatMap((part) => {
|
||||
if (typeof part !== "object" || part === null || !("type" in part) || part.type !== "text") return [];
|
||||
if (!("text" in part) || typeof part.text !== "string") return [];
|
||||
return [part.text];
|
||||
})
|
||||
.join("\n");
|
||||
};
|
||||
@@ -0,0 +1,40 @@
|
||||
import { describe, expect, test } from "vitest";
|
||||
|
||||
import { createAssistantMcpToolGroups } from "./assistant-tool-groups";
|
||||
|
||||
const createTool = (name: string, group: string) => ({
|
||||
name,
|
||||
description: `${name} description`,
|
||||
pathInRouter: [group, name],
|
||||
});
|
||||
|
||||
describe("Assistant MCP tool groups", () => {
|
||||
test("derives deterministic groups from router paths", () => {
|
||||
const groups = createAssistantMcpToolGroups([
|
||||
createTool("board_search", "board"),
|
||||
createTool("mediaRequests_search", "mediaRequests"),
|
||||
createTool("board_getAll", "board"),
|
||||
]);
|
||||
|
||||
expect(groups.ids).toEqual(["board", "mediaRequests"]);
|
||||
expect(groups.resolve(["board"])[0]?.tools.map(({ name }) => name)).toEqual(["board_getAll", "board_search"]);
|
||||
});
|
||||
|
||||
test("deduplicates requested groups while preserving request order", () => {
|
||||
const groups = createAssistantMcpToolGroups([
|
||||
createTool("board_getAll", "board"),
|
||||
createTool("docker_getContainers", "docker"),
|
||||
]);
|
||||
|
||||
expect(groups.resolve(["docker", "board", "docker"]).map(({ id }) => id)).toEqual(["docker", "board"]);
|
||||
});
|
||||
|
||||
test("rejects tools without a router group and unknown group IDs", () => {
|
||||
expect(() => createAssistantMcpToolGroups([{ ...createTool("info_get", "info"), pathInRouter: [] }])).toThrow(
|
||||
"has no router group",
|
||||
);
|
||||
|
||||
const groups = createAssistantMcpToolGroups([createTool("info_get", "info")]);
|
||||
expect(() => groups.resolve(["missing"])).toThrow("Unknown Assistant MCP tool group 'missing'");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,38 @@
|
||||
import type { McpTool } from "@homarr/api/mcp";
|
||||
|
||||
type AssistantMcpTool = Pick<McpTool, "name" | "pathInRouter">;
|
||||
|
||||
export interface AssistantMcpToolGroup<TTool extends AssistantMcpTool = AssistantMcpTool> {
|
||||
id: string;
|
||||
tools: readonly TTool[];
|
||||
}
|
||||
|
||||
export const createAssistantMcpToolGroups = <TTool extends AssistantMcpTool>(tools: readonly TTool[]) => {
|
||||
const toolsByGroup = new Map<string, TTool[]>();
|
||||
for (const tool of tools) {
|
||||
const groupId = tool.pathInRouter[0];
|
||||
if (!groupId) throw new Error(`Assistant MCP tool '${tool.name}' has no router group`);
|
||||
const groupedTools = toolsByGroup.get(groupId) ?? [];
|
||||
groupedTools.push(tool);
|
||||
toolsByGroup.set(groupId, groupedTools);
|
||||
}
|
||||
|
||||
const groups = [...toolsByGroup.entries()]
|
||||
.toSorted(([left], [right]) => left.localeCompare(right))
|
||||
.map(([id, groupedTools]) => ({
|
||||
id,
|
||||
tools: groupedTools.toSorted((left, right) => left.name.localeCompare(right.name)),
|
||||
}));
|
||||
const groupsById = new Map(groups.map((group) => [group.id, group]));
|
||||
|
||||
return {
|
||||
ids: groups.map(({ id }) => id),
|
||||
resolve(groupIds: readonly string[]): AssistantMcpToolGroup<TTool>[] {
|
||||
return [...new Set(groupIds)].map((groupId) => {
|
||||
const group = groupsById.get(groupId);
|
||||
if (!group) throw new Error(`Unknown Assistant MCP tool group '${groupId}'`);
|
||||
return group;
|
||||
});
|
||||
},
|
||||
};
|
||||
};
|
||||
@@ -30,7 +30,7 @@ describe("customWidgetAssistantInstructions", () => {
|
||||
test("loads authoring resources lazily and verifies every final preview", () => {
|
||||
expect(customWidgetAssistantInstructions.length).toBeLessThan(4_100);
|
||||
expect(customWidgetAssistantInstructions).toContain("customWidget_getSkill");
|
||||
expect(customWidgetAssistantInstructions).toContain("homarr_findTools");
|
||||
expect(customWidgetAssistantInstructions).toContain("staged by the authoring lifecycle");
|
||||
expect(customWidgetAssistantInstructions).toContain("task-needed");
|
||||
expect(customWidgetAssistantInstructions).toContain("Reuse loaded context");
|
||||
expect(customWidgetAssistantInstructions).toContain("customWidget_findComponents");
|
||||
|
||||
@@ -3,13 +3,13 @@ import type { UIMessage } from "ai";
|
||||
|
||||
import {
|
||||
CUSTOM_WIDGET_ASSISTANT_POLICY,
|
||||
CUSTOM_WIDGET_LAZY_TOOL_DISCOVERY_INSTRUCTION,
|
||||
CUSTOM_WIDGET_TOOL_STAGING_INSTRUCTION,
|
||||
} from "@homarr/custom-widgets/authoring-prompt";
|
||||
|
||||
const mutationApprovalInstruction =
|
||||
"Uses Homarr's native approval UI; call when inputs are ready without separate prose confirmation.";
|
||||
|
||||
export const customWidgetAssistantInstructions = `\n\n${CUSTOM_WIDGET_LAZY_TOOL_DISCOVERY_INSTRUCTION}\n\n${CUSTOM_WIDGET_ASSISTANT_POLICY}`;
|
||||
export const customWidgetAssistantInstructions = `\n\n${CUSTOM_WIDGET_TOOL_STAGING_INSTRUCTION}\n\n${CUSTOM_WIDGET_ASSISTANT_POLICY}`;
|
||||
|
||||
export const withAssistantToolPolicy = (description: string | undefined, requiresApproval: boolean) => {
|
||||
if (!requiresApproval) return description;
|
||||
|
||||
@@ -26,6 +26,13 @@ describe("Custom Widget authoring context", () => {
|
||||
expect(needsCustomWidgetAuthoringContext([userMessage(text)])).toBe(true);
|
||||
});
|
||||
|
||||
test.each(["List my custom widgets", "Delete a custom widget", "Explain custom widgets"])(
|
||||
"leaves management intent to the general Custom Widget MCP group: %s",
|
||||
(text) => {
|
||||
expect(needsCustomWidgetAuthoringContext([userMessage(text)])).toBe(false);
|
||||
},
|
||||
);
|
||||
|
||||
test("continues after a Custom Widget tool call", () => {
|
||||
const messages: UIMessage[] = [
|
||||
userMessage("Continue"),
|
||||
@@ -68,6 +75,8 @@ describe("Custom Widget authoring context", () => {
|
||||
"customWidget_schema",
|
||||
"customWidget_getComponentCatalog",
|
||||
"customWidget_getReference",
|
||||
"customWidget_findComponents",
|
||||
"customWidget_getComponents",
|
||||
"customWidget_getComponent",
|
||||
"customWidget_getSharedProps",
|
||||
"customWidget_getExample",
|
||||
@@ -78,7 +87,6 @@ describe("Custom Widget authoring context", () => {
|
||||
|
||||
expect(getActiveCustomWidgetToolNames(tools, [userMessage("Create two custom widgets")], true)).toEqual([
|
||||
"customWidget_getSkill",
|
||||
"customWidget_validateTemplate",
|
||||
]);
|
||||
expect(getActiveCustomWidgetToolNames(tools, [userMessage("Create a custom widget")], false)).toEqual([]);
|
||||
});
|
||||
@@ -100,11 +108,7 @@ describe("Custom Widget authoring context", () => {
|
||||
getCustomWidgetPhaseToolNames(tools, [
|
||||
{ toolResults: [{ toolName: "customWidget_validateTemplate", output: { valid: true } }] },
|
||||
]),
|
||||
).toEqual([
|
||||
"customWidget_validateTemplate",
|
||||
"customWidget_previewCreate",
|
||||
"customWidget_previewReviseTemplate",
|
||||
]);
|
||||
).toEqual(["customWidget_validateTemplate", "customWidget_previewCreate", "customWidget_previewReviseTemplate"]);
|
||||
expect(
|
||||
getCustomWidgetPhaseToolNames(tools, [
|
||||
{ toolResults: [{ toolName: "customWidget_previewCreate", output: { success: true } }] },
|
||||
@@ -135,6 +139,30 @@ describe("Custom Widget authoring context", () => {
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
test("exposes focused context tools only after the skill entrypoint is loaded", () => {
|
||||
const tools = [
|
||||
"customWidget_getSkill",
|
||||
"customWidget_getReference",
|
||||
"customWidget_findComponents",
|
||||
"customWidget_getComponents",
|
||||
"customWidget_getExample",
|
||||
"customWidget_validateTemplate",
|
||||
"customWidget_previewCreate",
|
||||
];
|
||||
|
||||
expect(
|
||||
getCustomWidgetPhaseToolNames(tools, [
|
||||
{ toolResults: [{ toolName: "customWidget_getSkill", output: { content: "skill" } }] },
|
||||
]),
|
||||
).toEqual([
|
||||
"customWidget_getReference",
|
||||
"customWidget_findComponents",
|
||||
"customWidget_getComponents",
|
||||
"customWidget_getExample",
|
||||
"customWidget_validateTemplate",
|
||||
]);
|
||||
});
|
||||
|
||||
test("keeps one concrete component repair path open for validation warnings", () => {
|
||||
const tools = [
|
||||
"customWidget_findComponents",
|
||||
@@ -179,16 +207,10 @@ describe("Custom Widget authoring context", () => {
|
||||
],
|
||||
},
|
||||
{
|
||||
toolResults: [
|
||||
{ toolName: "customWidget_getComponent", output: { name: "RefreshButton", props: [] } },
|
||||
],
|
||||
toolResults: [{ toolName: "customWidget_getComponent", output: { name: "RefreshButton", props: [] } }],
|
||||
},
|
||||
]),
|
||||
).toEqual([
|
||||
"customWidget_validateTemplate",
|
||||
"customWidget_previewCreate",
|
||||
"customWidget_previewReviseTemplate",
|
||||
]);
|
||||
).toEqual(["customWidget_validateTemplate", "customWidget_previewCreate", "customWidget_previewReviseTemplate"]);
|
||||
});
|
||||
|
||||
test("keeps evidence tools until every preview request succeeds, then allows correction or persistence", () => {
|
||||
@@ -257,11 +279,7 @@ describe("Custom Widget authoring context", () => {
|
||||
...completeEvidence,
|
||||
{ toolResults: [{ toolName: "customWidget_validateTemplate", output: { valid: true } }] },
|
||||
]),
|
||||
).toEqual([
|
||||
"customWidget_validateTemplate",
|
||||
"customWidget_previewCreate",
|
||||
"customWidget_previewReviseTemplate",
|
||||
]);
|
||||
).toEqual(["customWidget_validateTemplate", "customWidget_previewCreate", "customWidget_previewReviseTemplate"]);
|
||||
|
||||
const revisedPreview = [
|
||||
...completeEvidence,
|
||||
@@ -310,9 +328,7 @@ describe("Custom Widget authoring context", () => {
|
||||
],
|
||||
},
|
||||
];
|
||||
expect(getCustomWidgetPhaseToolNames(tools, completeRevisedEvidence)).toEqual([
|
||||
"customWidget_createFromPreview",
|
||||
]);
|
||||
expect(getCustomWidgetPhaseToolNames(tools, completeRevisedEvidence)).toEqual(["customWidget_createFromPreview"]);
|
||||
});
|
||||
|
||||
test("closes focused discovery after four searches until validation", () => {
|
||||
|
||||
@@ -5,9 +5,9 @@ import { getCustomWidgetPhaseToolNames } from "@homarr/custom-widgets/core";
|
||||
export { getCustomWidgetPhaseToolNames };
|
||||
|
||||
const customWidgetIntentPattern =
|
||||
/(?:\bcustom[\s-]+widgets?\b|\bcustom\s+jsx\b|\bhomarr-custom-widget-v\d+\b|\bcustomWidget_[A-Za-z\d_]+\b|\b(?:build|create|design|make)\b[^\n]{0,80}\bwidgets?\s+(?:for|using|with)\b|\b(?:i|we)\s+(?:need|want)\b[^\n]{0,60}\bwidgets?\s+(?:for|using|with)\b)/iu;
|
||||
/(?:\bcustom\s+jsx\b|\bhomarr-custom-widget-v\d+\b|\b(?:build|create|design|edit|fix|make|repair|update|validate)\b[^\n]{0,80}\bcustom[\s-]+widgets?\b|\b(?:build|create|design|make)\b[^\n]{0,80}\bwidgets?\s+(?:for|using|with)\b|\b(?:i|we)\s+(?:need|want)\b[^\n]{0,60}\bwidgets?\s+(?:for|using|with)\b)/iu;
|
||||
|
||||
const customWidgetBootstrapToolNames = new Set(["customWidget_getSkill", "customWidget_validateTemplate"]);
|
||||
const customWidgetBootstrapToolNames = new Set(["customWidget_getSkill"]);
|
||||
const maxFocusedComponentSearchesPerPhase = 4;
|
||||
const customWidgetContextToolBudgets: Readonly<Record<string, number>> = {
|
||||
customWidget_findComponents: maxFocusedComponentSearchesPerPhase,
|
||||
|
||||
@@ -49,10 +49,7 @@ import {
|
||||
sanitizeAttachmentFilename,
|
||||
} from "./assistant-chat-input";
|
||||
import { getAssistantModelLookupStatus } from "./assistant-model-lookup";
|
||||
import {
|
||||
compactAssistantStepMessages,
|
||||
convertAssistantMessagesToModelMessages,
|
||||
} from "./assistant-message-conversion";
|
||||
import { compactAssistantStepMessages, convertAssistantMessagesToModelMessages } from "./assistant-message-conversion";
|
||||
import {
|
||||
getOpenRouterWebSearchRequests,
|
||||
getOpenRouterWebSearchSources,
|
||||
@@ -77,11 +74,7 @@ import {
|
||||
getCustomWidgetPhaseToolNames,
|
||||
needsCustomWidgetAuthoringContext,
|
||||
} from "./custom-widget-authoring-context";
|
||||
import {
|
||||
filterAssistantMcpToolMatches,
|
||||
findAssistantMcpTools,
|
||||
getLatestAssistantUserText,
|
||||
} from "./assistant-tool-discovery";
|
||||
import { createAssistantMcpToolGroups } from "./assistant-tool-groups";
|
||||
import {
|
||||
customWidgetAssistantInstructions,
|
||||
getForcedAssistantToolName,
|
||||
@@ -157,20 +150,12 @@ const requestSchema = z.object({
|
||||
.optional(),
|
||||
});
|
||||
|
||||
const assistantToolDiscoverySchema = z.object({
|
||||
query: z
|
||||
.string()
|
||||
.trim()
|
||||
.min(2)
|
||||
.max(240)
|
||||
.describe("A concise Homarr capability or action, such as 'search media requests' or 'list Kubernetes pods'."),
|
||||
});
|
||||
|
||||
const assistantToolDiscoveryName = "homarr_findTools";
|
||||
const assistantToolGroupActivationName = "homarr_enableToolGroups";
|
||||
const maxAssistantToolGroupsPerActivation = 4;
|
||||
|
||||
const assistantInstructions = `You are Homarr Assistant, embedded in the user's self-hosted Homarr dashboard.
|
||||
|
||||
Use Homarr tools for live instance data or actions; never invent resources, IDs, state, or results. Only a small relevant tool set is initially visible. Call ${assistantToolDiscoveryName} with a concise capability when a needed Homarr tool is not visible, then use the activated tool. Follow each tool's description and use integration IDs returned by discovery tools.
|
||||
Use Homarr tools for live instance data or actions; never invent resources, IDs, state, or results. Homarr tools are grouped by their MCP router namespace. Call ${assistantToolGroupActivationName} with the task-needed group when its typed tools are not visible, then use the activated tool. Follow each tool's description and use integration IDs returned by Homarr tools.
|
||||
|
||||
Homarr permissions are authoritative. Explain denied access without suggesting a bypass. Read before changing when current state matters. Mutations use Homarr's native approval UI: when inputs are sufficient, call the mutation immediately and never ask for duplicate prose confirmation or retry a denial. Use ask_user only when a missing choice blocks the next action; do not end with a prose question expecting a reply.
|
||||
|
||||
@@ -547,10 +532,7 @@ export async function POST(request: Request) {
|
||||
input !== null &&
|
||||
!Array.isArray(input)
|
||||
) {
|
||||
executionInput = normalizeCustomWidgetLifecycleToolInput(
|
||||
mcpTool.name,
|
||||
input as Record<string, unknown>,
|
||||
);
|
||||
executionInput = normalizeCustomWidgetLifecycleToolInput(mcpTool.name, input as Record<string, unknown>);
|
||||
}
|
||||
const contextRequestKey = getCustomWidgetContextRequestKey(mcpTool.name, executionInput);
|
||||
if (contextRequestKey !== null && loadedCustomWidgetContextRequests.has(contextRequestKey)) {
|
||||
@@ -621,45 +603,41 @@ export async function POST(request: Request) {
|
||||
];
|
||||
}),
|
||||
) satisfies ToolSet;
|
||||
const discoveredToolNames = new Set<string>();
|
||||
const latestUserText = getLatestAssistantUserText(incomingMessages);
|
||||
const automaticToolCandidates = customWidgetAuthoringActive
|
||||
const groupedToolCandidates = customWidgetAuthoringActive
|
||||
? mcpTools.filter((mcpTool) => !mcpTool.name.startsWith("customWidget_"))
|
||||
: mcpTools;
|
||||
const automaticallySelectedTools = findAssistantMcpTools(automaticToolCandidates, latestUserText, 6).map(
|
||||
({ name }) => name,
|
||||
);
|
||||
const toolDiscovery = tool({
|
||||
const assistantToolGroups = createAssistantMcpToolGroups(groupedToolCandidates);
|
||||
const assistantToolGroupIds = assistantToolGroups.ids as [string, ...string[]];
|
||||
const assistantToolGroupActivationSchema = z.object({
|
||||
groups: z
|
||||
.array(z.enum(assistantToolGroupIds))
|
||||
.min(1)
|
||||
.max(maxAssistantToolGroupsPerActivation)
|
||||
.describe("MCP router groups whose typed Homarr tools are needed for the current task."),
|
||||
});
|
||||
const enabledToolGroupIds = new Set<string>();
|
||||
const toolGroupActivation = tool({
|
||||
description:
|
||||
"Find Homarr tools by capability without loading the complete tool catalog. Matching tools become callable on the next step. Use a focused query and call again with a different capability when needed.",
|
||||
inputSchema: jsonSchema(z.toJSONSchema(assistantToolDiscoverySchema) as Parameters<typeof jsonSchema>[0]),
|
||||
"Enable task-needed typed Homarr tools by MCP router group. Enabled groups remain available on later steps.",
|
||||
inputSchema: jsonSchema(z.toJSONSchema(assistantToolGroupActivationSchema) as Parameters<typeof jsonSchema>[0]),
|
||||
execute: (value) => {
|
||||
if (customWidgetAuthoringActive && !customWidgetToolStepGate.claim(assistantToolDiscoveryName)) {
|
||||
if (customWidgetAuthoringActive && !customWidgetToolStepGate.claim(assistantToolGroupActivationName)) {
|
||||
return {
|
||||
error:
|
||||
"A Custom Widget tool must run in its own model step. Use that result before calling another tool.",
|
||||
error: "A Custom Widget tool must run in its own model step. Use that result before calling another tool.",
|
||||
};
|
||||
}
|
||||
const input = assistantToolDiscoverySchema.parse(value);
|
||||
const matches = filterAssistantMcpToolMatches(
|
||||
findAssistantMcpTools(mcpTools, input.query),
|
||||
input.query,
|
||||
customWidgetAuthoringActive,
|
||||
);
|
||||
for (const match of matches) discoveredToolNames.add(match.name);
|
||||
const input = assistantToolGroupActivationSchema.parse(value);
|
||||
const groups = assistantToolGroups.resolve(input.groups);
|
||||
for (const group of groups) enabledToolGroupIds.add(group.id);
|
||||
return {
|
||||
query: input.query,
|
||||
matches: matches.map((match) => ({ name: match.name, description: match.description })),
|
||||
nextStep:
|
||||
matches.length > 0
|
||||
? "Use one or more matching tools now. Search again only for a different capability."
|
||||
: "No matching Homarr tool was found. Refine the capability or explain that it is unavailable.",
|
||||
enabledGroups: groups.map(({ id }) => id),
|
||||
nextStep: "Use one or more enabled typed tools now. Enable another group only when the task needs it.",
|
||||
};
|
||||
},
|
||||
});
|
||||
const availableTools: ToolSet = {
|
||||
...homarrTools,
|
||||
[assistantToolDiscoveryName]: toolDiscovery,
|
||||
[assistantToolGroupActivationName]: toolGroupActivation,
|
||||
...frontendTools,
|
||||
};
|
||||
const frontendToolNames = Object.keys(frontendTools);
|
||||
@@ -669,14 +647,17 @@ export async function POST(request: Request) {
|
||||
canAuthorCustomWidgets,
|
||||
);
|
||||
const getActiveToolNames = (steps: Parameters<typeof getCustomWidgetPhaseToolNames>[1] = []) => {
|
||||
const enabledToolNames = assistantToolGroups
|
||||
.resolve([...enabledToolGroupIds])
|
||||
.flatMap((group) => group.tools.map(({ name }) => name));
|
||||
const phaseToolNames = customWidgetAuthoringActive
|
||||
? getCustomWidgetPhaseToolNames(Object.keys(homarrTools), steps)
|
||||
: null;
|
||||
if (phaseToolNames) return [...frontendToolNames, ...phaseToolNames];
|
||||
if (phaseToolNames) return [...frontendToolNames, ...enabledToolNames, ...phaseToolNames];
|
||||
return [
|
||||
assistantToolDiscoveryName,
|
||||
assistantToolGroupActivationName,
|
||||
...frontendToolNames,
|
||||
...new Set([...automaticallySelectedTools, ...discoveredToolNames, ...activeCustomWidgetToolNames]),
|
||||
...new Set([...enabledToolNames, ...activeCustomWidgetToolNames]),
|
||||
];
|
||||
};
|
||||
const forcedToolName = getForcedAssistantToolName(incomingMessages);
|
||||
|
||||
@@ -27,7 +27,7 @@ import {
|
||||
selectSequentialCustomWidgetToolCalls,
|
||||
} from "../src/core/assistant-tool-step";
|
||||
import type { HomarrCustomWidgetV2 } from "../src/core/custom-jsx-schema";
|
||||
import { CUSTOM_WIDGET_ASSISTANT_POLICY, CUSTOM_WIDGET_LAZY_TOOL_DISCOVERY_INSTRUCTION } from "../src/core/ai-prompt";
|
||||
import { CUSTOM_WIDGET_ASSISTANT_POLICY, CUSTOM_WIDGET_TOOL_STAGING_INSTRUCTION } from "../src/core/ai-prompt";
|
||||
import { getCustomWidgetJsonSchema } from "../src/core/schema";
|
||||
import { addCustomJsxDiagnosticSourceExcerpts, validateCustomJsxTemplate } from "../src/jsx";
|
||||
import type { CustomWidgetAiEvaluationCase } from "./ai-evaluation-cases";
|
||||
@@ -76,7 +76,6 @@ type OpenRouterMessage =
|
||||
|
||||
const assistantEvaluationContextMaxCharacters = 48_000;
|
||||
const reloadableEvaluationToolNames = new Set([
|
||||
"homarr_findTools",
|
||||
"customWidget_schema",
|
||||
"customWidget_getComponentCatalog",
|
||||
"customWidget_findComponents",
|
||||
@@ -210,7 +209,6 @@ interface AssistantEvaluationToolCall {
|
||||
export interface AssistantAttemptState {
|
||||
calledTools: string[];
|
||||
toolCalls: AssistantEvaluationToolCall[];
|
||||
activatedTools: Set<string>;
|
||||
validatedTemplates: Set<string>;
|
||||
previews: Map<string, PreviewState>;
|
||||
completedPreviewSignatures: Set<string>;
|
||||
@@ -253,15 +251,6 @@ const isRecord = (value: unknown): value is Record<string, unknown> =>
|
||||
typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
|
||||
export const customWidgetAssistantEvaluationToolDefinitions: ToolDefinition[] = [
|
||||
{
|
||||
type: "function",
|
||||
function: {
|
||||
name: "homarr_findTools",
|
||||
description:
|
||||
"Find Homarr tools by capability without loading every tool schema. Matching tools become callable on the next step.",
|
||||
parameters: objectSchema({ query: { type: "string", minLength: 2, maxLength: 240 } }, ["query"]),
|
||||
},
|
||||
},
|
||||
{
|
||||
type: "function",
|
||||
function: {
|
||||
@@ -453,12 +442,7 @@ export const customWidgetAssistantEvaluationToolDefinitions: ToolDefinition[] =
|
||||
},
|
||||
];
|
||||
|
||||
const initiallyActiveAssistantEvaluationTools = new Set([
|
||||
"homarr_findTools",
|
||||
"web_search",
|
||||
"customWidget_getSkill",
|
||||
"customWidget_validateTemplate",
|
||||
]);
|
||||
const initiallyActiveAssistantEvaluationTools = new Set(["web_search", "customWidget_getSkill"]);
|
||||
const maxFocusedComponentSearchesPerPhase = 4;
|
||||
const customWidgetContextToolBudgets: Readonly<Record<string, number>> = {
|
||||
customWidget_findComponents: maxFocusedComponentSearchesPerPhase,
|
||||
@@ -468,46 +452,6 @@ const customWidgetContextToolBudgets: Readonly<Record<string, number>> = {
|
||||
customWidget_getExample: 1,
|
||||
};
|
||||
|
||||
const normalizeToolSearchText = (value: string) =>
|
||||
value
|
||||
.replaceAll(/([a-z\d])([A-Z])/gu, "$1 $2")
|
||||
.toLowerCase()
|
||||
.replaceAll(/[^a-z\d]+/gu, " ")
|
||||
.trim();
|
||||
|
||||
const findAssistantEvaluationTools = (query: string) => {
|
||||
const terms = [
|
||||
...new Set(
|
||||
normalizeToolSearchText(query)
|
||||
.split(" ")
|
||||
.filter((term) => term.length > 2),
|
||||
),
|
||||
];
|
||||
const normalizedQuery = normalizeToolSearchText(query);
|
||||
const requestsCompleteCatalog =
|
||||
normalizedQuery.includes("all components") ||
|
||||
normalizedQuery.includes("full component catalog") ||
|
||||
normalizedQuery.includes("complete component catalog");
|
||||
return customWidgetAssistantEvaluationToolDefinitions
|
||||
.filter(({ function: definition }) => !initiallyActiveAssistantEvaluationTools.has(definition.name))
|
||||
.filter(({ function: definition }) => {
|
||||
if (definition.name === "customWidget_getComponentCatalog") return requestsCompleteCatalog;
|
||||
if (definition.name === "customWidget_schema") return false;
|
||||
return true;
|
||||
})
|
||||
.map((tool) => {
|
||||
const searchable = normalizeToolSearchText(`${tool.function.name} ${tool.function.description}`);
|
||||
const score = terms.reduce((total, term) => total + (searchable.includes(term) ? 1 : 0), 0);
|
||||
return { tool, score };
|
||||
})
|
||||
.filter(({ score }) => score > 0)
|
||||
.toSorted(
|
||||
(left, right) => right.score - left.score || left.tool.function.name.localeCompare(right.tool.function.name),
|
||||
)
|
||||
.slice(0, 8)
|
||||
.map(({ tool }) => tool);
|
||||
};
|
||||
|
||||
export const getActiveAssistantEvaluationToolDefinitions = (state: AssistantAttemptState) => {
|
||||
const availableNames = customWidgetAssistantEvaluationToolDefinitions.map(
|
||||
({ function: definition }) => definition.name,
|
||||
@@ -516,7 +460,7 @@ export const getActiveAssistantEvaluationToolDefinitions = (state: AssistantAtte
|
||||
toolResults: [{ toolName: toolCall.name, output: toolCall.output }],
|
||||
}));
|
||||
const phaseToolNames = getCustomWidgetPhaseToolNames(availableNames, steps);
|
||||
const activeToolNames = new Set(phaseToolNames ?? state.activatedTools);
|
||||
const activeToolNames = new Set(phaseToolNames ?? initiallyActiveAssistantEvaluationTools);
|
||||
return customWidgetAssistantEvaluationToolDefinitions.filter(({ function: definition }) =>
|
||||
activeToolNames.has(definition.name),
|
||||
);
|
||||
@@ -693,22 +637,6 @@ const executeAssistantEvaluationToolCore = (
|
||||
if (contextBudget !== undefined && getContextToolCallsInCurrentPhase(state, name) >= contextBudget) {
|
||||
return getContextPhaseCompleteOutput(name);
|
||||
}
|
||||
if (name === "homarr_findTools") {
|
||||
const query = typeof input.query === "string" ? input.query : "";
|
||||
const matches = findAssistantEvaluationTools(query);
|
||||
for (const match of matches) state.activatedTools.add(match.function.name);
|
||||
return {
|
||||
query,
|
||||
matches: matches.map(({ function: definition }) => ({
|
||||
name: definition.name,
|
||||
description: definition.description,
|
||||
})),
|
||||
nextStep:
|
||||
matches.length > 0
|
||||
? "Use the matching tools now. Search again only for a different capability."
|
||||
: "No matching Homarr tool was found. Refine the capability.",
|
||||
};
|
||||
}
|
||||
if (name === "web_search") {
|
||||
return {
|
||||
query: typeof input.query === "string" ? input.query : "",
|
||||
@@ -1067,7 +995,7 @@ export function getAssistantEvaluationEfficiencyIssues(
|
||||
if (skillCall && skillCall.outputCharacters > 10_000) {
|
||||
issues.push(`The compact skill entrypoint used ${skillCall.outputCharacters} output characters.`);
|
||||
}
|
||||
const otherReusableContextTools = new Set(["homarr_findTools", "web_search"]);
|
||||
const otherReusableContextTools = new Set(["web_search"]);
|
||||
const seenContextCalls = new Set<string>();
|
||||
for (const toolCall of state.toolCalls) {
|
||||
if (toolCall.phaseLimited) continue;
|
||||
@@ -1154,7 +1082,6 @@ export function createAssistantEvaluationState(): AssistantAttemptState {
|
||||
return {
|
||||
calledTools: [],
|
||||
toolCalls: [],
|
||||
activatedTools: new Set(initiallyActiveAssistantEvaluationTools),
|
||||
validatedTemplates: new Set(),
|
||||
previews: new Map(),
|
||||
completedPreviewSignatures: new Set(),
|
||||
@@ -1260,7 +1187,7 @@ async function runAssistantAttempt(args: {
|
||||
const messages: OpenRouterMessage[] = [
|
||||
{
|
||||
role: "system",
|
||||
content: `${CUSTOM_WIDGET_LAZY_TOOL_DISCOVERY_INSTRUCTION}\n\n${CUSTOM_WIDGET_ASSISTANT_POLICY}\n\nThis is an unassisted tool-use evaluation. No tool will be forced for you. Complete and persist all ${getExpectedWidgetCount(args.testCase)} requested widget jobs before returning prose.`,
|
||||
content: `${CUSTOM_WIDGET_TOOL_STAGING_INSTRUCTION}\n\n${CUSTOM_WIDGET_ASSISTANT_POLICY}\n\nThis is an unassisted tool-use evaluation. No tool will be forced for you. Complete and persist all ${getExpectedWidgetCount(args.testCase)} requested widget jobs before returning prose.`,
|
||||
},
|
||||
{ role: "user", content: buildAssistantPrompt(args.testCase, args.feedback) },
|
||||
];
|
||||
|
||||
@@ -110,8 +110,8 @@ Use customWidget_validateTemplate for focused JSX repair without resending the m
|
||||
|
||||
export const CUSTOM_WIDGET_AUTHORING_PROMPT = AUTHORING_PROMPT;
|
||||
|
||||
export const CUSTOM_WIDGET_LAZY_TOOL_DISCOVERY_INSTRUCTION =
|
||||
"Use homarr_findTools for task-needed component, reference, lifecycle, configuration, or placement tools; external MCP keeps live schema and Assistant context compact.";
|
||||
export const CUSTOM_WIDGET_TOOL_STAGING_INSTRUCTION =
|
||||
"Custom Widget tools are staged by the authoring lifecycle. Use only visible task-needed tools; successful phases expose the next typed tools without loading the full catalog.";
|
||||
|
||||
export const CUSTOM_WIDGET_ASSISTANT_POLICY = `Custom Widget work:
|
||||
- Use customWidget tools; never substitute prose.
|
||||
|
||||
@@ -31,6 +31,15 @@ const customWidgetDraftPhaseToolNames = new Set([
|
||||
"customWidget_getSharedProps",
|
||||
"customWidget_validateTemplate",
|
||||
]);
|
||||
const customWidgetContextPhaseToolNames = new Set([
|
||||
"customWidget_getReference",
|
||||
"customWidget_findComponents",
|
||||
"customWidget_getComponents",
|
||||
"customWidget_getComponent",
|
||||
"customWidget_getSharedProps",
|
||||
"customWidget_getExample",
|
||||
"customWidget_validateTemplate",
|
||||
]);
|
||||
const maxFocusedComponentSearchesPerPhase = 4;
|
||||
|
||||
export interface CustomWidgetToolStep {
|
||||
@@ -182,5 +191,11 @@ export const getCustomWidgetPhaseToolNames = <TToolName extends string>(
|
||||
if (focusedSearches >= maxFocusedComponentSearchesPerPhase) {
|
||||
return availableToolNames.filter((toolName) => customWidgetDraftPhaseToolNames.has(toolName));
|
||||
}
|
||||
const skillLoaded = steps.some((step) =>
|
||||
step.toolResults.some((result) => result.toolName === "customWidget_getSkill"),
|
||||
);
|
||||
if (skillLoaded) {
|
||||
return availableToolNames.filter((toolName) => customWidgetContextPhaseToolNames.has(toolName));
|
||||
}
|
||||
return null;
|
||||
};
|
||||
@@ -336,50 +336,38 @@ describe("Custom Widget assistant live evaluation harness", () => {
|
||||
expect(names).not.toContain("customWidget_validate");
|
||||
});
|
||||
|
||||
it("starts with only the authoring bootstrap and discovers later lifecycle tools on demand", () => {
|
||||
it("starts with compact authoring context tools and stages later lifecycle tools by phase", () => {
|
||||
const testCase = getCase("seerr-media-workflows");
|
||||
const state = createAssistantEvaluationState();
|
||||
const getActiveNames = () =>
|
||||
getActiveAssistantEvaluationToolDefinitions(state).map(({ function: definition }) => definition.name);
|
||||
|
||||
expect(getActiveNames()).toEqual(["web_search", "customWidget_getSkill"]);
|
||||
executeAssistantEvaluationTool(testCase, state, "customWidget_getSkill", {});
|
||||
expect(getActiveNames()).toEqual([
|
||||
"homarr_findTools",
|
||||
"web_search",
|
||||
"customWidget_getSkill",
|
||||
"customWidget_getReference",
|
||||
"customWidget_findComponents",
|
||||
"customWidget_getComponents",
|
||||
"customWidget_getComponent",
|
||||
"customWidget_getSharedProps",
|
||||
"customWidget_getExample",
|
||||
"customWidget_validateTemplate",
|
||||
]);
|
||||
expect(
|
||||
executeAssistantEvaluationTool(testCase, state, "homarr_findTools", {
|
||||
query: "preview create revise template query action persist journal",
|
||||
}),
|
||||
).toMatchObject({
|
||||
matches: expect.arrayContaining([
|
||||
expect.objectContaining({ name: "customWidget_previewCreate" }),
|
||||
expect.objectContaining({ name: "customWidget_previewReviseTemplate" }),
|
||||
expect.objectContaining({ name: "customWidget_previewQuery" }),
|
||||
expect.objectContaining({ name: "customWidget_previewAction" }),
|
||||
expect.objectContaining({ name: "customWidget_createFromPreview" }),
|
||||
]),
|
||||
});
|
||||
expect(getActiveNames()).toEqual(
|
||||
expect.arrayContaining([
|
||||
"customWidget_previewCreate",
|
||||
"customWidget_previewReviseTemplate",
|
||||
"customWidget_previewQuery",
|
||||
"customWidget_previewAction",
|
||||
"customWidget_createFromPreview",
|
||||
]),
|
||||
expect(getActiveNames()).not.toEqual(
|
||||
expect.arrayContaining(["customWidget_previewCreate", "customWidget_createFromPreview"]),
|
||||
);
|
||||
expect(
|
||||
executeAssistantEvaluationTool(testCase, state, "homarr_findTools", {
|
||||
query: "customWidget_findComponents focused component search and selected component documentation batch",
|
||||
}),
|
||||
).toMatchObject({
|
||||
matches: expect.arrayContaining([
|
||||
expect.objectContaining({ name: "customWidget_findComponents" }),
|
||||
expect.objectContaining({ name: "customWidget_getComponents" }),
|
||||
]),
|
||||
|
||||
executeAssistantEvaluationTool(testCase, state, "customWidget_getComponents", {
|
||||
names: ["Stack", "TextInput", "Button"],
|
||||
});
|
||||
expect(getActiveNames()).toEqual(["customWidget_getReference", "customWidget_validateTemplate"]);
|
||||
|
||||
validateTemplate(testCase, state, mediaResearchWidget);
|
||||
expect(getActiveNames()).toEqual([
|
||||
"customWidget_validateTemplate",
|
||||
"customWidget_previewCreate",
|
||||
"customWidget_previewReviseTemplate",
|
||||
]);
|
||||
});
|
||||
|
||||
it("compacts obsolete documentation and validation calls but retains the current preview", () => {
|
||||
@@ -855,9 +843,7 @@ describe("Custom Widget assistant live evaluation harness", () => {
|
||||
const testCase = getCase("fake-service-health");
|
||||
const state = createAssistantEvaluationState();
|
||||
|
||||
executeAssistantEvaluationTool(testCase, state, "homarr_findTools", {
|
||||
query: "focused component search and selected component documentation batch",
|
||||
});
|
||||
executeAssistantEvaluationTool(testCase, state, "customWidget_getSkill", {});
|
||||
executeAssistantEvaluationTool(testCase, state, "customWidget_findComponents", {
|
||||
query: "health status layout",
|
||||
});
|
||||
|
||||
@@ -5,7 +5,7 @@ import {
|
||||
buildCustomWidgetMcpPrompt,
|
||||
CUSTOM_WIDGET_ASSISTANT_POLICY,
|
||||
CUSTOM_WIDGET_FINAL_OUTPUT_INSTRUCTION,
|
||||
CUSTOM_WIDGET_LAZY_TOOL_DISCOVERY_INSTRUCTION,
|
||||
CUSTOM_WIDGET_TOOL_STAGING_INSTRUCTION,
|
||||
CUSTOM_WIDGET_MCP_AUTHORING_PROMPT,
|
||||
} from "../core/ai-prompt";
|
||||
|
||||
@@ -54,8 +54,8 @@ describe("AI prompt", () => {
|
||||
expect(CUSTOM_WIDGET_ASSISTANT_POLICY).toContain("change the active phase");
|
||||
expect(CUSTOM_WIDGET_ASSISTANT_POLICY).toContain("customWidget_getComponents");
|
||||
expect(CUSTOM_WIDGET_ASSISTANT_POLICY).toContain("Do not load the full catalog");
|
||||
expect(CUSTOM_WIDGET_LAZY_TOOL_DISCOVERY_INSTRUCTION).toContain("homarr_findTools");
|
||||
expect(CUSTOM_WIDGET_LAZY_TOOL_DISCOVERY_INSTRUCTION).toContain("task-needed");
|
||||
expect(CUSTOM_WIDGET_TOOL_STAGING_INSTRUCTION).toContain("staged by the authoring lifecycle");
|
||||
expect(CUSTOM_WIDGET_TOOL_STAGING_INSTRUCTION).toContain("task-needed");
|
||||
expect(CUSTOM_WIDGET_ASSISTANT_POLICY).toContain("Reuse loaded context");
|
||||
expect(CUSTOM_WIDGET_ASSISTANT_POLICY).toContain("no arbitrary documentation or creativity cap");
|
||||
expect(CUSTOM_WIDGET_ASSISTANT_POLICY).toContain("one focused component search per widget job");
|
||||
|
||||
Reference in new issue
Block a user