enhance assistant batch responses
This commit is contained in:
1 parent
edf3f6cd84
commit
8d06201b41
10 files changed
+207
-38
No files matched your search
@@ -103,6 +103,9 @@ Files returned by a compatible model can be downloaded directly from the message
|
||||
Assistant responses render GitHub-flavored Markdown, including headings, links, lists, task lists, tables,
|
||||
quotes, inline code, and fenced code blocks. Line breaks are preserved, and Homarr repairs common streamed
|
||||
responses where a provider returns escaped line breaks or places a bold-label list on one line.
|
||||
When Homarr's icon search returns an image URL, visual result tables render that actual icon in a compact
|
||||
image column instead of substituting an emoji. Assistant-provided Markdown images accept safe relative,
|
||||
HTTP, and HTTPS sources and load without forwarding the Homarr page as the referrer.
|
||||
Images up to 1 MB and text, Markdown, CSV, HTML, XML, or JSON documents up to 350 KB can be attached by
|
||||
using the paperclip or dropping files onto the composer. A request can contain up to five attachments.
|
||||
Image understanding depends on the selected model; document text is extracted locally by Homarr and sent
|
||||
@@ -143,6 +146,14 @@ Common collection and widget results are summarized inline as compact responsive
|
||||
structured result remains available in the tool card details. Icon searches show the returned image variants,
|
||||
file formats, and repositories instead of the size of the complete icon catalog.
|
||||
|
||||
Large tool-driven requests can use up to 32 agent steps and a larger response budget within one request, so
|
||||
multi-item dashboard changes normally finish without requiring a follow-up “Continue” message. Independent
|
||||
read-only lookups may be performed together, while every mutation still pauses for its normal approval.
|
||||
|
||||
The assistant can create a formatted dashboard note by adding a **Notebook** widget. Ask it for a note with
|
||||
headings, lists, tasks, links, emphasis, tables, or images; it generates compatible rich-text content, shows
|
||||
the standard mutation approval, and places the widget in the next available dashboard position.
|
||||
|
||||
Custom-widget tools let an administrator list, inspect, create, and update definitions. The custom-widget
|
||||
form's **Design with AI** action sends a focused request into the current conversation, where the assistant
|
||||
can propose the matching mutation and use the same approval controls. Existing stored secret values are
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
import { describe, expect, test } from "vitest";
|
||||
|
||||
import { assistantExecutionPolicy } from "./assistant-execution-policy";
|
||||
|
||||
describe("assistantExecutionPolicy", () => {
|
||||
test("allows a sizeable batch of tool calls to finish in one request", () => {
|
||||
expect(assistantExecutionPolicy.maxSteps).toBeGreaterThanOrEqual(32);
|
||||
expect(assistantExecutionPolicy.maxOutputTokens).toBeGreaterThanOrEqual(8_192);
|
||||
expect(assistantExecutionPolicy.totalTimeoutMs).toBeGreaterThan(assistantExecutionPolicy.stepTimeoutMs);
|
||||
expect(assistantExecutionPolicy.toolTimeoutMs).toBeGreaterThanOrEqual(60_000);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,7 @@
|
||||
export const assistantExecutionPolicy = {
|
||||
maxSteps: 32,
|
||||
maxOutputTokens: 8_192,
|
||||
totalTimeoutMs: 240_000,
|
||||
stepTimeoutMs: 60_000,
|
||||
toolTimeoutMs: 60_000,
|
||||
} as const;
|
||||
@@ -32,10 +32,11 @@ import type {
|
||||
import { extractMcpTools } from "../../mcp/_extract-tools";
|
||||
import { getRequestedMentionIds, sanitizeAttachmentFilename } from "./assistant-chat-input";
|
||||
import { getAssistantModelLookupStatus } from "./assistant-model-lookup";
|
||||
import { assistantExecutionPolicy } from "./assistant-execution-policy";
|
||||
import { getAssistantStreamErrorMessage } from "./assistant-stream-error";
|
||||
import { getForcedAssistantToolName, withAssistantToolPolicy } from "./assistant-tool-policy";
|
||||
|
||||
export const maxDuration = 60;
|
||||
export const maxDuration = 300;
|
||||
|
||||
const logger = createLogger({ module: "assistant" });
|
||||
const getToolApprovalSecret = () =>
|
||||
@@ -98,9 +99,12 @@ Action rules:
|
||||
- Mutating Homarr tools already pause for native user approval. Calling a mutation only proposes the action; it cannot execute until the user selects Approve and run. Once the requested change is sufficiently specified, call the mutation immediately so the approval UI appears.
|
||||
- A prose response such as "Please confirm if you would like me to proceed", "Would you like me to proceed?", or a parameter summary that asks for confirmation is incorrect. Never ask for a second textual confirmation before an approval-gated tool call.
|
||||
- Do not retry a denied action.
|
||||
- Before creating an app, call configure_app with the app name and every useful default you can infer, including its description, href, pingUrl, and an icon URL returned by icon_findIcons. The user reviews Homarr's native app form. Its icon picker searches Homarr's local icon repository. Use the returned values for app_create.
|
||||
- Before creating an app, call configure_app with the app name and every useful default you can infer, including its description, href, pingUrl, and an icon URL returned by icon_findIcons. The user reviews Homarr's native app form. Its icon picker searches Homarr's local icon repository. Use the returned values for app_create. If the user asked to place it on a board, use the appId returned by app_create in a board_addItem call with kind "app" and options { appId } before reporting success.
|
||||
- Before changing a board's custom CSS or visual and behavior settings, call board_getBoardSettings to read the current values, then call configure_board_settings with only the requested proposed changes. The user can edit the complete CSS and settings in Homarr's native form. Use its returned flat object for board_savePartialBoardSettings, which then shows the mutation approval. Never overwrite existing CSS rules unless the user explicitly requests replacement.
|
||||
- To create a formatted note on a dashboard, call board_addItem with kind "notebook". Put valid Tiptap-compatible HTML in options.content and set options.showToolbar and options.allowReadOnlyCheck when useful. Use semantic paragraphs, headings, lists, task lists, blockquotes, tables, links, emphasis, and images as appropriate. Never include scripts, styles, iframes, event handlers, or unsafe URL protocols.
|
||||
- When choosing an app icon without configure_app, call the Homarr icon findIcons tool first and use one of its returned local icon URLs. Never invent a third-party icon CDN URL.
|
||||
- When a Homarr tool returns a usable image or icon URL and a visual summary helps, embed that exact URL with Markdown image syntax. App summary tables must use an Icon column such as . Never replace an available returned icon with emoji, and never invent or transform image URLs.
|
||||
- Complete batch requests in the same run. Use parallel independent read-only calls when supported, reuse results, continue calling tools until every requested item has been attempted, and summarize only after the batch is complete. Do not stop after a partial batch or ask the user to type "Continue".
|
||||
- Browser tools can navigate within Homarr, open existing Homarr UI, or refresh the active view after a completed change. Never navigate to an arbitrary external URL and never refresh before a mutation finishes.
|
||||
- Keep responses concise and lead with the result. Summarize tool output instead of dumping JSON.
|
||||
- Use well-formed GitHub-flavored Markdown. Put each list item on its own line and leave a blank line before lists.
|
||||
@@ -516,9 +520,14 @@ export async function POST(request: Request) {
|
||||
toolChoice: { type: "tool", toolName: forcedToolName },
|
||||
};
|
||||
},
|
||||
stopWhen: stepCountIs(8),
|
||||
stopWhen: stepCountIs(assistantExecutionPolicy.maxSteps),
|
||||
abortSignal: request.signal,
|
||||
timeout: { totalMs: 55_000, stepMs: 30_000, toolMs: 30_000 },
|
||||
timeout: {
|
||||
totalMs: assistantExecutionPolicy.totalTimeoutMs,
|
||||
stepMs: assistantExecutionPolicy.stepTimeoutMs,
|
||||
toolMs: assistantExecutionPolicy.toolTimeoutMs,
|
||||
},
|
||||
maxOutputTokens: assistantExecutionPolicy.maxOutputTokens,
|
||||
maxRetries: 2,
|
||||
reasoning: parsed.data.reasoning === "auto" ? undefined : parsed.data.reasoning,
|
||||
providerOptions:
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
import { describe, expect, test } from "vitest";
|
||||
|
||||
import { getSafeAssistantMarkdownImageSource } from "./assistant-markdown-image";
|
||||
|
||||
describe("getSafeAssistantMarkdownImageSource", () => {
|
||||
test("allows Homarr-relative and ordinary remote images", () => {
|
||||
expect(getSafeAssistantMarkdownImageSource("/logo/logo.png")).toBe("/logo/logo.png");
|
||||
expect(getSafeAssistantMarkdownImageSource("https://cdn.example.com/icons/discord.svg")).toBe(
|
||||
"https://cdn.example.com/icons/discord.svg",
|
||||
);
|
||||
});
|
||||
|
||||
test("rejects unsafe protocols, protocol-relative URLs, and embedded credentials", () => {
|
||||
expect(getSafeAssistantMarkdownImageSource("javascript:alert(1)")).toBeNull();
|
||||
expect(getSafeAssistantMarkdownImageSource("data:image/svg+xml;base64,PHN2Zy8+")).toBeNull();
|
||||
expect(getSafeAssistantMarkdownImageSource("//example.com/icon.svg")).toBeNull();
|
||||
expect(getSafeAssistantMarkdownImageSource("https://user:secret@example.com/icon.svg")).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,12 @@
|
||||
export const getSafeAssistantMarkdownImageSource = (source: string | Blob | undefined) => {
|
||||
if (typeof source !== "string" || !source) return null;
|
||||
if (source.startsWith("/") && !source.startsWith("//")) return source;
|
||||
|
||||
try {
|
||||
const url = new URL(source);
|
||||
if ((url.protocol !== "http:" && url.protocol !== "https:") || url.username || url.password) return null;
|
||||
return url.toString();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
@@ -466,11 +466,13 @@
|
||||
border-top: 1px solid var(--mantine-color-default-border);
|
||||
}
|
||||
|
||||
.messageMarkdown img {
|
||||
.markdownImage {
|
||||
display: block;
|
||||
max-width: 100%;
|
||||
max-height: 24rem;
|
||||
height: auto;
|
||||
margin-block: 0.65rem;
|
||||
object-fit: contain;
|
||||
border-radius: var(--mantine-radius-md);
|
||||
}
|
||||
|
||||
@@ -511,6 +513,13 @@
|
||||
border-bottom: 0;
|
||||
}
|
||||
|
||||
.markdownTableWrap .markdownImage {
|
||||
width: 2rem;
|
||||
height: 2rem;
|
||||
margin: 0 auto;
|
||||
border-radius: var(--mantine-radius-sm);
|
||||
}
|
||||
|
||||
.messageQuote {
|
||||
margin: 0 0 0.55rem;
|
||||
padding-inline-start: 0.65rem;
|
||||
|
||||
@@ -104,6 +104,7 @@ import { useScopedI18n } from "@homarr/translation/client";
|
||||
import classes from "./assistant-panel.module.css";
|
||||
import { useAssistantAutoApproval, useAssistantAutomaticAction } from "./assistant-auto-approval";
|
||||
import { normalizeAssistantMarkdown } from "./assistant-markdown";
|
||||
import { getSafeAssistantMarkdownImageSource } from "./assistant-markdown-image";
|
||||
import { getAssistantTelemetry, getAssistantUsage } from "./assistant-message-metadata";
|
||||
import type { AssistantPendingAction } from "./assistant-pending-action";
|
||||
import { AssistantQuestionPortalProvider } from "./assistant-question-portal";
|
||||
@@ -153,12 +154,29 @@ const MarkdownTable = ({ children, ...props }: ComponentPropsWithoutRef<"table">
|
||||
</div>
|
||||
);
|
||||
|
||||
const MarkdownImage = ({ src, alt = "", ...props }: ComponentPropsWithoutRef<"img">) => {
|
||||
const safeSource = getSafeAssistantMarkdownImageSource(src);
|
||||
if (!safeSource) return null;
|
||||
|
||||
return (
|
||||
<img
|
||||
{...props}
|
||||
src={safeSource}
|
||||
alt={alt}
|
||||
className={classes.markdownImage}
|
||||
loading="lazy"
|
||||
decoding="async"
|
||||
referrerPolicy="no-referrer"
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
const AssistantTextPart = () => (
|
||||
<MarkdownTextPrimitive
|
||||
className={classes.messageMarkdown}
|
||||
preprocess={normalizeAssistantMarkdown}
|
||||
remarkPlugins={markdownRemarkPlugins}
|
||||
components={{ a: MarkdownLink, table: MarkdownTable }}
|
||||
components={{ a: MarkdownLink, img: MarkdownImage, table: MarkdownTable }}
|
||||
defer
|
||||
/>
|
||||
);
|
||||
|
||||
@@ -10,47 +10,50 @@ export const iconsRouter = createTRPCRouter({
|
||||
mcp: {
|
||||
enabled: true,
|
||||
description:
|
||||
"Search for icons by name across all icon repositories. OPTIONAL: searchText (string to filter), limitPerGroup (number 1-500, default 12). Call with no arguments to browse all icons",
|
||||
"Search for icons by name across all icon repositories. Returns matching icon names and exact image URLs; use those URLs directly for app icons and Markdown image previews. OPTIONAL: searchText (string to filter), limitPerGroup (number 1-500, default 12). Call with no arguments to browse all icons",
|
||||
},
|
||||
})
|
||||
.input(iconsFindSchema)
|
||||
.query(async ({ ctx, input }) => {
|
||||
const term = normalizeImageName(input.searchText ?? "");
|
||||
const repositories = await ctx.db.query.iconRepositories.findMany({
|
||||
orderBy: (table, { asc, sql }) => [sql`CASE WHEN ${table.slug} = 'local' THEN 0 ELSE 1 END`, asc(table.slug)],
|
||||
with: {
|
||||
icons: {
|
||||
columns: { id: true, name: true, url: true },
|
||||
orderBy: (table, { asc, sql }) => [
|
||||
sql`CASE WHEN ${table.name} LIKE '%.svg' THEN 0 ELSE 1 END`,
|
||||
asc(table.name),
|
||||
],
|
||||
limit: term.length === 0 ? input.limitPerGroup : undefined,
|
||||
const [repositories, countIcons] = await Promise.all([
|
||||
ctx.db.query.iconRepositories.findMany({
|
||||
orderBy: (table, { asc, sql }) => [sql`CASE WHEN ${table.slug} = 'local' THEN 0 ELSE 1 END`, asc(table.slug)],
|
||||
with: {
|
||||
icons: {
|
||||
columns: { id: true, name: true, url: true },
|
||||
orderBy: (table, { asc, sql }) => [
|
||||
sql`CASE WHEN ${table.name} LIKE '%.svg' THEN 0 ELSE 1 END`,
|
||||
asc(table.name),
|
||||
],
|
||||
limit: term.length === 0 ? input.limitPerGroup : undefined,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
}),
|
||||
ctx.db.$count(icons),
|
||||
]);
|
||||
|
||||
const matchedRepositories = repositories.map((repository) => ({
|
||||
...repository,
|
||||
icons: repository.icons
|
||||
.flatMap((icon) => {
|
||||
const rank = getImageMatchRank(term, icon.name || icon.url);
|
||||
return rank === null ? [] : [{ icon, rank }];
|
||||
})
|
||||
.toSorted((a, b) =>
|
||||
a.rank !== b.rank
|
||||
? a.rank - b.rank
|
||||
: Number(b.icon.name.toLowerCase().endsWith(".svg")) -
|
||||
Number(a.icon.name.toLowerCase().endsWith(".svg")) || a.icon.name.localeCompare(b.icon.name),
|
||||
)
|
||||
.slice(0, input.limitPerGroup)
|
||||
.map(({ icon }) => icon),
|
||||
}));
|
||||
|
||||
return {
|
||||
icons:
|
||||
term.length === 0
|
||||
? repositories
|
||||
: repositories.map((repository) => ({
|
||||
...repository,
|
||||
icons: repository.icons
|
||||
.flatMap((icon) => {
|
||||
const rank = getImageMatchRank(term, icon.name || icon.url);
|
||||
return rank === null ? [] : [{ icon, rank }];
|
||||
})
|
||||
.toSorted((a, b) =>
|
||||
a.rank !== b.rank
|
||||
? a.rank - b.rank
|
||||
: Number(b.icon.name.toLowerCase().endsWith(".svg")) -
|
||||
Number(a.icon.name.toLowerCase().endsWith(".svg")) || a.icon.name.localeCompare(b.icon.name),
|
||||
)
|
||||
.slice(0, input.limitPerGroup)
|
||||
.map(({ icon }) => icon),
|
||||
})),
|
||||
countIcons: await ctx.db.$count(icons),
|
||||
term.length === 0 ? repositories : matchedRepositories.filter((repository) => repository.icons.length > 0),
|
||||
countIcons,
|
||||
};
|
||||
}),
|
||||
});
|
||||
@@ -1,3 +1,5 @@
|
||||
// @vitest-environment node
|
||||
|
||||
import { describe, expect, test } from "vitest";
|
||||
|
||||
import { iconRepositories, icons } from "@homarr/db/schema";
|
||||
@@ -6,6 +8,73 @@ import { createDb } from "@homarr/db/test";
|
||||
import { iconsRouter } from "../icons";
|
||||
|
||||
describe("findIcons", () => {
|
||||
test("returns only repositories containing matches for a search", async () => {
|
||||
const db = createDb();
|
||||
await db.insert(iconRepositories).values([
|
||||
{ id: "dashboard-icons", slug: "homarr-labs/dashboard-icons" },
|
||||
{ id: "other-icons", slug: "example/other-icons" },
|
||||
]);
|
||||
await db.insert(icons).values([
|
||||
{
|
||||
id: "discord-svg",
|
||||
name: "discord.svg",
|
||||
url: "https://cdn.example.com/discord.svg",
|
||||
checksum: "discord",
|
||||
iconRepositoryId: "dashboard-icons",
|
||||
},
|
||||
{
|
||||
id: "homarr-svg",
|
||||
name: "homarr.svg",
|
||||
url: "https://cdn.example.com/homarr.svg",
|
||||
checksum: "homarr",
|
||||
iconRepositoryId: "other-icons",
|
||||
},
|
||||
]);
|
||||
const caller = iconsRouter.createCaller({ db, deviceType: undefined, session: null });
|
||||
|
||||
const result = await caller.findIcons({ searchText: "discord", limitPerGroup: 6 });
|
||||
|
||||
expect(result.countIcons).toBe(2);
|
||||
expect(result.icons).toHaveLength(1);
|
||||
expect(result.icons[0]).toMatchObject({
|
||||
slug: "homarr-labs/dashboard-icons",
|
||||
icons: [{ name: "discord.svg", url: "https://cdn.example.com/discord.svg" }],
|
||||
});
|
||||
});
|
||||
|
||||
test("keeps repositories when browsing without a search", async () => {
|
||||
const db = createDb();
|
||||
await db.insert(iconRepositories).values({ id: "empty", slug: "example/empty" });
|
||||
const caller = iconsRouter.createCaller({ db, deviceType: undefined, session: null });
|
||||
|
||||
const result = await caller.findIcons({ limitPerGroup: 6 });
|
||||
|
||||
expect(result.icons).toMatchObject([{ slug: "example/empty", icons: [] }]);
|
||||
});
|
||||
|
||||
test("finds the Homarr icon and returns its exact local repository URL", async () => {
|
||||
const db = createDb();
|
||||
await db.insert(iconRepositories).values({ id: "dashboard-icons", slug: "homarr-labs/dashboard-icons" });
|
||||
await db.insert(icons).values({
|
||||
id: "homarr-svg",
|
||||
name: "homarr.svg",
|
||||
url: "https://cdn.example.com/homarr.svg",
|
||||
checksum: "homarr",
|
||||
iconRepositoryId: "dashboard-icons",
|
||||
});
|
||||
const caller = iconsRouter.createCaller({ db, deviceType: undefined, session: null });
|
||||
|
||||
await expect(caller.findIcons({ searchText: "homarr" })).resolves.toMatchObject({
|
||||
countIcons: 1,
|
||||
icons: [
|
||||
{
|
||||
slug: "homarr-labs/dashboard-icons",
|
||||
icons: [{ name: "homarr.svg", url: "https://cdn.example.com/homarr.svg" }],
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
test.each(["homeassistant", "home_assistant", "home assistant.png"])(
|
||||
"matches normalized search %s before applying the group limit",
|
||||
async (searchText) => {
|
||||
|
||||
Reference in new issue
Block a user