Harden Homarr provider integration

This commit is contained in:
ajnart committed 2026-08-11 19:47:11 +02:00
1 parent 165201632d
commit 4ebaea29fd
23 files changed
+427 -159

No files matched your search

+1 -1
View File
@@ -480,7 +480,7 @@ export const assistantRouter = createTRPCRouter({
const configuration = await getConfigurationAsync(ctx.db);
if (isDemoMode) {
return {
provider: "homarr",
provider: "custom" as const,
defaultModelId: "homarr-assistant",
models: [
{
@@ -121,6 +121,30 @@ describe("assistantRouter.updateConnection", () => {
modelId: "example/model",
});
});
test("never stores administrator credentials for the Homarr provider", async () => {
const db = await createConfiguredAssistantAsync();
const caller = assistantRouter.createCaller({ db, deviceType: undefined, session: adminSession });
await caller.updateConnection({
provider: "homarr",
baseUrl: "https://homarr.dev/api/ai/v1",
modelDiscoveryPath: "/models",
apiKey: "must-not-be-stored",
customHeaders: { "X-Secret": "must-not-be-stored" },
clearApiKey: false,
clearCustomHeaders: false,
});
const [configuration] = await db.select().from(assistantConfigurations);
expect(configuration).toMatchObject({
provider: "homarr",
encryptedApiKey: null,
encryptedHeaders: null,
enabled: false,
modelId: null,
});
});
});
describe("assistantRouter.updateConfiguration", () => {
@@ -170,6 +194,49 @@ describe("assistantRouter.updateConfiguration", () => {
expect(configuration?.webSearchEnabled).toBe(true);
});
test("allows the Homarr OpenRouter proxy to enable web search", async () => {
const db = await createConfiguredAssistantAsync();
await db
.update(assistantConfigurations)
.set({
provider: "homarr",
baseUrl: "https://homarr.dev/api/ai/v1",
encryptedApiKey: null,
encryptedHeaders: null,
modelId: "homarr/deepseek-v4-flash-latest",
})
.where(eq(assistantConfigurations.id, "default"));
const caller = assistantRouter.createCaller({ db, deviceType: undefined, session: adminSession });
vi.stubGlobal(
"fetch",
vi.fn().mockResolvedValue(
Response.json({
data: [
{
id: "homarr/deepseek-v4-flash-latest",
name: "DeepSeek V4 Flash Latest",
supported_parameters: ["tools"],
architecture: { output_modalities: ["text"] },
},
],
}),
),
);
try {
await caller.updateConfiguration({
enabled: true,
modelId: "homarr/deepseek-v4-flash-latest",
webSearchEnabled: true,
});
} finally {
vi.unstubAllGlobals();
}
const [configuration] = await db.select().from(assistantConfigurations);
expect(configuration?.webSearchEnabled).toBe(true);
});
test("rejects OpenRouter server tools for an incompatible provider", async () => {
const db = await createConfiguredAssistantAsync();
await db
@@ -21,6 +21,7 @@ import { getCustomWidgetJsonSchema } from "../src/core/schema";
import type { CustomWidgetAiEvaluationCase } from "./ai-evaluation-cases";
import {
DEFAULT_GENERATOR_MODEL,
getAiProviderChatCompletionsUrl,
getDeterministicEvaluationIssues,
judgePasses,
parseJudgeResult,
@@ -441,12 +442,13 @@ function buildAssistantPrompt(testCase: CustomWidgetAiEvaluationCase, feedback:
async function callAssistantStep(args: {
apiKey: string;
baseUrl?: string;
model: string;
messages: OpenRouterMessage[];
tools: ToolDefinition[];
toolChoice?: string;
}): Promise<OpenRouterAssistantMessage> {
const response = await fetch("https://openrouter.ai/api/v1/chat/completions", {
const response = await fetch(getAiProviderChatCompletionsUrl(args.baseUrl), {
method: "POST",
headers: {
Authorization: `Bearer ${args.apiKey}`,
@@ -467,10 +469,10 @@ async function callAssistantStep(args: {
});
const payload = (await response.json()) as OpenRouterResponse;
if (!response.ok) {
throw new Error(`OpenRouter request failed (${response.status}): ${payload.error?.message ?? "Unknown error"}`);
throw new Error(`AI provider request failed (${response.status}): ${payload.error?.message ?? "Unknown error"}`);
}
const message = payload.choices?.[0]?.message;
if (!message) throw new Error("OpenRouter returned no assistant message");
if (!message) throw new Error("AI provider returned no assistant message");
return message;
}
@@ -498,6 +500,7 @@ export const getAssistantEvaluationRecoveryToolName = (state: AssistantAttemptSt
async function callAssistantStepWithRecovery(args: {
apiKey: string;
baseUrl?: string;
model: string;
messages: OpenRouterMessage[];
tools: ToolDefinition[];
@@ -530,6 +533,7 @@ async function callAssistantStepWithRecovery(args: {
async function runAssistantAttempt(args: {
testCase: CustomWidgetAiEvaluationCase;
apiKey: string;
baseUrl?: string;
model: string;
feedback: readonly string[];
}) {
@@ -553,6 +557,7 @@ async function runAssistantAttempt(args: {
: customWidgetAssistantEvaluationToolDefinitions;
const message = await callAssistantStepWithRecovery({
apiKey: args.apiKey,
baseUrl: args.baseUrl,
model: args.model,
messages,
tools,
@@ -592,6 +597,7 @@ async function runAssistantAttempt(args: {
export async function evaluateCustomWidgetAssistantCase(args: {
testCase: CustomWidgetAiEvaluationCase;
apiKey: string;
baseUrl?: string;
outputRoot: string;
maxLoops: number;
generatorModel?: string;
@@ -612,6 +618,7 @@ export async function evaluateCustomWidgetAssistantCase(args: {
run = await runAssistantAttempt({
testCase: args.testCase,
apiKey: args.apiKey,
baseUrl: args.baseUrl,
model: args.generatorModel ?? DEFAULT_GENERATOR_MODEL,
feedback,
});
@@ -651,6 +658,7 @@ export async function evaluateCustomWidgetAssistantCase(args: {
testCase: args.testCase,
widget: run.state.createdWidget,
apiKey: args.apiKey,
baseUrl: args.baseUrl,
judgeModel: args.judgeModel,
});
await writeFile(path.join(caseDirectory, `judge-${attempt}.json`), judgeRaw, "utf8");
@@ -14,7 +14,24 @@ import type { CustomWidgetAiEvaluationCase } from "./ai-evaluation-cases";
// deepseek/deepseek-v4-flash-latest alias is rejected by the chat-completions API.
export const DEFAULT_GENERATOR_MODEL = "~deepseek/deepseek-v4-flash-latest";
export const DEFAULT_JUDGE_MODEL = "~deepseek/deepseek-v4-flash-latest";
export const DEFAULT_AI_PROVIDER_BASE_URL = "https://openrouter.ai/api/v1";
export const MAX_AI_EVALUATION_LOOPS = 10;
export const getAiProviderChatCompletionsUrl = (baseUrl = DEFAULT_AI_PROVIDER_BASE_URL) =>
`${baseUrl.replace(/\/+$/u, "")}/chat/completions`;
export const resolveAiEvaluationProviderConfig = (environment: Record<string, string | undefined>) => {
const baseUrl = (environment.AI_PROVIDER_BASE_URL ?? DEFAULT_AI_PROVIDER_BASE_URL).replace(/\/+$/u, "");
const homarrProvider = baseUrl.endsWith("/api/ai/v1");
const providerDefaultModel = homarrProvider ? "homarr/deepseek-v4-flash-latest" : DEFAULT_GENERATOR_MODEL;
return {
apiKey: environment.AI_PROVIDER_API_KEY ?? environment.OPENROUTER_API_KEY,
baseUrl,
generatorModel: environment.AI_PROVIDER_MODEL ?? environment.OPENROUTER_GENERATOR_MODEL ?? providerDefaultModel,
judgeModel:
environment.AI_PROVIDER_JUDGE_MODEL ??
environment.OPENROUTER_JUDGE_MODEL ??
(homarrProvider ? providerDefaultModel : DEFAULT_JUDGE_MODEL),
};
};
const scoreSchema = z.number().int().min(0).max(100);
const judgeCategoriesSchema = z.strictObject({
@@ -263,10 +280,12 @@ export async function requestCustomWidgetJudge(args: {
testCase: CustomWidgetAiEvaluationCase;
widget: HomarrCustomWidgetV2;
apiKey: string;
baseUrl?: string;
judgeModel?: string;
}) {
return callOpenRouter({
apiKey: args.apiKey,
baseUrl: args.baseUrl,
model: args.judgeModel ?? DEFAULT_JUDGE_MODEL,
prompt: buildJudgePrompt(args.testCase, args.widget),
purpose: "judge",
@@ -277,6 +296,7 @@ export async function judgeCustomWidgetCase(args: {
testCase: CustomWidgetAiEvaluationCase;
widget: HomarrCustomWidgetV2;
apiKey: string;
baseUrl?: string;
judgeModel?: string;
}) {
const raw = await requestCustomWidgetJudge(args);
@@ -286,6 +306,7 @@ export async function judgeCustomWidgetCase(args: {
export async function evaluateCustomWidgetCase(args: {
testCase: CustomWidgetAiEvaluationCase;
apiKey: string;
baseUrl?: string;
outputRoot: string;
maxLoops: number;
generatorModel?: string;
@@ -305,6 +326,7 @@ export async function evaluateCustomWidgetCase(args: {
try {
response = await callOpenRouter({
apiKey: args.apiKey,
baseUrl: args.baseUrl,
model: args.generatorModel ?? DEFAULT_GENERATOR_MODEL,
prompt,
purpose: "generation",
@@ -340,6 +362,7 @@ export async function evaluateCustomWidgetCase(args: {
testCase: args.testCase,
widget: canonical,
apiKey: args.apiKey,
baseUrl: args.baseUrl,
judgeModel: args.judgeModel,
});
await writeFile(path.join(caseDirectory, `judge-${attempt}.json`), judgeRaw, "utf8");
@@ -413,12 +436,13 @@ async function writeWidgetFiles(directory: string, widget: HomarrCustomWidgetV2,
async function callOpenRouter(args: {
apiKey: string;
baseUrl?: string;
model: string;
prompt: string;
purpose: "generation" | "judge";
}): Promise<string> {
const isJudge = args.purpose === "judge";
const response = await fetch("https://openrouter.ai/api/v1/chat/completions", {
const response = await fetch(getAiProviderChatCompletionsUrl(args.baseUrl), {
method: "POST",
headers: {
Authorization: `Bearer ${args.apiKey}`,
@@ -438,8 +462,8 @@ async function callOpenRouter(args: {
});
const payload = (await response.json()) as OpenRouterResponse;
if (!response.ok)
throw new Error(`OpenRouter request failed (${response.status}): ${payload.error?.message ?? "Unknown error"}`);
throw new Error(`AI provider request failed (${response.status}): ${payload.error?.message ?? "Unknown error"}`);
const content = payload.choices?.[0]?.message?.content;
if (!content) throw new Error("OpenRouter returned no message content");
if (!content) throw new Error("AI provider returned no message content");
return content;
}
@@ -4,11 +4,13 @@ import path from "node:path";
import { CUSTOM_WIDGET_AI_EVALUATION_CASES } from "./ai-evaluation-cases";
import { evaluateCustomWidgetAssistantCase } from "./ai-assistant-evaluation";
import type { CustomWidgetAssistantEvaluationResult } from "./ai-assistant-evaluation";
import { DEFAULT_GENERATOR_MODEL, DEFAULT_JUDGE_MODEL, evaluateCustomWidgetCase } from "./ai-evaluation";
import { evaluateCustomWidgetCase, resolveAiEvaluationProviderConfig } from "./ai-evaluation";
import type { AiEvaluationResult } from "./ai-evaluation";
const apiKey = process.env.OPENROUTER_API_KEY;
if (!apiKey) throw new Error("OPENROUTER_API_KEY is required for the live Custom Widget AI evaluation");
const { apiKey, baseUrl: providerBaseUrl, generatorModel, judgeModel } = resolveAiEvaluationProviderConfig(process.env);
if (!apiKey) {
throw new Error("AI_PROVIDER_API_KEY or OPENROUTER_API_KEY is required for the live Custom Widget AI evaluation");
}
const requestedCase = process.argv.find((value) => value.startsWith("--case="))?.slice("--case=".length);
const assistantMode = process.argv.includes("--assistant");
@@ -43,18 +45,20 @@ for (const testCase of selectedCases) {
? await evaluateCustomWidgetAssistantCase({
testCase,
apiKey,
baseUrl: providerBaseUrl,
outputRoot,
maxLoops,
generatorModel: process.env.OPENROUTER_GENERATOR_MODEL ?? DEFAULT_GENERATOR_MODEL,
judgeModel: process.env.OPENROUTER_JUDGE_MODEL ?? DEFAULT_JUDGE_MODEL,
generatorModel,
judgeModel,
})
: await evaluateCustomWidgetCase({
testCase,
apiKey,
baseUrl: providerBaseUrl,
outputRoot,
maxLoops,
generatorModel: process.env.OPENROUTER_GENERATOR_MODEL ?? DEFAULT_GENERATOR_MODEL,
judgeModel: process.env.OPENROUTER_JUDGE_MODEL ?? DEFAULT_JUDGE_MODEL,
generatorModel,
judgeModel,
});
results.push(result);
remainingLoopBudget -= result.attempts;
@@ -68,8 +72,9 @@ for (const testCase of selectedCases) {
const summary = {
generatedAt: new Date().toISOString(),
mode: assistantMode ? "assistant-tool-loop" : "manifest",
generatorModel: process.env.OPENROUTER_GENERATOR_MODEL ?? DEFAULT_GENERATOR_MODEL,
judgeModel: process.env.OPENROUTER_JUDGE_MODEL ?? DEFAULT_JUDGE_MODEL,
providerBaseUrl,
generatorModel,
judgeModel,
results: results.map((result) => ({
caseId: result.caseId,
attempts: result.attempts,
@@ -7,11 +7,13 @@ import {
buildRepairPrompt,
DEFAULT_GENERATOR_MODEL,
DEFAULT_JUDGE_MODEL,
getAiProviderChatCompletionsUrl,
getDeterministicEvaluationIssues,
getEvaluationResponseFixtureText,
getJudgeResponseFormat,
judgePasses,
parseJudgeResult,
resolveAiEvaluationProviderConfig,
} from "../../scripts/ai-evaluation";
import type { CustomWidgetJudgeResult } from "../../scripts/ai-evaluation";
@@ -178,6 +180,24 @@ describe("AI authoring evaluation", () => {
expect(format.json_schema.schema).toMatchObject({ type: "object", additionalProperties: false });
});
it("can run the same evaluation against the Homarr provider endpoint", () => {
expect(getAiProviderChatCompletionsUrl("https://homarr.dev/api/ai/v1/")).toBe(
"https://homarr.dev/api/ai/v1/chat/completions",
);
expect(getAiProviderChatCompletionsUrl()).toBe("https://openrouter.ai/api/v1/chat/completions");
expect(
resolveAiEvaluationProviderConfig({
AI_PROVIDER_BASE_URL: "https://homarr.dev/api/ai/v1/",
AI_PROVIDER_API_KEY: "workshop-token",
}),
).toEqual({
apiKey: "workshop-token",
baseUrl: "https://homarr.dev/api/ai/v1",
generatorModel: "homarr/deepseek-v4-flash-latest",
judgeModel: "homarr/deepseek-v4-flash-latest",
});
});
it("requires exceptional goal, design, practicality, and complexity scores", () => {
expect(judgePasses(makeJudgeResult(90))).toBe(true);
expect(
+1
View File
@@ -488,6 +488,7 @@ export const assistantConfigurations = mysqlTable("assistant_configuration", {
webSearchEnabled: boolean().notNull().default(false),
provider: varchar({ length: 32 })
.$type<
| "homarr"
| "openrouter"
| "openai"
| "anthropic"
+1
View File
@@ -486,6 +486,7 @@ export const assistantConfigurations = pgTable("assistant_configuration", {
webSearchEnabled: boolean().notNull().default(false),
provider: varchar({ length: 32 })
.$type<
| "homarr"
| "openrouter"
| "openai"
| "anthropic"
+1
View File
@@ -471,6 +471,7 @@ export const assistantConfigurations = sqliteTable("assistant_configuration", {
webSearchEnabled: int({ mode: "boolean" }).notNull().default(false),
provider: text()
.$type<
| "homarr"
| "openrouter"
| "openai"
| "anthropic"
+2
View File
@@ -2091,6 +2091,8 @@
"loading": "Loading Homarr provider allowance",
"signInRequired": "Workshop sign-in required for the Homarr provider",
"signInDescription": "Sign in with GitHub through the Community Workshop to receive your separate daily allowance.",
"exhaustedDescription": "Your Homarr provider allowance is used for today. It resets at midnight UTC.",
"unavailableDescription": "Sign in to the Workshop or wait for your Homarr provider allowance to reset.",
"signIn": "Sign in with GitHub",
"reset": "Resets {relative} at",
"toolCalls": "Each model request and completed tool call uses one request.",