feat: add the community-funded Homarr assistant provider (#6563)
* Add Homarr provider proxy foundation * Complete Homarr provider integration * Harden Homarr provider integration * Address Homarr provider review feedback * Address CodeRabbit provider feedback * Localize provider errors * Handle provider activity load failures * docs: improve workshop and homepage * Harden Homarr provider limits * fix: harden the Homarr provider * fix: address provider review feedback * fix: clarify provider request accounting * feat: complete the Homarr provider endpoint * fix: terminate invalid provider streams
This commit is contained in:
1 parent
b8f360c04e
commit
45d8bc9443
77 files changed
+4035
-763
No files matched your search
@@ -29,6 +29,7 @@ import {
|
||||
|
||||
import type { createTRPCContext } from "../trpc";
|
||||
import { fetchOpenRouterGenerationTelemetryAsync } from "../assistant-generation-telemetry";
|
||||
import { env } from "../env";
|
||||
import { orderMessagesByParent } from "../assistant-message-order";
|
||||
import { verifyAssistantGenerationAccessToken } from "../assistant-generation-access";
|
||||
import { createTRPCRouter, isDemoMode, permissionRequiredProcedure, protectedProcedure } from "../trpc";
|
||||
@@ -129,6 +130,9 @@ const normalizeDiscoveryPath = (value: string | null) => {
|
||||
return path;
|
||||
};
|
||||
|
||||
const getHomarrProviderBaseUrl = () =>
|
||||
`${(env.WORKSHOP_API_URL ?? env.HOMARR_WEBSITE_URL).replace(/\/+$/u, "")}/api/ai/v1`;
|
||||
|
||||
const getConfigurationAsync = async (db: Database) => {
|
||||
return await db.query.assistantConfigurations.findFirst({
|
||||
where: eq(assistantConfigurations.id, configurationId),
|
||||
@@ -160,7 +164,7 @@ const getProviderHeaders = (configuration: AssistantConfiguration) => {
|
||||
}
|
||||
if (configuration.provider === "openrouter") {
|
||||
headers["HTTP-Referer"] ??= "https://homarr.dev";
|
||||
headers["X-Title"] ??= "Homarr Assistant";
|
||||
headers["X-OpenRouter-Title"] ??= "Homarr AI Assistant";
|
||||
}
|
||||
return { ...headers, ...decryptCustomHeaders(configuration.encryptedHeaders) };
|
||||
};
|
||||
@@ -231,7 +235,7 @@ const fetchModelsAsync = async (configuration: AssistantConfiguration) => {
|
||||
const models = (Array.isArray(body) ? body : (body.data ?? body.models ?? [])).slice(0, 1_000);
|
||||
return models
|
||||
.filter((model) => {
|
||||
if (configuration.provider !== "openrouter") return true;
|
||||
if (configuration.provider !== "openrouter" && configuration.provider !== "homarr") return true;
|
||||
const parameters = Array.isArray(model.supported_parameters) ? model.supported_parameters : [];
|
||||
const outputModalities = Array.isArray(model.architecture?.output_modalities)
|
||||
? model.architecture.output_modalities
|
||||
@@ -260,6 +264,7 @@ const fetchModelsAsync = async (configuration: AssistantConfiguration) => {
|
||||
: null;
|
||||
const toolSupport =
|
||||
configuration.provider === "openrouter" ||
|
||||
configuration.provider === "homarr" ||
|
||||
model.capabilities?.function_calling === true ||
|
||||
configuration.provider === "anthropic"
|
||||
? ("confirmed" as const)
|
||||
@@ -479,7 +484,7 @@ export const assistantRouter = createTRPCRouter({
|
||||
const configuration = await getConfigurationAsync(ctx.db);
|
||||
if (isDemoMode) {
|
||||
return {
|
||||
provider: "homarr",
|
||||
provider: "custom" as const,
|
||||
defaultModelId: "homarr-assistant",
|
||||
models: [
|
||||
{
|
||||
@@ -546,23 +551,30 @@ export const assistantRouter = createTRPCRouter({
|
||||
}),
|
||||
)
|
||||
.mutation(async ({ ctx, input }) => {
|
||||
const baseUrl = normalizeBaseUrl(input.baseUrl);
|
||||
const modelDiscoveryPath = normalizeDiscoveryPath(input.modelDiscoveryPath);
|
||||
const baseUrl = input.provider === "homarr" ? getHomarrProviderBaseUrl() : normalizeBaseUrl(input.baseUrl);
|
||||
const modelDiscoveryPath =
|
||||
input.provider === "homarr" ? "/models" : normalizeDiscoveryPath(input.modelDiscoveryPath);
|
||||
const existing = await getConfigurationAsync(ctx.db);
|
||||
const destinationChanged =
|
||||
existing !== undefined && (existing.provider !== input.provider || existing.baseUrl !== baseUrl);
|
||||
const connectionChanged =
|
||||
existing !== undefined && (destinationChanged || existing.modelDiscoveryPath !== modelDiscoveryPath);
|
||||
const encryptedApiKey = input.apiKey
|
||||
? encryptSecret(input.apiKey)
|
||||
: input.clearApiKey || destinationChanged
|
||||
const encryptedApiKey =
|
||||
input.provider === "homarr"
|
||||
? null
|
||||
: (existing?.encryptedApiKey ?? null);
|
||||
const encryptedHeaders = input.customHeaders
|
||||
? encryptSecret(JSON.stringify(input.customHeaders))
|
||||
: input.clearCustomHeaders || destinationChanged
|
||||
: input.apiKey
|
||||
? encryptSecret(input.apiKey)
|
||||
: input.clearApiKey || destinationChanged
|
||||
? null
|
||||
: (existing?.encryptedApiKey ?? null);
|
||||
const encryptedHeaders =
|
||||
input.provider === "homarr"
|
||||
? null
|
||||
: (existing?.encryptedHeaders ?? null);
|
||||
: input.customHeaders
|
||||
? encryptSecret(JSON.stringify(input.customHeaders))
|
||||
: input.clearCustomHeaders || destinationChanged
|
||||
? null
|
||||
: (existing?.encryptedHeaders ?? null);
|
||||
|
||||
if (existing) {
|
||||
await ctx.db
|
||||
|
||||
@@ -121,6 +121,32 @@ describe("assistantRouter.updateConnection", () => {
|
||||
modelId: "example/model",
|
||||
});
|
||||
});
|
||||
|
||||
test("never stores administrator credentials for the Homarr provider", async () => {
|
||||
const db = await createConfiguredAssistantAsync();
|
||||
const caller = assistantRouter.createCaller({ db, deviceType: undefined, session: adminSession });
|
||||
|
||||
await caller.updateConnection({
|
||||
provider: "homarr",
|
||||
baseUrl: "https://attacker.example/v1",
|
||||
modelDiscoveryPath: "/attacker-models",
|
||||
apiKey: "must-not-be-stored",
|
||||
customHeaders: { "X-Secret": "must-not-be-stored" },
|
||||
clearApiKey: false,
|
||||
clearCustomHeaders: false,
|
||||
});
|
||||
|
||||
const [configuration] = await db.select().from(assistantConfigurations);
|
||||
expect(configuration).toMatchObject({
|
||||
provider: "homarr",
|
||||
baseUrl: "https://homarr.dev/api/ai/v1",
|
||||
modelDiscoveryPath: "/models",
|
||||
encryptedApiKey: null,
|
||||
encryptedHeaders: null,
|
||||
enabled: false,
|
||||
modelId: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("assistantRouter.updateConfiguration", () => {
|
||||
@@ -170,6 +196,49 @@ describe("assistantRouter.updateConfiguration", () => {
|
||||
expect(configuration?.webSearchEnabled).toBe(true);
|
||||
});
|
||||
|
||||
test("allows the Homarr OpenRouter proxy to enable web search", async () => {
|
||||
const db = await createConfiguredAssistantAsync();
|
||||
await db
|
||||
.update(assistantConfigurations)
|
||||
.set({
|
||||
provider: "homarr",
|
||||
baseUrl: "https://homarr.dev/api/ai/v1",
|
||||
encryptedApiKey: null,
|
||||
encryptedHeaders: null,
|
||||
modelId: "homarr/model",
|
||||
})
|
||||
.where(eq(assistantConfigurations.id, "default"));
|
||||
const caller = assistantRouter.createCaller({ db, deviceType: undefined, session: adminSession });
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn().mockResolvedValue(
|
||||
Response.json({
|
||||
data: [
|
||||
{
|
||||
id: "homarr/model",
|
||||
name: "Homarr model",
|
||||
supported_parameters: ["tools"],
|
||||
architecture: { output_modalities: ["text"] },
|
||||
},
|
||||
],
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
try {
|
||||
await caller.updateConfiguration({
|
||||
enabled: true,
|
||||
modelId: "homarr/model",
|
||||
webSearchEnabled: true,
|
||||
});
|
||||
} finally {
|
||||
vi.unstubAllGlobals();
|
||||
}
|
||||
|
||||
const [configuration] = await db.select().from(assistantConfigurations);
|
||||
expect(configuration?.webSearchEnabled).toBe(true);
|
||||
});
|
||||
|
||||
test("rejects OpenRouter server tools for an incompatible provider", async () => {
|
||||
const db = await createConfiguredAssistantAsync();
|
||||
await db
|
||||
|
||||
@@ -25,7 +25,9 @@ describe("assistant provider presets", () => {
|
||||
|
||||
test("requires credentials for hosted providers but not local or custom endpoints", () => {
|
||||
for (const provider of assistantProviderIds) {
|
||||
expect(assistantProviderRequiresApiKey(provider)).toBe(assistantProviderPresets[provider].category === "hosted");
|
||||
expect(assistantProviderRequiresApiKey(provider)).toBe(
|
||||
assistantProviderPresets[provider].category === "hosted" && provider !== "homarr",
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ import { getCustomWidgetJsonSchema } from "../src/core/schema";
|
||||
import type { CustomWidgetAiEvaluationCase } from "./ai-evaluation-cases";
|
||||
import {
|
||||
DEFAULT_GENERATOR_MODEL,
|
||||
getAiProviderChatCompletionsUrl,
|
||||
getDeterministicEvaluationIssues,
|
||||
judgePasses,
|
||||
parseJudgeResult,
|
||||
@@ -441,12 +442,13 @@ function buildAssistantPrompt(testCase: CustomWidgetAiEvaluationCase, feedback:
|
||||
|
||||
async function callAssistantStep(args: {
|
||||
apiKey: string;
|
||||
baseUrl?: string;
|
||||
model: string;
|
||||
messages: OpenRouterMessage[];
|
||||
tools: ToolDefinition[];
|
||||
toolChoice?: string;
|
||||
}): Promise<OpenRouterAssistantMessage> {
|
||||
const response = await fetch("https://openrouter.ai/api/v1/chat/completions", {
|
||||
const response = await fetch(getAiProviderChatCompletionsUrl(args.baseUrl), {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${args.apiKey}`,
|
||||
@@ -467,10 +469,10 @@ async function callAssistantStep(args: {
|
||||
});
|
||||
const payload = (await response.json()) as OpenRouterResponse;
|
||||
if (!response.ok) {
|
||||
throw new Error(`OpenRouter request failed (${response.status}): ${payload.error?.message ?? "Unknown error"}`);
|
||||
throw new Error(`AI provider request failed (${response.status}): ${payload.error?.message ?? "Unknown error"}`);
|
||||
}
|
||||
const message = payload.choices?.[0]?.message;
|
||||
if (!message) throw new Error("OpenRouter returned no assistant message");
|
||||
if (!message) throw new Error("AI provider returned no assistant message");
|
||||
return message;
|
||||
}
|
||||
|
||||
@@ -498,6 +500,7 @@ export const getAssistantEvaluationRecoveryToolName = (state: AssistantAttemptSt
|
||||
|
||||
async function callAssistantStepWithRecovery(args: {
|
||||
apiKey: string;
|
||||
baseUrl?: string;
|
||||
model: string;
|
||||
messages: OpenRouterMessage[];
|
||||
tools: ToolDefinition[];
|
||||
@@ -530,6 +533,7 @@ async function callAssistantStepWithRecovery(args: {
|
||||
async function runAssistantAttempt(args: {
|
||||
testCase: CustomWidgetAiEvaluationCase;
|
||||
apiKey: string;
|
||||
baseUrl?: string;
|
||||
model: string;
|
||||
feedback: readonly string[];
|
||||
}) {
|
||||
@@ -553,6 +557,7 @@ async function runAssistantAttempt(args: {
|
||||
: customWidgetAssistantEvaluationToolDefinitions;
|
||||
const message = await callAssistantStepWithRecovery({
|
||||
apiKey: args.apiKey,
|
||||
baseUrl: args.baseUrl,
|
||||
model: args.model,
|
||||
messages,
|
||||
tools,
|
||||
@@ -592,6 +597,7 @@ async function runAssistantAttempt(args: {
|
||||
export async function evaluateCustomWidgetAssistantCase(args: {
|
||||
testCase: CustomWidgetAiEvaluationCase;
|
||||
apiKey: string;
|
||||
baseUrl?: string;
|
||||
outputRoot: string;
|
||||
maxLoops: number;
|
||||
generatorModel?: string;
|
||||
@@ -612,6 +618,7 @@ export async function evaluateCustomWidgetAssistantCase(args: {
|
||||
run = await runAssistantAttempt({
|
||||
testCase: args.testCase,
|
||||
apiKey: args.apiKey,
|
||||
baseUrl: args.baseUrl,
|
||||
model: args.generatorModel ?? DEFAULT_GENERATOR_MODEL,
|
||||
feedback,
|
||||
});
|
||||
@@ -651,6 +658,7 @@ export async function evaluateCustomWidgetAssistantCase(args: {
|
||||
testCase: args.testCase,
|
||||
widget: run.state.createdWidget,
|
||||
apiKey: args.apiKey,
|
||||
baseUrl: args.baseUrl,
|
||||
judgeModel: args.judgeModel,
|
||||
});
|
||||
await writeFile(path.join(caseDirectory, `judge-${attempt}.json`), judgeRaw, "utf8");
|
||||
|
||||
@@ -14,7 +14,29 @@ import type { CustomWidgetAiEvaluationCase } from "./ai-evaluation-cases";
|
||||
// deepseek/deepseek-v4-flash-latest alias is rejected by the chat-completions API.
|
||||
export const DEFAULT_GENERATOR_MODEL = "~deepseek/deepseek-v4-flash-latest";
|
||||
export const DEFAULT_JUDGE_MODEL = "~deepseek/deepseek-v4-flash-latest";
|
||||
export const DEFAULT_AI_PROVIDER_BASE_URL = "https://openrouter.ai/api/v1";
|
||||
export const MAX_AI_EVALUATION_LOOPS = 10;
|
||||
export const getAiProviderChatCompletionsUrl = (baseUrl = DEFAULT_AI_PROVIDER_BASE_URL) =>
|
||||
`${baseUrl.replace(/\/+$/u, "")}/chat/completions`;
|
||||
export const resolveAiEvaluationProviderConfig = (environment: Record<string, string | undefined>) => {
|
||||
const configuredBaseUrl = environment.AI_PROVIDER_BASE_URL?.trim();
|
||||
const baseUrl = (configuredBaseUrl || DEFAULT_AI_PROVIDER_BASE_URL).replace(/\/+$/u, "");
|
||||
const homarrProvider = baseUrl.endsWith("/api/ai/v1");
|
||||
const openRouterProvider = baseUrl === DEFAULT_AI_PROVIDER_BASE_URL;
|
||||
const providerDefaultModel = homarrProvider ? "homarr/model" : DEFAULT_GENERATOR_MODEL;
|
||||
return {
|
||||
apiKey: environment.AI_PROVIDER_API_KEY ?? (openRouterProvider ? environment.OPENROUTER_API_KEY : undefined),
|
||||
baseUrl,
|
||||
generatorModel:
|
||||
environment.AI_PROVIDER_MODEL ??
|
||||
(openRouterProvider ? environment.OPENROUTER_GENERATOR_MODEL : undefined) ??
|
||||
providerDefaultModel,
|
||||
judgeModel:
|
||||
environment.AI_PROVIDER_JUDGE_MODEL ??
|
||||
(openRouterProvider ? environment.OPENROUTER_JUDGE_MODEL : undefined) ??
|
||||
(homarrProvider ? providerDefaultModel : DEFAULT_JUDGE_MODEL),
|
||||
};
|
||||
};
|
||||
|
||||
const scoreSchema = z.number().int().min(0).max(100);
|
||||
const judgeCategoriesSchema = z.strictObject({
|
||||
@@ -263,10 +285,12 @@ export async function requestCustomWidgetJudge(args: {
|
||||
testCase: CustomWidgetAiEvaluationCase;
|
||||
widget: HomarrCustomWidgetV2;
|
||||
apiKey: string;
|
||||
baseUrl?: string;
|
||||
judgeModel?: string;
|
||||
}) {
|
||||
return callOpenRouter({
|
||||
apiKey: args.apiKey,
|
||||
baseUrl: args.baseUrl,
|
||||
model: args.judgeModel ?? DEFAULT_JUDGE_MODEL,
|
||||
prompt: buildJudgePrompt(args.testCase, args.widget),
|
||||
purpose: "judge",
|
||||
@@ -277,6 +301,7 @@ export async function judgeCustomWidgetCase(args: {
|
||||
testCase: CustomWidgetAiEvaluationCase;
|
||||
widget: HomarrCustomWidgetV2;
|
||||
apiKey: string;
|
||||
baseUrl?: string;
|
||||
judgeModel?: string;
|
||||
}) {
|
||||
const raw = await requestCustomWidgetJudge(args);
|
||||
@@ -286,6 +311,7 @@ export async function judgeCustomWidgetCase(args: {
|
||||
export async function evaluateCustomWidgetCase(args: {
|
||||
testCase: CustomWidgetAiEvaluationCase;
|
||||
apiKey: string;
|
||||
baseUrl?: string;
|
||||
outputRoot: string;
|
||||
maxLoops: number;
|
||||
generatorModel?: string;
|
||||
@@ -305,6 +331,7 @@ export async function evaluateCustomWidgetCase(args: {
|
||||
try {
|
||||
response = await callOpenRouter({
|
||||
apiKey: args.apiKey,
|
||||
baseUrl: args.baseUrl,
|
||||
model: args.generatorModel ?? DEFAULT_GENERATOR_MODEL,
|
||||
prompt,
|
||||
purpose: "generation",
|
||||
@@ -340,6 +367,7 @@ export async function evaluateCustomWidgetCase(args: {
|
||||
testCase: args.testCase,
|
||||
widget: canonical,
|
||||
apiKey: args.apiKey,
|
||||
baseUrl: args.baseUrl,
|
||||
judgeModel: args.judgeModel,
|
||||
});
|
||||
await writeFile(path.join(caseDirectory, `judge-${attempt}.json`), judgeRaw, "utf8");
|
||||
@@ -413,12 +441,13 @@ async function writeWidgetFiles(directory: string, widget: HomarrCustomWidgetV2,
|
||||
|
||||
async function callOpenRouter(args: {
|
||||
apiKey: string;
|
||||
baseUrl?: string;
|
||||
model: string;
|
||||
prompt: string;
|
||||
purpose: "generation" | "judge";
|
||||
}): Promise<string> {
|
||||
const isJudge = args.purpose === "judge";
|
||||
const response = await fetch("https://openrouter.ai/api/v1/chat/completions", {
|
||||
const response = await fetch(getAiProviderChatCompletionsUrl(args.baseUrl), {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${args.apiKey}`,
|
||||
@@ -438,8 +467,8 @@ async function callOpenRouter(args: {
|
||||
});
|
||||
const payload = (await response.json()) as OpenRouterResponse;
|
||||
if (!response.ok)
|
||||
throw new Error(`OpenRouter request failed (${response.status}): ${payload.error?.message ?? "Unknown error"}`);
|
||||
throw new Error(`AI provider request failed (${response.status}): ${payload.error?.message ?? "Unknown error"}`);
|
||||
const content = payload.choices?.[0]?.message?.content;
|
||||
if (!content) throw new Error("OpenRouter returned no message content");
|
||||
if (!content) throw new Error("AI provider returned no message content");
|
||||
return content;
|
||||
}
|
||||
@@ -4,11 +4,13 @@ import path from "node:path";
|
||||
import { CUSTOM_WIDGET_AI_EVALUATION_CASES } from "./ai-evaluation-cases";
|
||||
import { evaluateCustomWidgetAssistantCase } from "./ai-assistant-evaluation";
|
||||
import type { CustomWidgetAssistantEvaluationResult } from "./ai-assistant-evaluation";
|
||||
import { DEFAULT_GENERATOR_MODEL, DEFAULT_JUDGE_MODEL, evaluateCustomWidgetCase } from "./ai-evaluation";
|
||||
import { evaluateCustomWidgetCase, resolveAiEvaluationProviderConfig } from "./ai-evaluation";
|
||||
import type { AiEvaluationResult } from "./ai-evaluation";
|
||||
|
||||
const apiKey = process.env.OPENROUTER_API_KEY;
|
||||
if (!apiKey) throw new Error("OPENROUTER_API_KEY is required for the live Custom Widget AI evaluation");
|
||||
const { apiKey, baseUrl: providerBaseUrl, generatorModel, judgeModel } = resolveAiEvaluationProviderConfig(process.env);
|
||||
if (!apiKey) {
|
||||
throw new Error("AI_PROVIDER_API_KEY or OPENROUTER_API_KEY is required for the live Custom Widget AI evaluation");
|
||||
}
|
||||
|
||||
const requestedCase = process.argv.find((value) => value.startsWith("--case="))?.slice("--case=".length);
|
||||
const assistantMode = process.argv.includes("--assistant");
|
||||
@@ -43,18 +45,20 @@ for (const testCase of selectedCases) {
|
||||
? await evaluateCustomWidgetAssistantCase({
|
||||
testCase,
|
||||
apiKey,
|
||||
baseUrl: providerBaseUrl,
|
||||
outputRoot,
|
||||
maxLoops,
|
||||
generatorModel: process.env.OPENROUTER_GENERATOR_MODEL ?? DEFAULT_GENERATOR_MODEL,
|
||||
judgeModel: process.env.OPENROUTER_JUDGE_MODEL ?? DEFAULT_JUDGE_MODEL,
|
||||
generatorModel,
|
||||
judgeModel,
|
||||
})
|
||||
: await evaluateCustomWidgetCase({
|
||||
testCase,
|
||||
apiKey,
|
||||
baseUrl: providerBaseUrl,
|
||||
outputRoot,
|
||||
maxLoops,
|
||||
generatorModel: process.env.OPENROUTER_GENERATOR_MODEL ?? DEFAULT_GENERATOR_MODEL,
|
||||
judgeModel: process.env.OPENROUTER_JUDGE_MODEL ?? DEFAULT_JUDGE_MODEL,
|
||||
generatorModel,
|
||||
judgeModel,
|
||||
});
|
||||
results.push(result);
|
||||
remainingLoopBudget -= result.attempts;
|
||||
@@ -68,8 +72,9 @@ for (const testCase of selectedCases) {
|
||||
const summary = {
|
||||
generatedAt: new Date().toISOString(),
|
||||
mode: assistantMode ? "assistant-tool-loop" : "manifest",
|
||||
generatorModel: process.env.OPENROUTER_GENERATOR_MODEL ?? DEFAULT_GENERATOR_MODEL,
|
||||
judgeModel: process.env.OPENROUTER_JUDGE_MODEL ?? DEFAULT_JUDGE_MODEL,
|
||||
providerBaseUrl,
|
||||
generatorModel,
|
||||
judgeModel,
|
||||
results: results.map((result) => ({
|
||||
caseId: result.caseId,
|
||||
attempts: result.attempts,
|
||||
|
||||
@@ -5,13 +5,16 @@ import {
|
||||
buildEvaluationPrompt,
|
||||
buildJudgePrompt,
|
||||
buildRepairPrompt,
|
||||
DEFAULT_AI_PROVIDER_BASE_URL,
|
||||
DEFAULT_GENERATOR_MODEL,
|
||||
DEFAULT_JUDGE_MODEL,
|
||||
getAiProviderChatCompletionsUrl,
|
||||
getDeterministicEvaluationIssues,
|
||||
getEvaluationResponseFixtureText,
|
||||
getJudgeResponseFormat,
|
||||
judgePasses,
|
||||
parseJudgeResult,
|
||||
resolveAiEvaluationProviderConfig,
|
||||
} from "../../scripts/ai-evaluation";
|
||||
import type { CustomWidgetJudgeResult } from "../../scripts/ai-evaluation";
|
||||
|
||||
@@ -178,6 +181,51 @@ describe("AI authoring evaluation", () => {
|
||||
expect(format.json_schema.schema).toMatchObject({ type: "object", additionalProperties: false });
|
||||
});
|
||||
|
||||
it("can run the same evaluation against the Homarr provider endpoint", () => {
|
||||
expect(getAiProviderChatCompletionsUrl("https://homarr.dev/api/ai/v1/")).toBe(
|
||||
"https://homarr.dev/api/ai/v1/chat/completions",
|
||||
);
|
||||
expect(getAiProviderChatCompletionsUrl()).toBe("https://openrouter.ai/api/v1/chat/completions");
|
||||
expect(
|
||||
resolveAiEvaluationProviderConfig({
|
||||
AI_PROVIDER_BASE_URL: "https://homarr.dev/api/ai/v1/",
|
||||
AI_PROVIDER_API_KEY: "workshop-token",
|
||||
}),
|
||||
).toEqual({
|
||||
apiKey: "workshop-token",
|
||||
baseUrl: "https://homarr.dev/api/ai/v1",
|
||||
generatorModel: "homarr/model",
|
||||
judgeModel: "homarr/model",
|
||||
});
|
||||
expect(
|
||||
resolveAiEvaluationProviderConfig({
|
||||
AI_PROVIDER_BASE_URL: " ",
|
||||
OPENROUTER_API_KEY: "legacy-key",
|
||||
OPENROUTER_GENERATOR_MODEL: "legacy-generator",
|
||||
OPENROUTER_JUDGE_MODEL: "legacy-judge",
|
||||
}),
|
||||
).toEqual({
|
||||
apiKey: "legacy-key",
|
||||
baseUrl: DEFAULT_AI_PROVIDER_BASE_URL,
|
||||
generatorModel: "legacy-generator",
|
||||
judgeModel: "legacy-judge",
|
||||
});
|
||||
expect(
|
||||
resolveAiEvaluationProviderConfig({
|
||||
AI_PROVIDER_BASE_URL: "https://homarr.dev/api/ai/v1",
|
||||
AI_PROVIDER_API_KEY: "workshop-token",
|
||||
OPENROUTER_API_KEY: "must-not-leak",
|
||||
OPENROUTER_GENERATOR_MODEL: "must-not-apply",
|
||||
OPENROUTER_JUDGE_MODEL: "must-not-apply",
|
||||
}),
|
||||
).toEqual({
|
||||
apiKey: "workshop-token",
|
||||
baseUrl: "https://homarr.dev/api/ai/v1",
|
||||
generatorModel: "homarr/model",
|
||||
judgeModel: "homarr/model",
|
||||
});
|
||||
});
|
||||
|
||||
it("requires exceptional goal, design, practicality, and complexity scores", () => {
|
||||
expect(judgePasses(makeJudgeResult(90))).toBe(true);
|
||||
expect(
|
||||
|
||||
@@ -486,6 +486,7 @@ export const assistantConfigurations = pgTable("assistant_configuration", {
|
||||
webSearchEnabled: boolean().notNull().default(false),
|
||||
provider: varchar({ length: 32 })
|
||||
.$type<
|
||||
| "homarr"
|
||||
| "openrouter"
|
||||
| "openai"
|
||||
| "anthropic"
|
||||
|
||||
@@ -471,6 +471,7 @@ export const assistantConfigurations = sqliteTable("assistant_configuration", {
|
||||
webSearchEnabled: int({ mode: "boolean" }).notNull().default(false),
|
||||
provider: text()
|
||||
.$type<
|
||||
| "homarr"
|
||||
| "openrouter"
|
||||
| "openai"
|
||||
| "anthropic"
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
export const assistantProviderIds = [
|
||||
"homarr",
|
||||
"openrouter",
|
||||
"openai",
|
||||
"anthropic",
|
||||
@@ -15,6 +16,7 @@ export const assistantProviderIds = [
|
||||
|
||||
export type AssistantProvider = (typeof assistantProviderIds)[number];
|
||||
export type AssistantProviderCategory = "hosted" | "local" | "custom";
|
||||
export const assistantHomarrProviderTokenHeader = "X-Homarr-Provider-Token";
|
||||
|
||||
export const assistantReasoningModes = ["auto", "none", "minimal", "low", "medium", "high"] as const;
|
||||
export type AssistantReasoningMode = (typeof assistantReasoningModes)[number];
|
||||
@@ -43,6 +45,15 @@ interface AssistantProviderPreset {
|
||||
}
|
||||
|
||||
export const assistantProviderPresets = {
|
||||
homarr: {
|
||||
// Browser and server code replace this public fallback with WORKSHOP_API_URL at runtime.
|
||||
baseUrl: "https://homarr.dev/api/ai/v1",
|
||||
modelDiscoveryPath: "/models",
|
||||
requiresApiKey: false,
|
||||
category: "hosted",
|
||||
discoveryAuthentication: "bearer",
|
||||
iconUrl: "https://homarr.dev/img/logo.png",
|
||||
},
|
||||
openrouter: {
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
modelDiscoveryPath: "/models",
|
||||
@@ -149,5 +160,5 @@ export const assistantProviderRequiresApiKey = (provider: AssistantProvider) =>
|
||||
* OpenRouter and Homarr Router implement OpenRouter server tools directly.
|
||||
* Custom endpoints may opt in when they proxy the OpenRouter request format.
|
||||
*/
|
||||
export const assistantProviderCanUseOpenRouterServerTools = (provider: AssistantProvider | "homarr") =>
|
||||
export const assistantProviderCanUseOpenRouterServerTools = (provider: AssistantProvider) =>
|
||||
provider === "openrouter" || provider === "homarr" || provider === "custom";
|
||||
@@ -13,6 +13,7 @@ import type { HomarrCustomWidgetV2 } from "@homarr/custom-widgets/core";
|
||||
|
||||
import type {
|
||||
WorkshopComment,
|
||||
WorkshopAssistantUsage,
|
||||
WorkshopReport,
|
||||
WorkshopReportSummary,
|
||||
WorkshopSubmissionDetail,
|
||||
@@ -28,6 +29,7 @@ import {
|
||||
WORKSHOP_REQUEST_TIMEOUT_MS,
|
||||
WORKSHOP_SCHEMA_BY_TYPE,
|
||||
workshopCommentSchema,
|
||||
workshopAssistantUsageSchema,
|
||||
workshopReportSchema,
|
||||
workshopReportSummarySchema,
|
||||
workshopSubmissionDetailSchema,
|
||||
@@ -271,13 +273,19 @@ export class WorkshopBackend {
|
||||
}
|
||||
|
||||
public get currentUser(): WorkshopUser | null {
|
||||
if (!this.pocketBase.authStore.isValid) return null;
|
||||
const record = this.pocketBase.authStore.record;
|
||||
if (!record) return null;
|
||||
return workshopUserSchema.parse({
|
||||
const parsed = workshopUserSchema.safeParse({
|
||||
id: record.id,
|
||||
name: record.name || "",
|
||||
isAdmin: record.isAdmin === true,
|
||||
});
|
||||
return parsed.success ? parsed.data : null;
|
||||
}
|
||||
|
||||
public get authToken(): string | null {
|
||||
return this.pocketBase.authStore.isValid && this.currentUser ? this.pocketBase.authStore.token : null;
|
||||
}
|
||||
|
||||
public subscribeToAuth(listener: (user: WorkshopUser | null) => void) {
|
||||
@@ -285,7 +293,10 @@ export class WorkshopBackend {
|
||||
}
|
||||
|
||||
public async refreshAuth() {
|
||||
if (!this.pocketBase.authStore.isValid) return null;
|
||||
if (!this.pocketBase.authStore.isValid) {
|
||||
this.pocketBase.authStore.clear();
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
await this.pocketBase.collection("users").authRefresh();
|
||||
return this.currentUser;
|
||||
@@ -304,7 +315,9 @@ export class WorkshopBackend {
|
||||
provider: "github",
|
||||
});
|
||||
this.pocketBase.authStore.save(auth.token, auth.record);
|
||||
return this.currentUser;
|
||||
const user = this.currentUser;
|
||||
if (!user) throw new Error("The Workshop returned an invalid user session.");
|
||||
return user;
|
||||
} catch (error) {
|
||||
throw workshopError(error, "GitHub sign-in failed");
|
||||
}
|
||||
@@ -314,6 +327,19 @@ export class WorkshopBackend {
|
||||
this.pocketBase.authStore.clear();
|
||||
}
|
||||
|
||||
public async getAssistantUsage(signal?: AbortSignal): Promise<WorkshopAssistantUsage> {
|
||||
try {
|
||||
const value = await this.pocketBase.send("/api/ai/usage", {
|
||||
method: "GET",
|
||||
signal: requestSignal(signal),
|
||||
requestKey: null,
|
||||
});
|
||||
return workshopAssistantUsageSchema.parse(value);
|
||||
} catch (error) {
|
||||
throw workshopError(error, "Failed to load the Homarr provider allowance");
|
||||
}
|
||||
}
|
||||
|
||||
public async list(options: WorkshopListOptions = {}): Promise<WorkshopPage<WorkshopSubmissionSummary>> {
|
||||
try {
|
||||
const page = options.page ?? 1;
|
||||
|
||||
@@ -6,17 +6,31 @@ const mocks = vi.hoisted(() => ({
|
||||
create: vi.fn(),
|
||||
getFullList: vi.fn(),
|
||||
getList: vi.fn(),
|
||||
send: vi.fn(),
|
||||
update: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("pocketbase", () => {
|
||||
class MockPocketBase {
|
||||
public readonly authStore = {
|
||||
token: "",
|
||||
record: null as Record<string, unknown> | null,
|
||||
isValid: false,
|
||||
save: (_token: string, record: Record<string, unknown>) => {
|
||||
get isValid() {
|
||||
try {
|
||||
const payload = JSON.parse(atob(this.token.split(".")[1] ?? "")) as { exp?: unknown };
|
||||
return typeof payload.exp === "number" && payload.exp > Date.now() / 1000;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
},
|
||||
save: (token: string, record: Record<string, unknown>) => {
|
||||
this.authStore.token = token;
|
||||
this.authStore.record = record;
|
||||
},
|
||||
clear: () => {
|
||||
this.authStore.token = "";
|
||||
this.authStore.record = null;
|
||||
},
|
||||
};
|
||||
|
||||
public collection() {
|
||||
@@ -28,6 +42,10 @@ vi.mock("pocketbase", () => {
|
||||
return this;
|
||||
}
|
||||
|
||||
public send(path: string, options: unknown) {
|
||||
return mocks.send(path, options);
|
||||
}
|
||||
|
||||
public filter(template: string, values: Record<string, string>) {
|
||||
return template.replaceAll(/\{:(\w+)\}/gu, (_match, key: string) => `'${values[key]}'`);
|
||||
}
|
||||
@@ -50,6 +68,8 @@ import { WorkshopBackend } from "./backend";
|
||||
import type { WorkshopSubmissionRecord } from "./backend";
|
||||
import * as workshopClient from "./client";
|
||||
|
||||
const validWorkshopToken = "header.eyJleHAiOjQxMDI0NDQ4MDB9.signature";
|
||||
|
||||
describe("WorkshopBackend", () => {
|
||||
test("resolves the advertised client module", () => {
|
||||
expect(workshopClient.WorkshopBackend).toBe(WorkshopBackend);
|
||||
@@ -62,12 +82,13 @@ describe("WorkshopBackend", () => {
|
||||
mocks.create.mockReset();
|
||||
mocks.getFullList.mockReset();
|
||||
mocks.getList.mockReset();
|
||||
mocks.send.mockReset();
|
||||
mocks.update.mockReset();
|
||||
});
|
||||
|
||||
test("lets PocketBase own the complete GitHub popup flow", async () => {
|
||||
mocks.authWithOAuth2.mockResolvedValue({
|
||||
token: "token",
|
||||
token: validWorkshopToken,
|
||||
record: {
|
||||
id: "user-id",
|
||||
name: "octocat",
|
||||
@@ -91,9 +112,42 @@ describe("WorkshopBackend", () => {
|
||||
expect(mocks.autoCancellation).toHaveBeenCalledWith(false);
|
||||
});
|
||||
|
||||
test("reuses a valid Workshop session for the Homarr provider allowance", async () => {
|
||||
const client = new WorkshopBackend("https://workshop.example.com");
|
||||
expect(client.authToken).toBeNull();
|
||||
client.pocketBase.authStore.save(validWorkshopToken, { id: "author-id", name: "octocat" } as never);
|
||||
mocks.send.mockResolvedValue({
|
||||
limit: 50,
|
||||
used: 7,
|
||||
remaining: 43,
|
||||
resetsAt: "2026-08-12T00:00:00Z",
|
||||
});
|
||||
|
||||
await expect(client.getAssistantUsage()).resolves.toMatchObject({ limit: 50, remaining: 43 });
|
||||
expect(client.authToken).toBe(validWorkshopToken);
|
||||
expect(mocks.send).toHaveBeenCalledWith(
|
||||
"/api/ai/usage",
|
||||
expect.objectContaining({ method: "GET", requestKey: null, signal: expect.any(AbortSignal) }),
|
||||
);
|
||||
});
|
||||
|
||||
test("fails closed for expired tokens and malformed Workshop user records", () => {
|
||||
const client = new WorkshopBackend("https://workshop.example.com");
|
||||
client.pocketBase.authStore.save("header.eyJleHAiOjF9.signature", {
|
||||
id: "author-id",
|
||||
name: "octocat",
|
||||
} as never);
|
||||
expect(client.currentUser).toBeNull();
|
||||
expect(client.authToken).toBeNull();
|
||||
|
||||
client.pocketBase.authStore.save(validWorkshopToken, { id: 42 } as never);
|
||||
expect(client.currentUser).toBeNull();
|
||||
expect(client.authToken).toBeNull();
|
||||
});
|
||||
|
||||
test("sends the exported schema while leaving initial revision ownership to PocketBase", async () => {
|
||||
const client = new WorkshopBackend("https://workshop.example.com");
|
||||
client.pocketBase.authStore.save("token", { id: "author-id" } as never);
|
||||
client.pocketBase.authStore.save(validWorkshopToken, { id: "author-id" } as never);
|
||||
mocks.create.mockResolvedValue({ id: "submission-id" });
|
||||
vi.spyOn(client, "get").mockResolvedValue(listingRecord() as never);
|
||||
|
||||
|
||||
@@ -102,6 +102,14 @@ export const workshopUserSchema = z.object({
|
||||
});
|
||||
export type WorkshopUser = z.infer<typeof workshopUserSchema>;
|
||||
|
||||
export const workshopAssistantUsageSchema = z.object({
|
||||
limit: z.number().int().nonnegative(),
|
||||
used: z.number().int().nonnegative(),
|
||||
remaining: z.number().int().nonnegative(),
|
||||
resetsAt: z.iso.datetime({ offset: true }),
|
||||
});
|
||||
export type WorkshopAssistantUsage = z.infer<typeof workshopAssistantUsageSchema>;
|
||||
|
||||
const normalizeWorkshopSubmissionRecord = (value: unknown) => {
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) return value;
|
||||
const record = value as Record<string, unknown>;
|
||||
|
||||
Reference in new issue
Block a user