Harden Homarr provider integration

packages/custom-widgets/scripts/ai-assistant-evaluation.ts

@@ -21,6 +21,7 @@ import { getCustomWidgetJsonSchema } from "../src/core/schema";
 import type { CustomWidgetAiEvaluationCase } from "./ai-evaluation-cases";
 import {
   DEFAULT_GENERATOR_MODEL,
+  getAiProviderChatCompletionsUrl,
   getDeterministicEvaluationIssues,
   judgePasses,
   parseJudgeResult,
@@ -441,12 +442,13 @@ function buildAssistantPrompt(testCase: CustomWidgetAiEvaluationCase, feedback:
 
 async function callAssistantStep(args: {
   apiKey: string;
+  baseUrl?: string;
   model: string;
   messages: OpenRouterMessage[];
   tools: ToolDefinition[];
   toolChoice?: string;
 }): Promise<OpenRouterAssistantMessage> {
-  const response = await fetch("https://openrouter.ai/api/v1/chat/completions", {
+  const response = await fetch(getAiProviderChatCompletionsUrl(args.baseUrl), {
     method: "POST",
     headers: {
       Authorization: `Bearer ${args.apiKey}`,
@@ -467,10 +469,10 @@ async function callAssistantStep(args: {
   });
   const payload = (await response.json()) as OpenRouterResponse;
   if (!response.ok) {
-    throw new Error(`OpenRouter request failed (${response.status}): ${payload.error?.message ?? "Unknown error"}`);
+    throw new Error(`AI provider request failed (${response.status}): ${payload.error?.message ?? "Unknown error"}`);
   }
   const message = payload.choices?.[0]?.message;
-  if (!message) throw new Error("OpenRouter returned no assistant message");
+  if (!message) throw new Error("AI provider returned no assistant message");
   return message;
 }
 
@@ -498,6 +500,7 @@ export const getAssistantEvaluationRecoveryToolName = (state: AssistantAttemptSt
 
 async function callAssistantStepWithRecovery(args: {
   apiKey: string;
+  baseUrl?: string;
   model: string;
   messages: OpenRouterMessage[];
   tools: ToolDefinition[];
@@ -530,6 +533,7 @@ async function callAssistantStepWithRecovery(args: {
 async function runAssistantAttempt(args: {
   testCase: CustomWidgetAiEvaluationCase;
   apiKey: string;
+  baseUrl?: string;
   model: string;
   feedback: readonly string[];
 }) {
@@ -553,6 +557,7 @@ async function runAssistantAttempt(args: {
         : customWidgetAssistantEvaluationToolDefinitions;
     const message = await callAssistantStepWithRecovery({
       apiKey: args.apiKey,
+      baseUrl: args.baseUrl,
       model: args.model,
       messages,
       tools,
@@ -592,6 +597,7 @@ async function runAssistantAttempt(args: {
 export async function evaluateCustomWidgetAssistantCase(args: {
   testCase: CustomWidgetAiEvaluationCase;
   apiKey: string;
+  baseUrl?: string;
   outputRoot: string;
   maxLoops: number;
   generatorModel?: string;
@@ -612,6 +618,7 @@ export async function evaluateCustomWidgetAssistantCase(args: {
       run = await runAssistantAttempt({
         testCase: args.testCase,
         apiKey: args.apiKey,
+        baseUrl: args.baseUrl,
         model: args.generatorModel ?? DEFAULT_GENERATOR_MODEL,
         feedback,
       });
@@ -651,6 +658,7 @@ export async function evaluateCustomWidgetAssistantCase(args: {
         testCase: args.testCase,
         widget: run.state.createdWidget,
         apiKey: args.apiKey,
+        baseUrl: args.baseUrl,
         judgeModel: args.judgeModel,
       });
       await writeFile(path.join(caseDirectory, `judge-${attempt}.json`), judgeRaw, "utf8");

packages/custom-widgets/scripts/ai-evaluation.ts

@@ -14,7 +14,24 @@ import type { CustomWidgetAiEvaluationCase } from "./ai-evaluation-cases";
 // deepseek/deepseek-v4-flash-latest alias is rejected by the chat-completions API.
 export const DEFAULT_GENERATOR_MODEL = "~deepseek/deepseek-v4-flash-latest";
 export const DEFAULT_JUDGE_MODEL = "~deepseek/deepseek-v4-flash-latest";
+export const DEFAULT_AI_PROVIDER_BASE_URL = "https://openrouter.ai/api/v1";
 export const MAX_AI_EVALUATION_LOOPS = 10;
+export const getAiProviderChatCompletionsUrl = (baseUrl = DEFAULT_AI_PROVIDER_BASE_URL) =>
+  `${baseUrl.replace(/\/+$/u, "")}/chat/completions`;
+export const resolveAiEvaluationProviderConfig = (environment: Record<string, string | undefined>) => {
+  const baseUrl = (environment.AI_PROVIDER_BASE_URL ?? DEFAULT_AI_PROVIDER_BASE_URL).replace(/\/+$/u, "");
+  const homarrProvider = baseUrl.endsWith("/api/ai/v1");
+  const providerDefaultModel = homarrProvider ? "homarr/deepseek-v4-flash-latest" : DEFAULT_GENERATOR_MODEL;
+  return {
+    apiKey: environment.AI_PROVIDER_API_KEY ?? environment.OPENROUTER_API_KEY,
+    baseUrl,
+    generatorModel: environment.AI_PROVIDER_MODEL ?? environment.OPENROUTER_GENERATOR_MODEL ?? providerDefaultModel,
+    judgeModel:
+      environment.AI_PROVIDER_JUDGE_MODEL ??
+      environment.OPENROUTER_JUDGE_MODEL ??
+      (homarrProvider ? providerDefaultModel : DEFAULT_JUDGE_MODEL),
+  };
+};
 
 const scoreSchema = z.number().int().min(0).max(100);
 const judgeCategoriesSchema = z.strictObject({
@@ -263,10 +280,12 @@ export async function requestCustomWidgetJudge(args: {
   testCase: CustomWidgetAiEvaluationCase;
   widget: HomarrCustomWidgetV2;
   apiKey: string;
+  baseUrl?: string;
   judgeModel?: string;
 }) {
   return callOpenRouter({
     apiKey: args.apiKey,
+    baseUrl: args.baseUrl,
     model: args.judgeModel ?? DEFAULT_JUDGE_MODEL,
     prompt: buildJudgePrompt(args.testCase, args.widget),
     purpose: "judge",
@@ -277,6 +296,7 @@ export async function judgeCustomWidgetCase(args: {
   testCase: CustomWidgetAiEvaluationCase;
   widget: HomarrCustomWidgetV2;
   apiKey: string;
+  baseUrl?: string;
   judgeModel?: string;
 }) {
   const raw = await requestCustomWidgetJudge(args);
@@ -286,6 +306,7 @@ export async function judgeCustomWidgetCase(args: {
 export async function evaluateCustomWidgetCase(args: {
   testCase: CustomWidgetAiEvaluationCase;
   apiKey: string;
+  baseUrl?: string;
   outputRoot: string;
   maxLoops: number;
   generatorModel?: string;
@@ -305,6 +326,7 @@ export async function evaluateCustomWidgetCase(args: {
     try {
       response = await callOpenRouter({
         apiKey: args.apiKey,
+        baseUrl: args.baseUrl,
         model: args.generatorModel ?? DEFAULT_GENERATOR_MODEL,
         prompt,
         purpose: "generation",
@@ -340,6 +362,7 @@ export async function evaluateCustomWidgetCase(args: {
         testCase: args.testCase,
         widget: canonical,
         apiKey: args.apiKey,
+        baseUrl: args.baseUrl,
         judgeModel: args.judgeModel,
       });
       await writeFile(path.join(caseDirectory, `judge-${attempt}.json`), judgeRaw, "utf8");
@@ -413,12 +436,13 @@ async function writeWidgetFiles(directory: string, widget: HomarrCustomWidgetV2,
 
 async function callOpenRouter(args: {
   apiKey: string;
+  baseUrl?: string;
   model: string;
   prompt: string;
   purpose: "generation" | "judge";
 }): Promise<string> {
   const isJudge = args.purpose === "judge";
-  const response = await fetch("https://openrouter.ai/api/v1/chat/completions", {
+  const response = await fetch(getAiProviderChatCompletionsUrl(args.baseUrl), {
     method: "POST",
     headers: {
       Authorization: `Bearer ${args.apiKey}`,
@@ -438,8 +462,8 @@ async function callOpenRouter(args: {
   });
   const payload = (await response.json()) as OpenRouterResponse;
   if (!response.ok)
-    throw new Error(`OpenRouter request failed (${response.status}): ${payload.error?.message ?? "Unknown error"}`);
+    throw new Error(`AI provider request failed (${response.status}): ${payload.error?.message ?? "Unknown error"}`);
   const content = payload.choices?.[0]?.message?.content;
-  if (!content) throw new Error("OpenRouter returned no message content");
+  if (!content) throw new Error("AI provider returned no message content");
   return content;
 }

packages/custom-widgets/scripts/evaluate-ai-authoring.ts

@@ -4,11 +4,13 @@ import path from "node:path";
 import { CUSTOM_WIDGET_AI_EVALUATION_CASES } from "./ai-evaluation-cases";
 import { evaluateCustomWidgetAssistantCase } from "./ai-assistant-evaluation";
 import type { CustomWidgetAssistantEvaluationResult } from "./ai-assistant-evaluation";
-import { DEFAULT_GENERATOR_MODEL, DEFAULT_JUDGE_MODEL, evaluateCustomWidgetCase } from "./ai-evaluation";
+import { evaluateCustomWidgetCase, resolveAiEvaluationProviderConfig } from "./ai-evaluation";
 import type { AiEvaluationResult } from "./ai-evaluation";
 
-const apiKey = process.env.OPENROUTER_API_KEY;
-if (!apiKey) throw new Error("OPENROUTER_API_KEY is required for the live Custom Widget AI evaluation");
+const { apiKey, baseUrl: providerBaseUrl, generatorModel, judgeModel } = resolveAiEvaluationProviderConfig(process.env);
+if (!apiKey) {
+  throw new Error("AI_PROVIDER_API_KEY or OPENROUTER_API_KEY is required for the live Custom Widget AI evaluation");
+}
 
 const requestedCase = process.argv.find((value) => value.startsWith("--case="))?.slice("--case=".length);
 const assistantMode = process.argv.includes("--assistant");
@@ -43,18 +45,20 @@ for (const testCase of selectedCases) {
     ? await evaluateCustomWidgetAssistantCase({
         testCase,
         apiKey,
+        baseUrl: providerBaseUrl,
         outputRoot,
         maxLoops,
-        generatorModel: process.env.OPENROUTER_GENERATOR_MODEL ?? DEFAULT_GENERATOR_MODEL,
-        judgeModel: process.env.OPENROUTER_JUDGE_MODEL ?? DEFAULT_JUDGE_MODEL,
+        generatorModel,
+        judgeModel,
       })
     : await evaluateCustomWidgetCase({
         testCase,
         apiKey,
+        baseUrl: providerBaseUrl,
         outputRoot,
         maxLoops,
-        generatorModel: process.env.OPENROUTER_GENERATOR_MODEL ?? DEFAULT_GENERATOR_MODEL,
-        judgeModel: process.env.OPENROUTER_JUDGE_MODEL ?? DEFAULT_JUDGE_MODEL,
+        generatorModel,
+        judgeModel,
       });
   results.push(result);
   remainingLoopBudget -= result.attempts;
@@ -68,8 +72,9 @@ for (const testCase of selectedCases) {
 const summary = {
   generatedAt: new Date().toISOString(),
   mode: assistantMode ? "assistant-tool-loop" : "manifest",
-  generatorModel: process.env.OPENROUTER_GENERATOR_MODEL ?? DEFAULT_GENERATOR_MODEL,
-  judgeModel: process.env.OPENROUTER_JUDGE_MODEL ?? DEFAULT_JUDGE_MODEL,
+  providerBaseUrl,
+  generatorModel,
+  judgeModel,
   results: results.map((result) => ({
     caseId: result.caseId,
     attempts: result.attempts,