diff --git a/apps/docs/docs/management/assistant.mdx b/apps/docs/docs/management/assistant.mdx index 25c1497c1..5129d6f48 100644 --- a/apps/docs/docs/management/assistant.mdx +++ b/apps/docs/docs/management/assistant.mdx @@ -113,8 +113,13 @@ Each completed response includes a circular context-window meter. Select it to o the provider reports them: input, output, cached-input and reasoning tokens, used and remaining context, time to first output, total duration, output speed, agent-step timing, and cost. OpenRouter-reported cost is labeled as reported. -OpenRouter responses also show the routed provider, generation identifier, and upstream inference cost when -the endpoint supplies those fields. +OpenRouter responses request inline usage and routing metadata. When request details are opened, Homarr also +loads the server-side generation record for each model step and briefly polls while OpenRouter finishes +processing it. The details separate Homarr's end-to-end timing from OpenRouter's provider latency, +generation time, and provider throughput, and can include normalized and native token counts, moderation +latency, cache discount, fallback count, service tier, data region, BYOK status, routed provider, generation +identifier, exact cost, and upstream inference cost. Provider credentials remain server-side; the browser +can request only generation identifiers already stored on a conversation owned by the signed-in user. When a provider omits cost but model discovery includes prompt and completion pricing, Homarr labels the calculated value as estimated. Missing provider data is shown as not reported rather than inferred. diff --git a/apps/nextjs/src/app/api/assistant/chat/route.ts b/apps/nextjs/src/app/api/assistant/chat/route.ts index 93a1da594..ebab3ab64 100644 --- a/apps/nextjs/src/app/api/assistant/chat/route.ts +++ b/apps/nextjs/src/app/api/assistant/chat/route.ts @@ -119,21 +119,33 @@ const createProviderTelemetryExtractor = (): MetadataExtractor => { if (!body) return; const usage = asRecord(body.usage); const costDetails = asRecord(usage?.cost_details); + const routerMetadata = asRecord(body.openrouter_metadata); + const endpoints = asRecord(routerMetadata?.endpoints); + const selectedEndpoint = Array.isArray(endpoints?.available) + ? endpoints.available.map(asRecord).find((endpoint) => endpoint?.selected === true) + : undefined; const generationId = typeof body.id === "string" ? body.id : undefined; const routedProvider = typeof body.provider === "string" ? body.provider : typeof usage?.provider === "string" ? usage.provider - : undefined; + : typeof selectedEndpoint?.provider === "string" + ? selectedEndpoint.provider + : undefined; const cost = asFiniteNumber(usage?.cost); const upstreamCost = asFiniteNumber(costDetails?.upstream_inference_cost); + const routerAttempt = asFiniteNumber(routerMetadata?.attempt); metadata = { ...metadata, ...(generationId ? { generationId } : {}), ...(routedProvider ? { routedProvider } : {}), ...(cost !== undefined ? { cost } : {}), ...(upstreamCost !== undefined ? { upstreamCost } : {}), + ...(routerAttempt !== undefined ? { fallbackCount: Math.max(routerAttempt - 1, 0) } : {}), + ...(typeof routerMetadata?.strategy === "string" ? { routerStrategy: routerMetadata.strategy } : {}), + ...(typeof routerMetadata?.region === "string" ? { routerRegion: routerMetadata.region } : {}), + ...(typeof routerMetadata?.is_byok === "boolean" ? { isByok: routerMetadata.is_byok ? 1 : 0 } : {}), }; }; const build = () => (Object.keys(metadata).length > 0 ? { homarrTelemetry: metadata } : undefined); @@ -252,6 +264,16 @@ const toUsageMetadata = (usage: { : {}), }); +const sumCompleteStepMetric = ( + steps: AssistantRequestStep[], + getValue: (step: AssistantRequestStep) => number | undefined, +) => { + if (steps.length === 0) return undefined; + const values = steps.map(getValue); + if (values.some((value) => value === undefined)) return undefined; + return (values as number[]).reduce((sum, value) => sum + value, 0); +}; + const safeToolError = (error: unknown) => { if (error instanceof Error && "code" in error) { switch ((error as Error & { code: string }).code) { @@ -428,14 +450,20 @@ export async function POST(request: Request) { : {}; const providerHeaders = { ...(configuration.provider === "openrouter" - ? { "HTTP-Referer": "https://homarr.dev", "X-Title": "Homarr Assistant" } + ? { + "HTTP-Referer": "https://homarr.dev", + "X-Title": "Homarr Assistant", + "X-OpenRouter-Metadata": "enabled", + } : {}), ...customHeaders, }; + const providerApiKey = configuration.encryptedApiKey ? decryptSecret(configuration.encryptedApiKey) : undefined; + const providerName = `homarr-${configuration.provider}`; const provider = createOpenAICompatible({ - name: `homarr-${configuration.provider}`, + name: providerName, baseURL: configuration.baseUrl, - apiKey: configuration.encryptedApiKey ? decryptSecret(configuration.encryptedApiKey) : undefined, + apiKey: providerApiKey, headers: providerHeaders, includeUsage: true, metadataExtractor: createProviderTelemetryExtractor(), @@ -464,6 +492,8 @@ export async function POST(request: Request) { timeout: { totalMs: 55_000, stepMs: 30_000, toolMs: 30_000 }, maxRetries: 2, reasoning: parsed.data.reasoning === "auto" ? undefined : parsed.data.reasoning, + providerOptions: + configuration.provider === "openrouter" ? { [providerName]: { usage: { include: true } } } : undefined, toolApproval, experimental_toolApprovalSecret: getToolApprovalSecret(), onChunk: ({ chunk }) => { @@ -484,17 +514,13 @@ export async function POST(request: Request) { }, onStepFinish: ({ performance, providerMetadata, response, usage }) => { const telemetry = asRecord(providerMetadata?.homarrTelemetry); - const cost = asFiniteNumber(telemetry?.cost); - const stepUpstreamCost = asFiniteNumber(telemetry?.upstreamCost); - if (cost !== undefined) { - reportedCost += cost; - hasReportedCost = true; - } - if (stepUpstreamCost !== undefined) { - upstreamCost += stepUpstreamCost; - hasUpstreamCost = true; - } - requestSteps.push({ + const generationId = + typeof telemetry?.generationId === "string" + ? telemetry.generationId + : typeof response.id === "string" + ? response.id + : undefined; + const step: AssistantRequestStep = { index: requestSteps.length + 1, durationMs: performance.stepTimeMs, modelDurationMs: performance.responseTimeMs, @@ -507,11 +533,28 @@ export async function POST(request: Request) { : {}), ...(usage.inputTokens !== undefined ? { inputTokens: usage.inputTokens } : {}), ...(usage.outputTokens !== undefined ? { outputTokens: usage.outputTokens } : {}), - ...(cost !== undefined ? { cost } : {}), - ...(stepUpstreamCost !== undefined ? { upstreamCost: stepUpstreamCost } : {}), - ...(typeof response.id === "string" ? { generationId: response.id } : {}), + ...(asFiniteNumber(telemetry?.cost) !== undefined ? { cost: asFiniteNumber(telemetry?.cost) } : {}), + ...(asFiniteNumber(telemetry?.upstreamCost) !== undefined + ? { upstreamCost: asFiniteNumber(telemetry?.upstreamCost) } + : {}), + ...(generationId ? { generationId } : {}), ...(typeof telemetry?.routedProvider === "string" ? { routedProvider: telemetry.routedProvider } : {}), - }); + ...(asFiniteNumber(telemetry?.fallbackCount) !== undefined + ? { fallbackCount: asFiniteNumber(telemetry?.fallbackCount) } + : {}), + ...(typeof telemetry?.routerStrategy === "string" ? { routerStrategy: telemetry.routerStrategy } : {}), + ...(typeof telemetry?.routerRegion === "string" ? { routerRegion: telemetry.routerRegion } : {}), + ...(telemetry?.isByok === 0 || telemetry?.isByok === 1 ? { isByok: telemetry.isByok === 1 } : {}), + }; + requestSteps.push(step); + if (step.cost !== undefined) { + reportedCost += step.cost; + hasReportedCost = true; + } + if (step.upstreamCost !== undefined) { + upstreamCost += step.upstreamCost; + hasUpstreamCost = true; + } }, onError: ({ error }) => { logger.error("Assistant response stream failed", { @@ -552,8 +595,29 @@ export async function POST(request: Request) { } const completedAt = Date.now(); - const usage = toUsageMetadata(part.totalUsage); - const contextUsed = (usage.inputTokens ?? 0) + (usage.outputTokens ?? 0); + const aiUsage = toUsageMetadata(part.totalUsage); + const providerInputTokens = sumCompleteStepMetric(requestSteps, (step) => step.inputTokens); + const providerOutputTokens = sumCompleteStepMetric(requestSteps, (step) => step.outputTokens); + const providerCachedInputTokens = sumCompleteStepMetric(requestSteps, (step) => step.cachedInputTokens); + const providerReasoningTokens = sumCompleteStepMetric(requestSteps, (step) => step.reasoningTokens); + const inputTokens = providerInputTokens ?? aiUsage.inputTokens; + const outputTokens = providerOutputTokens ?? aiUsage.outputTokens; + const usage: AssistantUsage = { + ...aiUsage, + ...(inputTokens !== undefined ? { inputTokens } : {}), + ...(outputTokens !== undefined ? { outputTokens } : {}), + ...(inputTokens !== undefined && outputTokens !== undefined + ? { totalTokens: inputTokens + outputTokens } + : {}), + ...(providerCachedInputTokens !== undefined ? { cachedInputTokens: providerCachedInputTokens } : {}), + ...(providerReasoningTokens !== undefined ? { reasoningTokens: providerReasoningTokens } : {}), + }; + const latestStep = requestSteps.at(-1); + const latestContextUsed = + latestStep?.inputTokens !== undefined && latestStep.outputTokens !== undefined + ? latestStep.inputTokens + latestStep.outputTokens + : undefined; + const contextUsed = latestContextUsed ?? (usage.inputTokens ?? 0) + (usage.outputTokens ?? 0); const promptPrice = asFiniteNumber(selectedModel?.promptPrice); const completionPrice = asFiniteNumber(selectedModel?.completionPrice); const estimatedCost = @@ -564,6 +628,16 @@ export async function POST(request: Request) { const generatedSeconds = requestSteps.reduce((sum, step) => sum + step.modelDurationMs, 0) / 1000; const outputTokensPerSecond = usage.outputTokens !== undefined && generatedSeconds > 0 ? usage.outputTokens / generatedSeconds : undefined; + const providerGenerationTimeMs = sumCompleteStepMetric(requestSteps, (step) => step.generationTimeMs); + const providerGeneratedTokens = sumCompleteStepMetric(requestSteps, (step) => step.outputTokens); + const providerOutputTokensPerSecond = + providerGenerationTimeMs !== undefined && + providerGenerationTimeMs > 0 && + providerGeneratedTokens !== undefined + ? providerGeneratedTokens / (providerGenerationTimeMs / 1000) + : undefined; + const cacheDiscount = sumCompleteStepMetric(requestSteps, (step) => step.cacheDiscount); + const fallbackCount = sumCompleteStepMetric(requestSteps, (step) => step.fallbackCount); return { usage, @@ -574,13 +648,17 @@ export async function POST(request: Request) { durationMs: completedAt - requestStartedAt, ...(firstOutputAt !== undefined ? { timeToFirstOutputMs: firstOutputAt - requestStartedAt } : {}), ...(outputTokensPerSecond !== undefined ? { outputTokensPerSecond } : {}), + ...(providerOutputTokensPerSecond !== undefined ? { providerOutputTokensPerSecond } : {}), + ...(providerGenerationTimeMs !== undefined ? { generationTimeMs: providerGenerationTimeMs } : {}), ...(contextUsed > 0 ? { contextUsed } : {}), ...(selectedModel?.contextLength && contextUsed > 0 ? { contextUtilization: Math.min(contextUsed / selectedModel.contextLength, 1) } : {}), ...(cost !== undefined ? { cost, costType: hasReportedCost ? "reported" : "estimated" } : {}), ...(hasUpstreamCost ? { upstreamCost } : {}), - finishReason: part.finishReason, + ...(cacheDiscount !== undefined ? { cacheDiscount } : {}), + ...(fallbackCount !== undefined ? { fallbackCount } : {}), + finishReason: latestStep?.finishReason ?? part.finishReason, }, }, }; diff --git a/apps/nextjs/src/components/assistant/assistant-message-metadata.spec.ts b/apps/nextjs/src/components/assistant/assistant-message-metadata.spec.ts index f9f695775..046e9c042 100644 --- a/apps/nextjs/src/components/assistant/assistant-message-metadata.spec.ts +++ b/apps/nextjs/src/components/assistant/assistant-message-metadata.spec.ts @@ -19,7 +19,35 @@ describe("getAssistantTelemetry", () => { contextUtilization: 0.015625, cost: 0.001, costType: "reported", - steps: [{ index: 1, durationMs: 900, modelDurationMs: 800, toolDurationMs: 100 }], + providerOutputTokensPerSecond: 37.9, + generationTimeMs: 1451, + cacheDiscount: 0.00003, + fallbackCount: 1, + steps: [ + { + index: 1, + durationMs: 900, + modelDurationMs: 800, + toolDurationMs: 100, + providerLatencyMs: 964, + generationTimeMs: 1451, + providerOutputTokensPerSecond: 37.9, + inputTokens: 12_777, + outputTokens: 55, + normalizedInputTokens: 12_799, + normalizedOutputTokens: 62, + cachedInputTokens: 2048, + reasoningTokens: 7, + cacheDiscount: 0.00003, + fallbackCount: 1, + routedProvider: "Alibaba", + serviceTier: "default", + dataRegion: "US", + isByok: false, + streamed: true, + cancelled: false, + }, + ], }, }, }), @@ -29,6 +57,19 @@ describe("getAssistantTelemetry", () => { durationMs: 1000, contextLength: 128_000, costType: "reported", + providerOutputTokensPerSecond: 37.9, + generationTimeMs: 1451, + cacheDiscount: 0.00003, + fallbackCount: 1, + steps: [ + expect.objectContaining({ + providerLatencyMs: 964, + inputTokens: 12_777, + cachedInputTokens: 2048, + routedProvider: "Alibaba", + serviceTier: "default", + }), + ], }); }); diff --git a/apps/nextjs/src/components/assistant/assistant-message-metadata.ts b/apps/nextjs/src/components/assistant/assistant-message-metadata.ts index 82e21e5d1..a37df1053 100644 --- a/apps/nextjs/src/components/assistant/assistant-message-metadata.ts +++ b/apps/nextjs/src/components/assistant/assistant-message-metadata.ts @@ -16,12 +16,32 @@ export type AssistantRequestStep = { toolDurationMs: number; timeToFirstOutputMs?: number; outputTokensPerSecond?: number; + providerLatencyMs?: number; + generationTimeMs?: number; + moderationLatencyMs?: number; + providerOutputTokensPerSecond?: number; inputTokens?: number; outputTokens?: number; + normalizedInputTokens?: number; + normalizedOutputTokens?: number; + cachedInputTokens?: number; + reasoningTokens?: number; cost?: number; upstreamCost?: number; + cacheDiscount?: number; generationId?: string; routedProvider?: string; + finishReason?: string; + nativeFinishReason?: string; + serviceTier?: string; + dataRegion?: string; + routerStrategy?: string; + routerRegion?: string; + fallbackCount?: number; + fallbackLatencyMs?: number; + isByok?: boolean; + streamed?: boolean; + cancelled?: boolean; }; export type AssistantRequestTelemetry = { @@ -33,11 +53,16 @@ export type AssistantRequestTelemetry = { durationMs?: number; timeToFirstOutputMs?: number; outputTokensPerSecond?: number; + providerOutputTokensPerSecond?: number; + generationTimeMs?: number; contextLength?: number; contextUsed?: number; contextUtilization?: number; cost?: number; upstreamCost?: number; + cacheDiscount?: number; + fallbackCount?: number; + fallbackLatencyMs?: number; costType?: "reported" | "estimated"; finishReason?: string; steps: AssistantRequestStep[]; @@ -124,20 +149,64 @@ export const getAssistantTelemetry = (metadata: unknown): AssistantRequestTeleme ...(getFiniteNonNegativeNumber(candidate.outputTokensPerSecond) !== undefined ? { outputTokensPerSecond: getFiniteNonNegativeNumber(candidate.outputTokensPerSecond) } : {}), + ...(getFiniteNonNegativeNumber(candidate.providerLatencyMs) !== undefined + ? { providerLatencyMs: getFiniteNonNegativeNumber(candidate.providerLatencyMs) } + : {}), + ...(getFiniteNonNegativeNumber(candidate.generationTimeMs) !== undefined + ? { generationTimeMs: getFiniteNonNegativeNumber(candidate.generationTimeMs) } + : {}), + ...(getFiniteNonNegativeNumber(candidate.moderationLatencyMs) !== undefined + ? { moderationLatencyMs: getFiniteNonNegativeNumber(candidate.moderationLatencyMs) } + : {}), + ...(getFiniteNonNegativeNumber(candidate.providerOutputTokensPerSecond) !== undefined + ? { providerOutputTokensPerSecond: getFiniteNonNegativeNumber(candidate.providerOutputTokensPerSecond) } + : {}), ...(getFiniteNonNegativeNumber(candidate.inputTokens) !== undefined ? { inputTokens: getFiniteNonNegativeNumber(candidate.inputTokens) } : {}), ...(getFiniteNonNegativeNumber(candidate.outputTokens) !== undefined ? { outputTokens: getFiniteNonNegativeNumber(candidate.outputTokens) } : {}), + ...(getFiniteNonNegativeNumber(candidate.normalizedInputTokens) !== undefined + ? { normalizedInputTokens: getFiniteNonNegativeNumber(candidate.normalizedInputTokens) } + : {}), + ...(getFiniteNonNegativeNumber(candidate.normalizedOutputTokens) !== undefined + ? { normalizedOutputTokens: getFiniteNonNegativeNumber(candidate.normalizedOutputTokens) } + : {}), + ...(getFiniteNonNegativeNumber(candidate.cachedInputTokens) !== undefined + ? { cachedInputTokens: getFiniteNonNegativeNumber(candidate.cachedInputTokens) } + : {}), + ...(getFiniteNonNegativeNumber(candidate.reasoningTokens) !== undefined + ? { reasoningTokens: getFiniteNonNegativeNumber(candidate.reasoningTokens) } + : {}), ...(getFiniteNonNegativeNumber(candidate.cost) !== undefined ? { cost: getFiniteNonNegativeNumber(candidate.cost) } : {}), ...(getFiniteNonNegativeNumber(candidate.upstreamCost) !== undefined ? { upstreamCost: getFiniteNonNegativeNumber(candidate.upstreamCost) } : {}), + ...(getFiniteNonNegativeNumber(candidate.cacheDiscount) !== undefined + ? { cacheDiscount: getFiniteNonNegativeNumber(candidate.cacheDiscount) } + : {}), ...(typeof candidate.generationId === "string" ? { generationId: candidate.generationId } : {}), ...(typeof candidate.routedProvider === "string" ? { routedProvider: candidate.routedProvider } : {}), + ...(typeof candidate.finishReason === "string" ? { finishReason: candidate.finishReason } : {}), + ...(typeof candidate.nativeFinishReason === "string" + ? { nativeFinishReason: candidate.nativeFinishReason } + : {}), + ...(typeof candidate.serviceTier === "string" ? { serviceTier: candidate.serviceTier } : {}), + ...(typeof candidate.dataRegion === "string" ? { dataRegion: candidate.dataRegion } : {}), + ...(typeof candidate.routerStrategy === "string" ? { routerStrategy: candidate.routerStrategy } : {}), + ...(typeof candidate.routerRegion === "string" ? { routerRegion: candidate.routerRegion } : {}), + ...(getFiniteNonNegativeNumber(candidate.fallbackCount) !== undefined + ? { fallbackCount: getFiniteNonNegativeNumber(candidate.fallbackCount) } + : {}), + ...(getFiniteNonNegativeNumber(candidate.fallbackLatencyMs) !== undefined + ? { fallbackLatencyMs: getFiniteNonNegativeNumber(candidate.fallbackLatencyMs) } + : {}), + ...(typeof candidate.isByok === "boolean" ? { isByok: candidate.isByok } : {}), + ...(typeof candidate.streamed === "boolean" ? { streamed: candidate.streamed } : {}), + ...(typeof candidate.cancelled === "boolean" ? { cancelled: candidate.cancelled } : {}), }, ]; }) @@ -158,6 +227,12 @@ export const getAssistantTelemetry = (metadata: unknown): AssistantRequestTeleme ...(getFiniteNonNegativeNumber(value.outputTokensPerSecond) !== undefined ? { outputTokensPerSecond: getFiniteNonNegativeNumber(value.outputTokensPerSecond) } : {}), + ...(getFiniteNonNegativeNumber(value.providerOutputTokensPerSecond) !== undefined + ? { providerOutputTokensPerSecond: getFiniteNonNegativeNumber(value.providerOutputTokensPerSecond) } + : {}), + ...(getFiniteNonNegativeNumber(value.generationTimeMs) !== undefined + ? { generationTimeMs: getFiniteNonNegativeNumber(value.generationTimeMs) } + : {}), ...(getFiniteNonNegativeNumber(value.contextLength) !== undefined ? { contextLength: getFiniteNonNegativeNumber(value.contextLength) } : {}), @@ -171,6 +246,15 @@ export const getAssistantTelemetry = (metadata: unknown): AssistantRequestTeleme ...(getFiniteNonNegativeNumber(value.upstreamCost) !== undefined ? { upstreamCost: getFiniteNonNegativeNumber(value.upstreamCost) } : {}), + ...(getFiniteNonNegativeNumber(value.cacheDiscount) !== undefined + ? { cacheDiscount: getFiniteNonNegativeNumber(value.cacheDiscount) } + : {}), + ...(getFiniteNonNegativeNumber(value.fallbackCount) !== undefined + ? { fallbackCount: getFiniteNonNegativeNumber(value.fallbackCount) } + : {}), + ...(getFiniteNonNegativeNumber(value.fallbackLatencyMs) !== undefined + ? { fallbackLatencyMs: getFiniteNonNegativeNumber(value.fallbackLatencyMs) } + : {}), ...(value.costType === "reported" || value.costType === "estimated" ? { costType: value.costType } : {}), ...(typeof value.finishReason === "string" ? { finishReason: value.finishReason } : {}), }; diff --git a/apps/nextjs/src/components/assistant/assistant-panel.tsx b/apps/nextjs/src/components/assistant/assistant-panel.tsx index 5e04172db..3815357ea 100644 --- a/apps/nextjs/src/components/assistant/assistant-panel.tsx +++ b/apps/nextjs/src/components/assistant/assistant-panel.tsx @@ -614,8 +614,8 @@ const formatDuration = (milliseconds: number) => const formatCost = (cost: number) => { if (cost === 0) return "$0"; - if (cost < 0.0001) return `<$0.0001`; - return `$${cost.toFixed(cost < 0.01 ? 4 : 3)}`; + if (cost < 0.00001) return `<$0.00001`; + return `$${cost.toFixed(cost < 0.001 ? 5 : cost < 0.01 ? 4 : 3)}`; }; const getContextColor = (percentage: number) => { @@ -628,9 +628,101 @@ const RequestTelemetry = () => { const t = useScopedI18n("common.assistant"); const [opened, setOpened] = useState(false); const metadata = useAuiState((state) => state.message.metadata); - const telemetry = getAssistantTelemetry(metadata); - const usage = getAssistantUsage(metadata); - if (!telemetry) return null; + const messageId = useAuiState((state) => state.message.id); + const threadId = useAuiState((state) => state.threadListItem.remoteId); + const persistedTelemetry = getAssistantTelemetry(metadata); + const persistedUsage = getAssistantUsage(metadata); + const generationQuery = clientApi.assistant.getGenerationTelemetry.useQuery( + { threadId: threadId ?? "", messageId }, + { + enabled: opened && persistedTelemetry?.provider === "openrouter" && Boolean(threadId), + refetchInterval: (query) => + query.state.data?.complete === false && query.state.dataUpdateCount < 6 ? 1500 : false, + retry: 2, + staleTime: (query) => (query.state.data?.complete ? 5 * 60_000 : 0), + }, + ); + if (!persistedTelemetry) return null; + + const generationById = new Map( + generationQuery.data?.generations.flatMap((generation) => + generation.generationId ? [[generation.generationId, generation] as const] : [], + ) ?? [], + ); + const steps = persistedTelemetry.steps.map((step) => ({ + ...step, + ...(step.generationId ? generationById.get(step.generationId) : undefined), + })); + const sumCompleteMetric = (getValue: (step: (typeof steps)[number]) => number | undefined) => { + if (steps.length === 0) return undefined; + const values = steps.map(getValue); + return values.some((value) => value === undefined) + ? undefined + : (values as number[]).reduce((sum, value) => sum + value, 0); + }; + const providerInputTokens = sumCompleteMetric((step) => step.inputTokens); + const providerOutputTokens = sumCompleteMetric((step) => step.outputTokens); + const providerGenerationTimeMs = sumCompleteMetric((step) => step.generationTimeMs); + const providerOutputTokensPerSecond = + providerOutputTokens !== undefined && providerGenerationTimeMs !== undefined && providerGenerationTimeMs > 0 + ? providerOutputTokens / (providerGenerationTimeMs / 1000) + : undefined; + const latestStep = steps.at(-1); + const providerContextUsed = + latestStep?.inputTokens !== undefined && latestStep.outputTokens !== undefined + ? latestStep.inputTokens + latestStep.outputTokens + : undefined; + const usage = { + ...persistedUsage, + ...(generationQuery.data?.complete && providerInputTokens !== undefined + ? { inputTokens: providerInputTokens } + : {}), + ...(generationQuery.data?.complete && providerOutputTokens !== undefined + ? { outputTokens: providerOutputTokens } + : {}), + ...(generationQuery.data?.complete && providerInputTokens !== undefined && providerOutputTokens !== undefined + ? { totalTokens: providerInputTokens + providerOutputTokens } + : {}), + ...(generationQuery.data?.complete && sumCompleteMetric((step) => step.cachedInputTokens) !== undefined + ? { cachedInputTokens: sumCompleteMetric((step) => step.cachedInputTokens) } + : {}), + ...(generationQuery.data?.complete && sumCompleteMetric((step) => step.reasoningTokens) !== undefined + ? { reasoningTokens: sumCompleteMetric((step) => step.reasoningTokens) } + : {}), + }; + const telemetry = { + ...persistedTelemetry, + steps, + ...(generationQuery.data?.complete && providerGenerationTimeMs !== undefined + ? { generationTimeMs: providerGenerationTimeMs } + : {}), + ...(generationQuery.data?.complete && providerOutputTokensPerSecond !== undefined + ? { providerOutputTokensPerSecond } + : {}), + ...(generationQuery.data?.complete && providerContextUsed !== undefined + ? { + contextUsed: providerContextUsed, + ...(persistedTelemetry.contextLength + ? { contextUtilization: Math.min(providerContextUsed / persistedTelemetry.contextLength, 1) } + : {}), + } + : {}), + ...(generationQuery.data?.complete && sumCompleteMetric((step) => step.cost) !== undefined + ? { cost: sumCompleteMetric((step) => step.cost), costType: "reported" as const } + : {}), + ...(generationQuery.data?.complete && sumCompleteMetric((step) => step.upstreamCost) !== undefined + ? { upstreamCost: sumCompleteMetric((step) => step.upstreamCost) } + : {}), + ...(generationQuery.data?.complete && sumCompleteMetric((step) => step.cacheDiscount) !== undefined + ? { cacheDiscount: sumCompleteMetric((step) => step.cacheDiscount) } + : {}), + ...(generationQuery.data?.complete && sumCompleteMetric((step) => step.fallbackCount) !== undefined + ? { fallbackCount: sumCompleteMetric((step) => step.fallbackCount) } + : {}), + ...(generationQuery.data?.complete && sumCompleteMetric((step) => step.fallbackLatencyMs) !== undefined + ? { fallbackLatencyMs: sumCompleteMetric((step) => step.fallbackLatencyMs) } + : {}), + }; const contextLength = telemetry.contextLength ?? 0; const contextUsed = telemetry.contextUsed ?? 0; @@ -645,6 +737,21 @@ const RequestTelemetry = () => { const contextLabel = hasContextWindow ? `${t("usage.contextWindow")}: ${contextUsed.toLocaleString()} / ${contextLength.toLocaleString()} (${contextPercentage}%)` : `${t("usage.contextWindow")}: ${t("usage.notReported")}`; + const displayedThroughput = telemetry.providerOutputTokensPerSecond ?? telemetry.outputTokensPerSecond; + const hasProviderMetrics = + generationQuery.isFetching || + telemetry.providerOutputTokensPerSecond !== undefined || + telemetry.generationTimeMs !== undefined || + telemetry.cacheDiscount !== undefined || + telemetry.fallbackCount !== undefined || + telemetry.fallbackLatencyMs !== undefined || + telemetry.steps.some( + (step) => + step.providerLatencyMs !== undefined || + step.moderationLatencyMs !== undefined || + step.serviceTier !== undefined || + step.dataRegion !== undefined, + ); return ( @@ -661,9 +768,9 @@ const RequestTelemetry = () => { {formatDuration(telemetry.durationMs)} )} - {telemetry.outputTokensPerSecond !== undefined && ( + {displayedThroughput !== undefined && ( - {telemetry.outputTokensPerSecond.toFixed(1)} tok/s + {displayedThroughput.toFixed(1)} tok/s )} {telemetry.cost !== undefined && ( @@ -676,7 +783,7 @@ const RequestTelemetry = () => { { - + + {t("usage.homarrTiming")} + + +
+ + {t("usage.endToEnd")} + + + {telemetry.durationMs !== undefined ? formatDuration(telemetry.durationMs) : t("usage.notReported")} + +
{t("usage.firstOutput")} @@ -814,6 +932,16 @@ const RequestTelemetry = () => { : t("usage.notReported")}
+
+ + {t("usage.observedThroughput")} + + + {telemetry.outputTokensPerSecond !== undefined + ? `${telemetry.outputTokensPerSecond.toFixed(1)} tok/s` + : t("usage.notReported")} + +
{t("usage.cost")} @@ -847,7 +975,74 @@ const RequestTelemetry = () => {
)} - + + + {hasProviderMetrics && ( + <> + + + {t("usage.providerTiming")} + + {generationQuery.isFetching && generationQuery.data?.complete !== true && ( + + + + {t("usage.loadingProviderDetails")} + + + )} + +
+ + {t("usage.generationTime")} + + + {telemetry.generationTimeMs !== undefined + ? formatDuration(telemetry.generationTimeMs) + : t("usage.notReported")} + +
+
+ + {t("usage.providerThroughput")} + + + {telemetry.providerOutputTokensPerSecond !== undefined + ? `${telemetry.providerOutputTokensPerSecond.toFixed(1)} tok/s` + : t("usage.notReported")} + +
+
+ + {t("usage.fallbacks")} + + + {telemetry.fallbackCount?.toLocaleString() ?? t("usage.notReported")} + +
+ {telemetry.fallbackLatencyMs !== undefined && ( +
+ + {t("usage.fallbackLatency")} + + + {formatDuration(telemetry.fallbackLatencyMs)} + +
+ )} + {telemetry.cacheDiscount !== undefined && ( +
+ + {t("usage.cacheDiscount")} + + + {formatCost(telemetry.cacheDiscount)} + +
+ )} + + + )} {telemetry.steps.length > 0 && ( @@ -863,12 +1058,86 @@ const RequestTelemetry = () => { {formatDuration(step.durationMs)} - {step.outputTokensPerSecond !== undefined - ? ` · ${step.outputTokensPerSecond.toFixed(1)} tok/s` - : ""} + {step.providerOutputTokensPerSecond !== undefined + ? ` · ${step.providerOutputTokensPerSecond.toFixed(1)} tok/s` + : step.outputTokensPerSecond !== undefined + ? ` · ${step.outputTokensPerSecond.toFixed(1)} tok/s` + : ""} {step.cost !== undefined ? ` · ${formatCost(step.cost)}` : ""} + {(step.providerLatencyMs !== undefined || + step.generationTimeMs !== undefined || + step.moderationLatencyMs !== undefined || + step.fallbackCount !== undefined || + step.fallbackLatencyMs !== undefined) && ( + + {[ + step.providerLatencyMs !== undefined + ? `${t("usage.providerLatency")} ${formatDuration(step.providerLatencyMs)}` + : undefined, + step.generationTimeMs !== undefined + ? `${t("usage.generationTime")} ${formatDuration(step.generationTimeMs)}` + : undefined, + step.moderationLatencyMs !== undefined + ? `${t("usage.moderationLatency")} ${formatDuration(step.moderationLatencyMs)}` + : undefined, + step.fallbackCount !== undefined + ? `${t("usage.fallbacks")} ${step.fallbackCount.toLocaleString()}` + : undefined, + step.fallbackLatencyMs !== undefined + ? `${t("usage.fallbackLatency")} ${formatDuration(step.fallbackLatencyMs)}` + : undefined, + ] + .filter(Boolean) + .join(" · ")} + + )} + {(step.inputTokens !== undefined || step.outputTokens !== undefined) && ( + + {[ + step.inputTokens !== undefined + ? `${step.inputTokens.toLocaleString()} ${t("usage.inputShort")}` + : undefined, + step.outputTokens !== undefined + ? `${step.outputTokens.toLocaleString()} ${t("usage.outputShort")}` + : undefined, + step.cachedInputTokens !== undefined + ? `${step.cachedInputTokens.toLocaleString()} ${t("usage.cachedShort")}` + : undefined, + step.reasoningTokens !== undefined + ? `${step.reasoningTokens.toLocaleString()} ${t("usage.reasoningShort")}` + : undefined, + ] + .filter(Boolean) + .join(" · ")} + + )} + {(step.serviceTier || + step.dataRegion || + step.routerStrategy || + step.routerRegion || + step.isByok) && ( + + {[ + step.routerStrategy, + step.routerRegion, + step.serviceTier, + step.dataRegion, + step.isByok ? t("usage.byok") : undefined, + ] + .filter(Boolean) + .join(" · ")} + + )} + {(step.normalizedInputTokens !== undefined || step.normalizedOutputTokens !== undefined) && + (step.normalizedInputTokens !== step.inputTokens || + step.normalizedOutputTokens !== step.outputTokens) && ( + + {t("usage.normalizedTokens")}: {step.normalizedInputTokens?.toLocaleString() ?? "–"} /{" "} + {step.normalizedOutputTokens?.toLocaleString() ?? "–"} + + )} {step.generationId && ( {t("usage.generation")}: {step.generationId} diff --git a/packages/api/src/assistant-generation-telemetry.ts b/packages/api/src/assistant-generation-telemetry.ts new file mode 100644 index 000000000..6a4106457 --- /dev/null +++ b/packages/api/src/assistant-generation-telemetry.ts @@ -0,0 +1,140 @@ +import { fetchWithTrustedCertificatesAsync } from "@homarr/core/infrastructure/http"; + +export type OpenRouterGenerationTelemetry = { + generationId?: string; + routedProvider?: string; + providerLatencyMs?: number; + generationTimeMs?: number; + moderationLatencyMs?: number; + providerOutputTokensPerSecond?: number; + inputTokens?: number; + outputTokens?: number; + normalizedInputTokens?: number; + normalizedOutputTokens?: number; + cachedInputTokens?: number; + reasoningTokens?: number; + cost?: number; + upstreamCost?: number; + cacheDiscount?: number; + finishReason?: string; + nativeFinishReason?: string; + serviceTier?: string; + dataRegion?: string; + fallbackCount?: number; + fallbackLatencyMs?: number; + isByok?: boolean; + streamed?: boolean; + cancelled?: boolean; +}; + +const asRecord = (value: unknown): Record | null => + typeof value === "object" && value !== null && !Array.isArray(value) ? (value as Record) : null; + +const asFiniteNonNegativeNumber = (value: unknown) => { + const number = typeof value === "number" ? value : typeof value === "string" ? Number(value) : Number.NaN; + return Number.isFinite(number) && number >= 0 ? number : undefined; +}; + +const asString = (value: unknown) => (typeof value === "string" && value.length > 0 ? value : undefined); +const asBoolean = (value: unknown) => (typeof value === "boolean" ? value : undefined); + +export const parseOpenRouterGenerationTelemetry = (value: unknown): OpenRouterGenerationTelemetry | null => { + const root = asRecord(value); + const data = asRecord(root?.data) ?? root; + if (!data) return null; + + const normalizedInputTokens = asFiniteNonNegativeNumber(data.tokens_prompt); + const normalizedOutputTokens = asFiniteNonNegativeNumber(data.tokens_completion); + const inputTokens = asFiniteNonNegativeNumber(data.native_tokens_prompt) ?? normalizedInputTokens; + const outputTokens = asFiniteNonNegativeNumber(data.native_tokens_completion) ?? normalizedOutputTokens; + const generationTimeMs = asFiniteNonNegativeNumber(data.generation_time); + const providerResponses = Array.isArray(data.provider_responses) ? data.provider_responses : undefined; + const parsedProviderResponses = providerResponses?.map(asRecord).filter((response) => response !== null); + const failedProviderResponses = parsedProviderResponses?.filter((response) => { + const status = asFiniteNonNegativeNumber(response.status); + return status !== undefined && (status < 200 || status >= 300); + }); + const fallbackCount = providerResponses + ? failedProviderResponses && failedProviderResponses.length > 0 + ? failedProviderResponses.length + : Math.max(providerResponses.length - 1, 0) + : undefined; + const fallbackLatencies = failedProviderResponses?.map((response) => asFiniteNonNegativeNumber(response.latency)); + const fallbackLatencyMs = + fallbackLatencies && fallbackLatencies.length > 0 && fallbackLatencies.every((latency) => latency !== undefined) + ? (fallbackLatencies as number[]).reduce((sum, latency) => sum + latency, 0) + : undefined; + const providerOutputTokensPerSecond = + outputTokens !== undefined && generationTimeMs !== undefined && generationTimeMs > 0 + ? outputTokens / (generationTimeMs / 1000) + : undefined; + const generationId = asString(data.id); + const routedProvider = asString(data.provider_name); + const finishReason = asString(data.finish_reason); + const nativeFinishReason = asString(data.native_finish_reason); + const serviceTier = asString(data.service_tier); + const dataRegion = asString(data.data_region); + + const telemetry: OpenRouterGenerationTelemetry = { + ...(generationId ? { generationId } : {}), + ...(routedProvider ? { routedProvider } : {}), + ...(asFiniteNonNegativeNumber(data.latency) !== undefined + ? { providerLatencyMs: asFiniteNonNegativeNumber(data.latency) } + : {}), + ...(generationTimeMs !== undefined ? { generationTimeMs } : {}), + ...(asFiniteNonNegativeNumber(data.moderation_latency) !== undefined + ? { moderationLatencyMs: asFiniteNonNegativeNumber(data.moderation_latency) } + : {}), + ...(providerOutputTokensPerSecond !== undefined ? { providerOutputTokensPerSecond } : {}), + ...(inputTokens !== undefined ? { inputTokens } : {}), + ...(outputTokens !== undefined ? { outputTokens } : {}), + ...(normalizedInputTokens !== undefined ? { normalizedInputTokens } : {}), + ...(normalizedOutputTokens !== undefined ? { normalizedOutputTokens } : {}), + ...(asFiniteNonNegativeNumber(data.native_tokens_cached) !== undefined + ? { cachedInputTokens: asFiniteNonNegativeNumber(data.native_tokens_cached) } + : {}), + ...(asFiniteNonNegativeNumber(data.native_tokens_reasoning) !== undefined + ? { reasoningTokens: asFiniteNonNegativeNumber(data.native_tokens_reasoning) } + : {}), + ...(asFiniteNonNegativeNumber(data.total_cost) !== undefined + ? { cost: asFiniteNonNegativeNumber(data.total_cost) } + : {}), + ...(asFiniteNonNegativeNumber(data.upstream_inference_cost) !== undefined + ? { upstreamCost: asFiniteNonNegativeNumber(data.upstream_inference_cost) } + : {}), + ...(asFiniteNonNegativeNumber(data.cache_discount) !== undefined + ? { cacheDiscount: asFiniteNonNegativeNumber(data.cache_discount) } + : {}), + ...(finishReason ? { finishReason } : {}), + ...(nativeFinishReason ? { nativeFinishReason } : {}), + ...(serviceTier ? { serviceTier } : {}), + ...(dataRegion ? { dataRegion } : {}), + ...(fallbackCount !== undefined ? { fallbackCount } : {}), + ...(fallbackLatencyMs !== undefined ? { fallbackLatencyMs } : {}), + ...(asBoolean(data.is_byok) !== undefined ? { isByok: asBoolean(data.is_byok) } : {}), + ...(asBoolean(data.streamed) !== undefined ? { streamed: asBoolean(data.streamed) } : {}), + ...(asBoolean(data.cancelled) !== undefined ? { cancelled: asBoolean(data.cancelled) } : {}), + }; + + return Object.keys(telemetry).length > 0 ? telemetry : null; +}; + +export const fetchOpenRouterGenerationTelemetryAsync = async ({ + baseUrl, + generationId, + headers, +}: { + baseUrl: string; + generationId: string; + headers: Record; +}): Promise => { + try { + const endpoint = new URL(`${baseUrl.replace(/\/$/, "")}/generation`); + endpoint.searchParams.set("id", generationId); + const response = await fetchWithTrustedCertificatesAsync(endpoint, { headers, timeout: 3000 }); + if (!response.ok) return null; + return parseOpenRouterGenerationTelemetry(await response.json()); + } catch { + return null; + } +}; diff --git a/packages/api/src/assistant.ts b/packages/api/src/assistant.ts index 6468bf426..c7731c22f 100644 --- a/packages/api/src/assistant.ts +++ b/packages/api/src/assistant.ts @@ -4,3 +4,4 @@ export { getSelectedModelDetailsAsync, type AssistantContextEntity, } from "./router/assistant"; +export type { OpenRouterGenerationTelemetry } from "./assistant-generation-telemetry"; diff --git a/packages/api/src/router/assistant.ts b/packages/api/src/router/assistant.ts index ee75b972c..a56abcc20 100644 --- a/packages/api/src/router/assistant.ts +++ b/packages/api/src/router/assistant.ts @@ -25,8 +25,9 @@ import { resolveAssistantModelId, } from "@homarr/definitions"; -import { createTRPCRouter, permissionRequiredProcedure, protectedProcedure } from "../trpc"; import type { createTRPCContext } from "../trpc"; +import { fetchOpenRouterGenerationTelemetryAsync } from "../assistant-generation-telemetry"; +import { createTRPCRouter, permissionRequiredProcedure, protectedProcedure } from "../trpc"; import { boardRouter } from "./board"; const adminProcedure = permissionRequiredProcedure.requiresPermission("admin"); @@ -159,6 +160,25 @@ const getProviderHeaders = (configuration: AssistantConfiguration) => { return { ...headers, ...decryptCustomHeaders(configuration.encryptedHeaders) }; }; +const getGenerationIdsFromMessageContent = (content: unknown) => { + if (!content || typeof content !== "object" || Array.isArray(content)) return []; + const metadata = (content as { metadata?: unknown }).metadata; + if (!metadata || typeof metadata !== "object" || Array.isArray(metadata)) return []; + const custom = (metadata as { custom?: unknown }).custom; + if (!custom || typeof custom !== "object" || Array.isArray(custom)) return []; + const telemetry = (custom as { telemetry?: unknown }).telemetry; + if (!telemetry || typeof telemetry !== "object" || Array.isArray(telemetry)) return []; + const steps = (telemetry as { steps?: unknown }).steps; + if (!Array.isArray(steps)) return []; + return steps + .flatMap((step) => { + if (!step || typeof step !== "object" || Array.isArray(step)) return []; + const generationId = (step as { generationId?: unknown }).generationId; + return typeof generationId === "string" && /^gen-[A-Za-z0-9_-]{1,128}$/.test(generationId) ? [generationId] : []; + }) + .slice(0, 8); +}; + const fetchModelsAsync = async (configuration: AssistantConfiguration) => { if (!configuration.modelDiscoveryPath) { throw new TRPCError({ @@ -674,6 +694,43 @@ export const assistantRouter = createTRPCRouter({ }; }), + getGenerationTelemetry: protectedProcedure + .input( + z.object({ + threadId: z.string().max(64), + messageId: z.string().min(1).max(128), + }), + ) + .query(async ({ ctx, input }) => { + const thread = await ownedThreadAsync(ctx.db, input.threadId, ctx.session.user.id); + const [message, configuration] = await Promise.all([ + ctx.db.query.assistantMessages.findFirst({ + where: and(eq(assistantMessages.id, input.messageId), eq(assistantMessages.threadId, thread.id)), + }), + getConfigurationAsync(ctx.db), + ]); + if (!message) throw new TRPCError({ code: "NOT_FOUND", message: "Message not found." }); + if (configuration?.provider !== "openrouter" || !configuration.encryptedApiKey) { + return { complete: true, generations: [] }; + } + + const generationIds = getGenerationIdsFromMessageContent(parse(message.content)); + if (generationIds.length === 0) return { complete: true, generations: [] }; + const generations = await Promise.all( + generationIds.map((generationId) => + fetchOpenRouterGenerationTelemetryAsync({ + baseUrl: configuration.baseUrl, + generationId, + headers: getProviderHeaders(configuration), + }), + ), + ); + return { + complete: generations.every((generation) => generation !== null), + generations: generations.filter((generation) => generation !== null), + }; + }), + renameThread: protectedProcedure .input(z.object({ threadId: z.string().max(64), title: z.string().trim().min(1).max(120) })) .mutation(async ({ ctx, input }) => { diff --git a/packages/api/src/test/assistant-generation-telemetry.spec.ts b/packages/api/src/test/assistant-generation-telemetry.spec.ts new file mode 100644 index 000000000..859b860a7 --- /dev/null +++ b/packages/api/src/test/assistant-generation-telemetry.spec.ts @@ -0,0 +1,66 @@ +import { describe, expect, test, vi } from "vitest"; + +vi.mock("@homarr/core/infrastructure/http", () => ({ fetchWithTrustedCertificatesAsync: vi.fn() })); + +import { parseOpenRouterGenerationTelemetry } from "../assistant-generation-telemetry"; + +describe("parseOpenRouterGenerationTelemetry", () => { + test("uses provider-native tokens for context and throughput", () => { + expect( + parseOpenRouterGenerationTelemetry({ + data: { + id: "gen-1", + provider_name: "Alibaba", + latency: 964, + generation_time: 1451, + moderation_latency: 12, + tokens_prompt: 13_343, + tokens_completion: 62, + native_tokens_prompt: 12_777, + native_tokens_completion: 55, + native_tokens_cached: 2048, + native_tokens_reasoning: 7, + total_cost: 0.00041, + upstream_inference_cost: 0.00039, + cache_discount: 0.00003, + finish_reason: "stop", + native_finish_reason: "stop", + service_tier: "default", + data_region: "US", + provider_responses: [ + { status: 429, latency: 620 }, + { status: 200, latency: 964 }, + ], + is_byok: false, + streamed: true, + cancelled: false, + }, + }), + ).toMatchObject({ + generationId: "gen-1", + routedProvider: "Alibaba", + providerLatencyMs: 964, + generationTimeMs: 1451, + providerOutputTokensPerSecond: 37.90489317711923, + inputTokens: 12_777, + outputTokens: 55, + normalizedInputTokens: 13_343, + normalizedOutputTokens: 62, + cachedInputTokens: 2048, + reasoningTokens: 7, + cost: 0.00041, + upstreamCost: 0.00039, + cacheDiscount: 0.00003, + fallbackCount: 1, + fallbackLatencyMs: 620, + serviceTier: "default", + dataRegion: "US", + }); + }); + + test("returns null for malformed generation data", () => { + expect(parseOpenRouterGenerationTelemetry(null)).toBeNull(); + expect(parseOpenRouterGenerationTelemetry({ data: [] })).toBeNull(); + expect(parseOpenRouterGenerationTelemetry({ data: { latency: -1, total_cost: "invalid" } })).toBeNull(); + }); +}); diff --git a/packages/translation/src/lang/en.json b/packages/translation/src/lang/en.json index 9aff59fc2..49cb88f60 100644 --- a/packages/translation/src/lang/en.json +++ b/packages/translation/src/lang/en.json @@ -1616,7 +1616,25 @@ "cached": "Cached input", "cacheWrite": "Cache write", "reasoning": "Reasoning", + "inputShort": "in", + "outputShort": "out", + "cachedShort": "cached", + "reasoningShort": "reasoning", "firstOutput": "First output", + "homarrTiming": "Homarr observed", + "endToEnd": "End to end", + "observedThroughput": "Observed throughput", + "providerTiming": "Provider reported", + "loadingProviderDetails": "Loading generation details…", + "providerLatency": "Provider latency", + "generationTime": "Generation time", + "moderationLatency": "Moderation", + "providerThroughput": "Provider throughput", + "fallbacks": "Fallbacks", + "fallbackLatency": "Fallback latency", + "cacheDiscount": "Cache discount", + "normalizedTokens": "OpenRouter normalized input / output", + "byok": "BYOK", "cost": "Cost", "reported": "Reported by provider", "estimated": "Estimated from model pricing",