feat: enrich assistant request telemetry

This commit is contained in:
ajnart committed 2026-08-02 08:51:48 +02:00
1 parent d562706c8b
commit 04520cf67c
10 files changed
+798 -39

No files matched your search

@@ -614,8 +614,8 @@ const formatDuration = (milliseconds: number) =>
const formatCost = (cost: number) => {
if (cost === 0) return "$0";
if (cost < 0.0001) return `<$0.0001`;
return `$${cost.toFixed(cost < 0.01 ? 4 : 3)}`;
if (cost < 0.00001) return `<$0.00001`;
return `$${cost.toFixed(cost < 0.001 ? 5 : cost < 0.01 ? 4 : 3)}`;
};
const getContextColor = (percentage: number) => {
@@ -628,9 +628,101 @@ const RequestTelemetry = () => {
const t = useScopedI18n("common.assistant");
const [opened, setOpened] = useState(false);
const metadata = useAuiState((state) => state.message.metadata);
const telemetry = getAssistantTelemetry(metadata);
const usage = getAssistantUsage(metadata);
if (!telemetry) return null;
const messageId = useAuiState((state) => state.message.id);
const threadId = useAuiState((state) => state.threadListItem.remoteId);
const persistedTelemetry = getAssistantTelemetry(metadata);
const persistedUsage = getAssistantUsage(metadata);
const generationQuery = clientApi.assistant.getGenerationTelemetry.useQuery(
{ threadId: threadId ?? "", messageId },
{
enabled: opened && persistedTelemetry?.provider === "openrouter" && Boolean(threadId),
refetchInterval: (query) =>
query.state.data?.complete === false && query.state.dataUpdateCount < 6 ? 1500 : false,
retry: 2,
staleTime: (query) => (query.state.data?.complete ? 5 * 60_000 : 0),
},
);
if (!persistedTelemetry) return null;
const generationById = new Map(
generationQuery.data?.generations.flatMap((generation) =>
generation.generationId ? [[generation.generationId, generation] as const] : [],
) ?? [],
);
const steps = persistedTelemetry.steps.map((step) => ({
...step,
...(step.generationId ? generationById.get(step.generationId) : undefined),
}));
const sumCompleteMetric = (getValue: (step: (typeof steps)[number]) => number | undefined) => {
if (steps.length === 0) return undefined;
const values = steps.map(getValue);
return values.some((value) => value === undefined)
? undefined
: (values as number[]).reduce((sum, value) => sum + value, 0);
};
const providerInputTokens = sumCompleteMetric((step) => step.inputTokens);
const providerOutputTokens = sumCompleteMetric((step) => step.outputTokens);
const providerGenerationTimeMs = sumCompleteMetric((step) => step.generationTimeMs);
const providerOutputTokensPerSecond =
providerOutputTokens !== undefined && providerGenerationTimeMs !== undefined && providerGenerationTimeMs > 0
? providerOutputTokens / (providerGenerationTimeMs / 1000)
: undefined;
const latestStep = steps.at(-1);
const providerContextUsed =
latestStep?.inputTokens !== undefined && latestStep.outputTokens !== undefined
? latestStep.inputTokens + latestStep.outputTokens
: undefined;
const usage = {
...persistedUsage,
...(generationQuery.data?.complete && providerInputTokens !== undefined
? { inputTokens: providerInputTokens }
: {}),
...(generationQuery.data?.complete && providerOutputTokens !== undefined
? { outputTokens: providerOutputTokens }
: {}),
...(generationQuery.data?.complete && providerInputTokens !== undefined && providerOutputTokens !== undefined
? { totalTokens: providerInputTokens + providerOutputTokens }
: {}),
...(generationQuery.data?.complete && sumCompleteMetric((step) => step.cachedInputTokens) !== undefined
? { cachedInputTokens: sumCompleteMetric((step) => step.cachedInputTokens) }
: {}),
...(generationQuery.data?.complete && sumCompleteMetric((step) => step.reasoningTokens) !== undefined
? { reasoningTokens: sumCompleteMetric((step) => step.reasoningTokens) }
: {}),
};
const telemetry = {
...persistedTelemetry,
steps,
...(generationQuery.data?.complete && providerGenerationTimeMs !== undefined
? { generationTimeMs: providerGenerationTimeMs }
: {}),
...(generationQuery.data?.complete && providerOutputTokensPerSecond !== undefined
? { providerOutputTokensPerSecond }
: {}),
...(generationQuery.data?.complete && providerContextUsed !== undefined
? {
contextUsed: providerContextUsed,
...(persistedTelemetry.contextLength
? { contextUtilization: Math.min(providerContextUsed / persistedTelemetry.contextLength, 1) }
: {}),
}
: {}),
...(generationQuery.data?.complete && sumCompleteMetric((step) => step.cost) !== undefined
? { cost: sumCompleteMetric((step) => step.cost), costType: "reported" as const }
: {}),
...(generationQuery.data?.complete && sumCompleteMetric((step) => step.upstreamCost) !== undefined
? { upstreamCost: sumCompleteMetric((step) => step.upstreamCost) }
: {}),
...(generationQuery.data?.complete && sumCompleteMetric((step) => step.cacheDiscount) !== undefined
? { cacheDiscount: sumCompleteMetric((step) => step.cacheDiscount) }
: {}),
...(generationQuery.data?.complete && sumCompleteMetric((step) => step.fallbackCount) !== undefined
? { fallbackCount: sumCompleteMetric((step) => step.fallbackCount) }
: {}),
...(generationQuery.data?.complete && sumCompleteMetric((step) => step.fallbackLatencyMs) !== undefined
? { fallbackLatencyMs: sumCompleteMetric((step) => step.fallbackLatencyMs) }
: {}),
};
const contextLength = telemetry.contextLength ?? 0;
const contextUsed = telemetry.contextUsed ?? 0;
@@ -645,6 +737,21 @@ const RequestTelemetry = () => {
const contextLabel = hasContextWindow
? `${t("usage.contextWindow")}: ${contextUsed.toLocaleString()} / ${contextLength.toLocaleString()} (${contextPercentage}%)`
: `${t("usage.contextWindow")}: ${t("usage.notReported")}`;
const displayedThroughput = telemetry.providerOutputTokensPerSecond ?? telemetry.outputTokensPerSecond;
const hasProviderMetrics =
generationQuery.isFetching ||
telemetry.providerOutputTokensPerSecond !== undefined ||
telemetry.generationTimeMs !== undefined ||
telemetry.cacheDiscount !== undefined ||
telemetry.fallbackCount !== undefined ||
telemetry.fallbackLatencyMs !== undefined ||
telemetry.steps.some(
(step) =>
step.providerLatencyMs !== undefined ||
step.moderationLatencyMs !== undefined ||
step.serviceTier !== undefined ||
step.dataRegion !== undefined,
);
return (
<Box className={classes.telemetry}>
@@ -661,9 +768,9 @@ const RequestTelemetry = () => {
{formatDuration(telemetry.durationMs)}
</Text>
)}
{telemetry.outputTokensPerSecond !== undefined && (
{displayedThroughput !== undefined && (
<Text size="xs" c="dimmed">
{telemetry.outputTokensPerSecond.toFixed(1)} tok/s
{displayedThroughput.toFixed(1)} tok/s
</Text>
)}
{telemetry.cost !== undefined && (
@@ -676,7 +783,7 @@ const RequestTelemetry = () => {
<Popover
opened={opened}
onChange={setOpened}
width="min(20rem, calc(100vw - 1.5rem))"
width="min(23rem, calc(100vw - 1.5rem))"
position="bottom-end"
shadow="md"
withArrow
@@ -803,7 +910,18 @@ const RequestTelemetry = () => {
</Box>
<Divider />
<Group gap="lg" align="flex-start">
<Text size="xs" fw={700} tt="uppercase" c="dimmed">
{t("usage.homarrTiming")}
</Text>
<Box className={classes.usageGrid}>
<div>
<Text size="xs" c="dimmed">
{t("usage.endToEnd")}
</Text>
<Text size="sm" fw={600}>
{telemetry.durationMs !== undefined ? formatDuration(telemetry.durationMs) : t("usage.notReported")}
</Text>
</div>
<div>
<Text size="xs" c="dimmed">
{t("usage.firstOutput")}
@@ -814,6 +932,16 @@ const RequestTelemetry = () => {
: t("usage.notReported")}
</Text>
</div>
<div>
<Text size="xs" c="dimmed">
{t("usage.observedThroughput")}
</Text>
<Text size="sm" fw={600}>
{telemetry.outputTokensPerSecond !== undefined
? `${telemetry.outputTokensPerSecond.toFixed(1)} tok/s`
: t("usage.notReported")}
</Text>
</div>
<div>
<Text size="xs" c="dimmed">
{t("usage.cost")}
@@ -847,7 +975,74 @@ const RequestTelemetry = () => {
</Text>
</div>
)}
</Group>
</Box>
{hasProviderMetrics && (
<>
<Divider />
<Text size="xs" fw={700} tt="uppercase" c="dimmed">
{t("usage.providerTiming")}
</Text>
{generationQuery.isFetching && generationQuery.data?.complete !== true && (
<Group gap="xs">
<Loader size="xs" type="bars" />
<Text size="xs" c="dimmed">
{t("usage.loadingProviderDetails")}
</Text>
</Group>
)}
<Box className={classes.usageGrid}>
<div>
<Text size="xs" c="dimmed">
{t("usage.generationTime")}
</Text>
<Text size="sm" fw={600}>
{telemetry.generationTimeMs !== undefined
? formatDuration(telemetry.generationTimeMs)
: t("usage.notReported")}
</Text>
</div>
<div>
<Text size="xs" c="dimmed">
{t("usage.providerThroughput")}
</Text>
<Text size="sm" fw={600}>
{telemetry.providerOutputTokensPerSecond !== undefined
? `${telemetry.providerOutputTokensPerSecond.toFixed(1)} tok/s`
: t("usage.notReported")}
</Text>
</div>
<div>
<Text size="xs" c="dimmed">
{t("usage.fallbacks")}
</Text>
<Text size="sm" fw={600}>
{telemetry.fallbackCount?.toLocaleString() ?? t("usage.notReported")}
</Text>
</div>
{telemetry.fallbackLatencyMs !== undefined && (
<div>
<Text size="xs" c="dimmed">
{t("usage.fallbackLatency")}
</Text>
<Text size="sm" fw={600}>
{formatDuration(telemetry.fallbackLatencyMs)}
</Text>
</div>
)}
{telemetry.cacheDiscount !== undefined && (
<div>
<Text size="xs" c="dimmed">
{t("usage.cacheDiscount")}
</Text>
<Text size="sm" fw={600}>
{formatCost(telemetry.cacheDiscount)}
</Text>
</div>
)}
</Box>
</>
)}
{telemetry.steps.length > 0 && (
<Stack gap={4}>
@@ -863,12 +1058,86 @@ const RequestTelemetry = () => {
</Text>
<Text size="xs" c="dimmed">
{formatDuration(step.durationMs)}
{step.outputTokensPerSecond !== undefined
? ` · ${step.outputTokensPerSecond.toFixed(1)} tok/s`
: ""}
{step.providerOutputTokensPerSecond !== undefined
? ` · ${step.providerOutputTokensPerSecond.toFixed(1)} tok/s`
: step.outputTokensPerSecond !== undefined
? ` · ${step.outputTokensPerSecond.toFixed(1)} tok/s`
: ""}
{step.cost !== undefined ? ` · ${formatCost(step.cost)}` : ""}
</Text>
</Group>
{(step.providerLatencyMs !== undefined ||
step.generationTimeMs !== undefined ||
step.moderationLatencyMs !== undefined ||
step.fallbackCount !== undefined ||
step.fallbackLatencyMs !== undefined) && (
<Text size="xs" c="dimmed">
{[
step.providerLatencyMs !== undefined
? `${t("usage.providerLatency")} ${formatDuration(step.providerLatencyMs)}`
: undefined,
step.generationTimeMs !== undefined
? `${t("usage.generationTime")} ${formatDuration(step.generationTimeMs)}`
: undefined,
step.moderationLatencyMs !== undefined
? `${t("usage.moderationLatency")} ${formatDuration(step.moderationLatencyMs)}`
: undefined,
step.fallbackCount !== undefined
? `${t("usage.fallbacks")} ${step.fallbackCount.toLocaleString()}`
: undefined,
step.fallbackLatencyMs !== undefined
? `${t("usage.fallbackLatency")} ${formatDuration(step.fallbackLatencyMs)}`
: undefined,
]
.filter(Boolean)
.join(" · ")}
</Text>
)}
{(step.inputTokens !== undefined || step.outputTokens !== undefined) && (
<Text size="xs" c="dimmed">
{[
step.inputTokens !== undefined
? `${step.inputTokens.toLocaleString()} ${t("usage.inputShort")}`
: undefined,
step.outputTokens !== undefined
? `${step.outputTokens.toLocaleString()} ${t("usage.outputShort")}`
: undefined,
step.cachedInputTokens !== undefined
? `${step.cachedInputTokens.toLocaleString()} ${t("usage.cachedShort")}`
: undefined,
step.reasoningTokens !== undefined
? `${step.reasoningTokens.toLocaleString()} ${t("usage.reasoningShort")}`
: undefined,
]
.filter(Boolean)
.join(" · ")}
</Text>
)}
{(step.serviceTier ||
step.dataRegion ||
step.routerStrategy ||
step.routerRegion ||
step.isByok) && (
<Text size="xs" c="dimmed">
{[
step.routerStrategy,
step.routerRegion,
step.serviceTier,
step.dataRegion,
step.isByok ? t("usage.byok") : undefined,
]
.filter(Boolean)
.join(" · ")}
</Text>
)}
{(step.normalizedInputTokens !== undefined || step.normalizedOutputTokens !== undefined) &&
(step.normalizedInputTokens !== step.inputTokens ||
step.normalizedOutputTokens !== step.outputTokens) && (
<Text size="xs" c="dimmed">
{t("usage.normalizedTokens")}: {step.normalizedInputTokens?.toLocaleString() ?? "–"} /{" "}
{step.normalizedOutputTokens?.toLocaleString() ?? "–"}
</Text>
)}
{step.generationId && (
<Text size="xs" c="dimmed" className={classes.generationId} title={step.generationId}>
{t("usage.generation")}: {step.generationId}