Merge commit '6cca757493385f7a2c42f36c170c51b37fd44e9d' into feat/widget-scaling-fix
This commit is contained in:
commit
56cfe3992b
33 files changed
+2071
-4
No files matched your search
@@ -497,6 +497,13 @@ export const userRouter = createTRPCRouter({
|
||||
mcp: { enabled: true, description: "Delete a user by ID. REQUIRED: userId (string)" },
|
||||
})
|
||||
.mutation(async ({ input, ctx }) => {
|
||||
if (isDemoMode) {
|
||||
throw new TRPCError({
|
||||
code: "FORBIDDEN",
|
||||
message: "User deletion is disabled in demo mode",
|
||||
});
|
||||
}
|
||||
|
||||
// Only admins and user itself can delete a user
|
||||
if (ctx.session.user.id !== input.userId && !ctx.session.user.permissions.includes("admin")) {
|
||||
throw new TRPCError({
|
||||
@@ -519,6 +526,13 @@ export const userRouter = createTRPCRouter({
|
||||
},
|
||||
})
|
||||
.mutation(async ({ ctx, input }) => {
|
||||
if (isDemoMode) {
|
||||
throw new TRPCError({
|
||||
code: "FORBIDDEN",
|
||||
message: "Password changes are disabled in demo mode",
|
||||
});
|
||||
}
|
||||
|
||||
const user = ctx.session.user;
|
||||
// Only admins can change other users' passwords
|
||||
if (!user.permissions.includes("admin") && user.id !== input.userId) {
|
||||
|
||||
@@ -46,4 +46,5 @@ export const widgetRouter = createTRPCRouter({
|
||||
customApi: lazy(() => import("./custom-api").then((mod) => mod.customApiRouter)),
|
||||
secrets: lazy(() => import("./widget-secrets").then((mod) => mod.widgetSecretsRouter)),
|
||||
wud: lazy(() => import("./wud").then((mod) => mod.wudRouter)),
|
||||
llamacpp: lazy(() => import("./llamacpp").then((mod) => mod.llamacppRouter)),
|
||||
});
|
||||
@@ -0,0 +1,29 @@
|
||||
import { llamacppStatsRequestHandler } from "@homarr/request-handler/llamacpp";
|
||||
import { mockWidgetData } from "@homarr/integrations";
|
||||
|
||||
import { createOneWidgetIntegrationMiddleware } from "../../middlewares/integration";
|
||||
import { createTRPCRouter, publicProcedure } from "../../trpc";
|
||||
|
||||
export const llamacppRouter = createTRPCRouter({
|
||||
getStats: publicProcedure
|
||||
.meta({
|
||||
mcp: {
|
||||
enabled: true,
|
||||
description:
|
||||
"Returns health, loaded model and Prometheus metrics (generation speed, token counts, request queue) from a llama.cpp (llama-server) integration. REQUIRED: integrationId from integration_all. The caller needs query permission for that integration.",
|
||||
},
|
||||
})
|
||||
.concat(createOneWidgetIntegrationMiddleware("query", "llamacpp"))
|
||||
.query(async ({ ctx }) => {
|
||||
if (ctx.integration.kind === "mock") {
|
||||
return { stats: mockWidgetData.llamacpp, updatedAt: new Date(mockWidgetData.timestamp) };
|
||||
}
|
||||
const handler = llamacppStatsRequestHandler.handler({ ...ctx.integration, kind: "llamacpp" }, {});
|
||||
const { data, timestamp } = await handler.getDataAsync();
|
||||
|
||||
return {
|
||||
stats: data,
|
||||
updatedAt: timestamp,
|
||||
};
|
||||
}),
|
||||
});
|
||||
@@ -505,6 +505,7 @@ export const assistantConfigurations = mysqlTable("assistant_configuration", {
|
||||
| "together"
|
||||
| "ollama"
|
||||
| "lm-studio"
|
||||
| "llama-cpp"
|
||||
| "custom"
|
||||
>()
|
||||
.notNull()
|
||||
|
||||
@@ -503,6 +503,7 @@ export const assistantConfigurations = pgTable("assistant_configuration", {
|
||||
| "together"
|
||||
| "ollama"
|
||||
| "lm-studio"
|
||||
| "llama-cpp"
|
||||
| "custom"
|
||||
>()
|
||||
.notNull()
|
||||
|
||||
@@ -488,6 +488,7 @@ export const assistantConfigurations = sqliteTable("assistant_configuration", {
|
||||
| "together"
|
||||
| "ollama"
|
||||
| "lm-studio"
|
||||
| "llama-cpp"
|
||||
| "custom"
|
||||
>()
|
||||
.notNull()
|
||||
|
||||
@@ -11,6 +11,7 @@ export const assistantProviderIds = [
|
||||
"together",
|
||||
"ollama",
|
||||
"lm-studio",
|
||||
"llama-cpp",
|
||||
"custom",
|
||||
] as const;
|
||||
|
||||
@@ -156,6 +157,15 @@ export const assistantProviderPresets = {
|
||||
discoveryAuthentication: "bearer",
|
||||
iconUrl: `${providerIconBaseUrl}/lmstudio.svg`,
|
||||
},
|
||||
"llama-cpp": {
|
||||
label: "llama.cpp",
|
||||
baseUrl: "http://localhost:8080/v1",
|
||||
modelDiscoveryPath: "/models",
|
||||
requiresApiKey: false,
|
||||
category: "local",
|
||||
discoveryAuthentication: "bearer",
|
||||
iconUrl: "https://cdn.jsdelivr.net/gh/selfhst/icons/svg/llama-cpp.svg",
|
||||
},
|
||||
custom: {
|
||||
label: undefined,
|
||||
baseUrl: "",
|
||||
|
||||
@@ -333,6 +333,7 @@ export type HomarrDocumentationPath =
|
||||
| "/docs/integrations/traefik"
|
||||
| "/docs/integrations/archiveteam-warrior"
|
||||
| "/docs/integrations/whats-up-docker"
|
||||
| "/docs/integrations/llama-cpp"
|
||||
| "/docs/widgets/clock"
|
||||
| "/docs/widgets/weather"
|
||||
| "/docs/widgets/air-quality"
|
||||
@@ -390,4 +391,5 @@ export type HomarrDocumentationPath =
|
||||
| "/docs/widgets/traefik"
|
||||
| "/docs/widgets/custom-api"
|
||||
| "/docs/widgets/assistant"
|
||||
| "/docs/widgets/whats-up-docker";
|
||||
| "/docs/widgets/whats-up-docker"
|
||||
| "/docs/widgets/llama-cpp";
|
||||
@@ -60,4 +60,5 @@ export const widgetDocSlugs = {
|
||||
customApi: "custom-api",
|
||||
assistant: "assistant",
|
||||
wud: "whats-up-docker",
|
||||
llamacpp: "llama-cpp",
|
||||
} satisfies Record<WidgetKind, string | null>;
|
||||
@@ -545,6 +545,14 @@ export const integrationDefs = {
|
||||
documentationSlug: "whats-up-docker",
|
||||
defaultPort: 3000,
|
||||
},
|
||||
llamacpp: {
|
||||
name: "llama.cpp",
|
||||
secretKinds: [[]],
|
||||
iconUrl: "https://cdn.jsdelivr.net/gh/selfhst/icons/svg/llama-cpp.svg",
|
||||
category: ["systemMonitoring"],
|
||||
documentationSlug: "llama-cpp",
|
||||
defaultPort: 8080,
|
||||
},
|
||||
// This integration only returns mock data, it is used during development (but can also be used in production by directly going to the create page)
|
||||
mock: {
|
||||
name: "Mock",
|
||||
@@ -713,6 +721,7 @@ export const integrationCategories = [
|
||||
"uptimeMonitoring",
|
||||
"subtitleManager",
|
||||
"reverseProxy",
|
||||
"systemMonitoring",
|
||||
] as const;
|
||||
|
||||
export type IntegrationCategory = (typeof integrationCategories)[number];
|
||||
@@ -48,6 +48,7 @@ describe("widget integration config", () => {
|
||||
archiveTeamWarrior: 1,
|
||||
anchorNote: 1,
|
||||
wud: 1,
|
||||
llamacpp: 1,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -87,6 +87,7 @@ export const widgetIntegrationConfigs = {
|
||||
anchorNote: { supportedIntegrations: ["anchor", "mock"], maxIntegrations: 1 },
|
||||
traefik: { supportedIntegrations: ["traefik", "mock"] },
|
||||
wud: { supportedIntegrations: ["wud", "mock"], maxIntegrations: 1 },
|
||||
llamacpp: { supportedIntegrations: ["llamacpp", "mock"], maxIntegrations: 1 },
|
||||
} satisfies Partial<Record<WidgetKind, WidgetIntegrationConfig>>;
|
||||
|
||||
export type WidgetKindWithIntegration = keyof typeof widgetIntegrationConfigs;
|
||||
|
||||
@@ -58,6 +58,7 @@ export const widgetKinds = [
|
||||
"customApi",
|
||||
"assistant",
|
||||
"wud",
|
||||
"llamacpp",
|
||||
] as const;
|
||||
|
||||
export type WidgetKind = (typeof widgetKinds)[number];
|
||||
|
||||
@@ -110,6 +110,8 @@ const integrationCreators = {
|
||||
traefik: async (input: IntegrationInput) =>
|
||||
new (await import("../traefik/traefik-integration")).TraefikIntegration(input),
|
||||
wud: async (input: IntegrationInput) => new (await import("../wud/wud-integration")).WudIntegration(input),
|
||||
llamacpp: async (input: IntegrationInput) =>
|
||||
new (await import("../llama-cpp/llamacpp-integration")).LlamacppIntegration(input),
|
||||
} satisfies Record<IntegrationKind, (input: IntegrationInput) => Promise<Integration>>;
|
||||
|
||||
type IntegrationCreators = typeof integrationCreators;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// General integrations
|
||||
export { ArchiveTeamWarriorIntegration } from "./archive-team-warrior/archive-team-warrior-integration";
|
||||
export { WudIntegration } from "./wud/wud-integration";
|
||||
export { LlamacppIntegration } from "./llama-cpp/llamacpp-integration";
|
||||
export { AdGuardHomeIntegration } from "./adguard-home/adguard-home-integration";
|
||||
export { TechnitiumDnsIntegration } from "./technitium/technitium-integration";
|
||||
export { AnchorIntegration } from "./anchor/anchor-integration";
|
||||
@@ -95,6 +96,7 @@ export type {
|
||||
ArchiveTeamWarriorStatus,
|
||||
} from "./archive-team-warrior/archive-team-warrior-types";
|
||||
export type { WudStats, WudContainerUpdate } from "./wud/wud-types";
|
||||
export type { LlamacppStats, LlamacppModel } from "./llama-cpp/llamacpp-types";
|
||||
|
||||
// Schemas
|
||||
export { anchorNotesListInputSchema, anchorNoteUpdateInputSchema } from "./anchor/anchor-types";
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
import { ParseError, ResponseError } from "@homarr/common/server";
|
||||
import { fetchWithTrustedCertificatesAsync } from "@homarr/core/infrastructure/http";
|
||||
|
||||
import type { IntegrationTestingInput } from "../base/integration";
|
||||
import { Integration } from "../base/integration";
|
||||
import { TestConnectionError } from "../base/test-connection/test-connection-error";
|
||||
import type { TestingResult } from "../base/test-connection/test-connection-service";
|
||||
|
||||
import type { LlamacppStats } from "./llamacpp-types";
|
||||
import {
|
||||
mapContextUsage,
|
||||
mapLlamacppModel,
|
||||
mapLlamacppPerRequest,
|
||||
mapLlamacppStats,
|
||||
mapSlotsSummary,
|
||||
parseLlamacppHealthAsync,
|
||||
parseLlamacppMetricsAsync,
|
||||
parseLlamacppModelsAsync,
|
||||
parseLlamacppSlotsAsync,
|
||||
} from "./llamacpp-types";
|
||||
|
||||
const LLMACPP_REQUEST_TIMEOUT_MS = 10_000;
|
||||
|
||||
export class LlamacppIntegration extends Integration {
|
||||
protected async testingAsync(input: IntegrationTestingInput): Promise<TestingResult> {
|
||||
const response = await input.fetchAsync(this.url("/health"), {
|
||||
signal: AbortSignal.timeout(LLMACPP_REQUEST_TIMEOUT_MS),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
return TestConnectionError.StatusResult(response);
|
||||
}
|
||||
|
||||
// Validate the health payload shape, not just that it is parseable JSON: an
|
||||
// unrelated JSON body must not pass connection testing.
|
||||
try {
|
||||
await parseLlamacppHealthAsync(response);
|
||||
} catch (error) {
|
||||
if (error instanceof ParseError) {
|
||||
return TestConnectionError.ParseResult(error);
|
||||
}
|
||||
return TestConnectionError.UnknownResult(error);
|
||||
}
|
||||
|
||||
return { success: true };
|
||||
}
|
||||
|
||||
public async getStatsAsync(): Promise<LlamacppStats> {
|
||||
const [healthResponse, modelsResponse, metricsResponse, slotsResponse] = await Promise.all([
|
||||
fetchWithTrustedCertificatesAsync(this.url("/health"), { timeout: LLMACPP_REQUEST_TIMEOUT_MS }),
|
||||
fetchWithTrustedCertificatesAsync(this.url("/v1/models"), { timeout: LLMACPP_REQUEST_TIMEOUT_MS }),
|
||||
fetchWithTrustedCertificatesAsync(this.url("/metrics"), { timeout: LLMACPP_REQUEST_TIMEOUT_MS }),
|
||||
// /slots is not present on every build; a missing endpoint degrades to no context usage.
|
||||
fetchWithTrustedCertificatesAsync(this.url("/slots"), { timeout: LLMACPP_REQUEST_TIMEOUT_MS }).catch(() => null),
|
||||
]);
|
||||
|
||||
if (!healthResponse.ok) {
|
||||
throw new ResponseError(healthResponse);
|
||||
}
|
||||
if (!modelsResponse.ok) {
|
||||
throw new ResponseError(modelsResponse);
|
||||
}
|
||||
|
||||
// /metrics only exists when llama-server was started with --metrics; a non-OK
|
||||
// response (typically 501 Not Implemented) degrades to no rate counters instead
|
||||
// of dropping the health and model data that the other endpoints still provide.
|
||||
const metricsAvailable = metricsResponse.ok;
|
||||
|
||||
const [health, models, metrics, slots] = await Promise.all([
|
||||
parseLlamacppHealthAsync(healthResponse),
|
||||
parseLlamacppModelsAsync(modelsResponse),
|
||||
metricsAvailable ? parseLlamacppMetricsAsync(metricsResponse) : Promise.resolve([]),
|
||||
slotsResponse?.ok ? parseLlamacppSlotsAsync(slotsResponse) : Promise.resolve(null),
|
||||
]);
|
||||
|
||||
const firstModel = models.data[0] ? mapLlamacppModel(models.data[0]) : null;
|
||||
const contextUsage = mapContextUsage(slots);
|
||||
const slotsSummary = mapSlotsSummary(slots);
|
||||
const perRequest = mapLlamacppPerRequest(slots);
|
||||
|
||||
return mapLlamacppStats(health, firstModel, metrics, contextUsage, slotsSummary, perRequest);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,381 @@
|
||||
import { ParseError } from "@homarr/common/server";
|
||||
import { z } from "zod/v4";
|
||||
|
||||
export const LLMACPP_HEALTH_PARSE_ERROR_MESSAGE = "Invalid llama.cpp /health response";
|
||||
export const LLMACPP_MODELS_PARSE_ERROR_MESSAGE = "Invalid llama.cpp /v1/models response";
|
||||
|
||||
export const parsePrometheusMetrics = (text: string): { name: string; value: number }[] => {
|
||||
const metrics: { name: string; value: number }[] = [];
|
||||
for (const rawLine of text.split(/\r?\n/)) {
|
||||
const line = rawLine.trim();
|
||||
if (line.length === 0 || line.startsWith("#")) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const match = /^([a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{[^}]*\})?\s+(-?[0-9]+(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?|NaN)$/.exec(
|
||||
line,
|
||||
);
|
||||
if (!match) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const name = match[1];
|
||||
const rawValue = match[2];
|
||||
if (!name || rawValue === undefined) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const value = Number.parseFloat(rawValue);
|
||||
if (!Number.isFinite(value)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
metrics.push({ name, value });
|
||||
}
|
||||
|
||||
return metrics;
|
||||
};
|
||||
|
||||
export const llamacppHealthSchema = z.object({
|
||||
status: z.string(),
|
||||
});
|
||||
|
||||
const llamacppModelMetaSchema = z
|
||||
.object({
|
||||
n_ctx: z.number().optional(),
|
||||
n_params: z.number().optional(),
|
||||
size: z.number().optional(),
|
||||
ftype: z.string().optional(),
|
||||
})
|
||||
.optional();
|
||||
|
||||
const llamacppModelSchema = z.object({
|
||||
id: z.string(),
|
||||
meta: llamacppModelMetaSchema,
|
||||
});
|
||||
|
||||
export const llamacppModelsSchema = z.object({
|
||||
data: z.array(llamacppModelSchema),
|
||||
});
|
||||
|
||||
export interface LlamacppModel {
|
||||
id: string;
|
||||
name: string;
|
||||
contextSize: number | null;
|
||||
parameterCount: number | null;
|
||||
fileSizeBytes: number | null;
|
||||
quantization: string | null;
|
||||
}
|
||||
|
||||
export interface LlamacppPerRequest {
|
||||
taskId: number | null;
|
||||
decodedTokens: number | null;
|
||||
}
|
||||
|
||||
export interface LlamacppSlotsSummary {
|
||||
total: number;
|
||||
processing: number;
|
||||
decoding: number;
|
||||
speculative: boolean | null;
|
||||
}
|
||||
|
||||
export interface LlamacppStats {
|
||||
health: string;
|
||||
model: LlamacppModel | null;
|
||||
contextUsage: LlamacppContextUsage | null;
|
||||
slots: LlamacppSlotsSummary | null;
|
||||
metrics: {
|
||||
generationSpeedTps: number | null;
|
||||
promptSpeedTps: number | null;
|
||||
avgGenerationSpeedTps: number | null;
|
||||
avgPromptSpeedTps: number | null;
|
||||
requestsProcessing: number | null;
|
||||
requestsDeferred: number | null;
|
||||
tokensProcessed: number | null;
|
||||
tokensGenerated: number | null;
|
||||
promptCacheHitRate: number | null;
|
||||
requestDecodedTokens: number | null;
|
||||
taskId: number | null;
|
||||
specDraftTokens: number | null;
|
||||
specAcceptedTokens: number | null;
|
||||
specDrafts: number | null;
|
||||
};
|
||||
}
|
||||
|
||||
const shortModelName = (id: string): string =>
|
||||
id
|
||||
.split("/")
|
||||
.pop()
|
||||
?.replace(/\.gguf$/i, "") ?? id;
|
||||
|
||||
export const mapLlamacppModel = (model: z.infer<typeof llamacppModelSchema>): LlamacppModel => ({
|
||||
id: model.id,
|
||||
name: shortModelName(model.id),
|
||||
contextSize: model.meta?.n_ctx ?? null,
|
||||
parameterCount: model.meta?.n_params ?? null,
|
||||
fileSizeBytes: model.meta?.size ?? null,
|
||||
quantization: model.meta?.ftype ?? null,
|
||||
});
|
||||
|
||||
export const getMetricValue = (metrics: readonly { name: string; value: number }[], name: string): number | null =>
|
||||
metrics.find((metric) => metric.name === name)?.value ?? null;
|
||||
|
||||
/**
|
||||
* Averages cumulative counters (tokens ÷ seconds) so a stable number is available while
|
||||
* the server is idle, mirroring how llama-monitor derives its always-on speeds.
|
||||
*/
|
||||
export const avgTokensPerSecond = (tokens: number | null, seconds: number | null): number | null =>
|
||||
tokens !== null && seconds !== null && seconds > 0 ? tokens / seconds : null;
|
||||
|
||||
interface LlamacppSlot {
|
||||
n_ctx?: number;
|
||||
n_prompt_tokens?: number;
|
||||
id_task?: number;
|
||||
is_processing?: boolean;
|
||||
speculative?: boolean;
|
||||
next_token?: { n_decoded?: number }[];
|
||||
}
|
||||
|
||||
export interface LlamacppContextUsage {
|
||||
usedTokens: number;
|
||||
contextSize: number;
|
||||
percent: number;
|
||||
}
|
||||
|
||||
const clampPercent = (value: number): number => Math.min(100, Math.max(0, value));
|
||||
|
||||
/**
|
||||
* Computes context (KV) usage from the /slots endpoint. The /metrics endpoint has no
|
||||
* KV metrics, so the largest used prompt tokens over the slot context size is used.
|
||||
*/
|
||||
export const mapContextUsage = (slots: readonly unknown[] | null): LlamacppContextUsage | null => {
|
||||
if (!Array.isArray(slots)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let largest: LlamacppContextUsage | null = null;
|
||||
for (const slot of slots) {
|
||||
if (typeof slot !== "object" || slot === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const record = slot as Partial<LlamacppSlot>;
|
||||
const contextSize = record.n_ctx;
|
||||
const usedTokens = record.n_prompt_tokens;
|
||||
|
||||
if (typeof contextSize === "number" && contextSize > 0 && typeof usedTokens === "number" && usedTokens >= 0) {
|
||||
const usage: LlamacppContextUsage = {
|
||||
usedTokens,
|
||||
contextSize,
|
||||
percent: clampPercent(Math.round((usedTokens / contextSize) * 1000) / 10),
|
||||
};
|
||||
if (largest === null || usage.usedTokens > largest.usedTokens) {
|
||||
largest = usage;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return largest;
|
||||
};
|
||||
|
||||
/**
|
||||
* Summarizes the /slots endpoint: total slot count, how many are currently
|
||||
* processing, how many are actively decoding, and whether any slot has
|
||||
* speculative decoding enabled. Null when /slots is unavailable.
|
||||
*/
|
||||
export const mapSlotsSummary = (slots: readonly unknown[] | null): LlamacppSlotsSummary | null => {
|
||||
if (!Array.isArray(slots) || slots.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let total = 0;
|
||||
let processing = 0;
|
||||
let decoding = 0;
|
||||
let speculative: boolean | null = null;
|
||||
|
||||
for (const slot of slots) {
|
||||
if (typeof slot !== "object" || slot === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
total += 1;
|
||||
|
||||
const record = slot as Partial<LlamacppSlot>;
|
||||
if (record.is_processing === true) {
|
||||
processing += 1;
|
||||
if (Array.isArray(record.next_token) && record.next_token.length > 0) {
|
||||
decoding += 1;
|
||||
}
|
||||
}
|
||||
if (record.speculative === true) {
|
||||
speculative = true;
|
||||
} else if (typeof record.speculative === "boolean" && speculative !== true) {
|
||||
speculative = record.speculative;
|
||||
}
|
||||
}
|
||||
|
||||
return total > 0
|
||||
? {
|
||||
total,
|
||||
processing,
|
||||
decoding,
|
||||
speculative,
|
||||
}
|
||||
: null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Extracts the active request's identity and generated-token count from /slots. `id_task`
|
||||
* identifies the in-flight request (llama.cpp task IDs are monotonically increasing and
|
||||
* never reused) and `next_token[0].n_decoded` is `stats.n_gen`, the tokens this request has
|
||||
* generated so far, which the server resets to 0 whenever a new generation starts. The widget
|
||||
* takes deltas between polls to derive the per-request average speed.
|
||||
*/
|
||||
export const mapLlamacppPerRequest = (slots: readonly unknown[] | null): LlamacppPerRequest => {
|
||||
if (!Array.isArray(slots)) {
|
||||
return { taskId: null, decodedTokens: null };
|
||||
}
|
||||
|
||||
for (const slot of slots) {
|
||||
if (typeof slot !== "object" || slot === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const record = slot as LlamacppSlot;
|
||||
if (record.is_processing !== true) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const taskId = typeof record.id_task === "number" ? record.id_task : null;
|
||||
const nextToken = record.next_token?.[0];
|
||||
const decodedTokens = typeof nextToken?.n_decoded === "number" ? nextToken.n_decoded : null;
|
||||
|
||||
return { taskId, decodedTokens };
|
||||
}
|
||||
|
||||
return { taskId: null, decodedTokens: null };
|
||||
};
|
||||
|
||||
export const requestSpeedTps = (tokens: number | null, elapsedSeconds: number | null): number | null => {
|
||||
if (tokens === null || elapsedSeconds === null || elapsedSeconds <= 0) {
|
||||
return null;
|
||||
}
|
||||
const speed = tokens / elapsedSeconds;
|
||||
return Number.isFinite(speed) && speed > 0 ? speed : null;
|
||||
};
|
||||
|
||||
export const parseLlamacppSlotsAsync = async (response: {
|
||||
json: () => Promise<unknown>;
|
||||
}): Promise<unknown[] | null> => {
|
||||
let json: unknown;
|
||||
try {
|
||||
json = await response.json();
|
||||
} catch (error) {
|
||||
throw new ParseError("Invalid llama.cpp /slots response", {
|
||||
cause: error instanceof Error ? error : new Error(String(error)),
|
||||
});
|
||||
}
|
||||
|
||||
return Array.isArray(json) ? json : null;
|
||||
};
|
||||
|
||||
export const mapLlamacppStats = (
|
||||
health: z.infer<typeof llamacppHealthSchema>,
|
||||
model: LlamacppModel | null,
|
||||
metrics: readonly { name: string; value: number }[],
|
||||
contextUsage: LlamacppContextUsage | null,
|
||||
slotsSummary: LlamacppSlotsSummary | null,
|
||||
perRequest: LlamacppPerRequest,
|
||||
): LlamacppStats => {
|
||||
const promptTokens = getMetricValue(metrics, "llamacpp:prompt_tokens_total");
|
||||
const cachedTokens = getMetricValue(metrics, "llamacpp:prompt_tokens_cached_total");
|
||||
|
||||
const promptCacheHitRate =
|
||||
promptTokens !== null && cachedTokens !== null && promptTokens + cachedTokens > 0
|
||||
? Math.round((cachedTokens / (promptTokens + cachedTokens)) * 100)
|
||||
: null;
|
||||
|
||||
return {
|
||||
health: health.status,
|
||||
model,
|
||||
contextUsage,
|
||||
slots: slotsSummary,
|
||||
metrics: {
|
||||
generationSpeedTps: getMetricValue(metrics, "llamacpp:predicted_tokens_seconds"),
|
||||
promptSpeedTps: getMetricValue(metrics, "llamacpp:prompt_tokens_seconds"),
|
||||
avgGenerationSpeedTps: avgTokensPerSecond(
|
||||
getMetricValue(metrics, "llamacpp:tokens_predicted_total"),
|
||||
getMetricValue(metrics, "llamacpp:tokens_predicted_seconds_total"),
|
||||
),
|
||||
avgPromptSpeedTps: avgTokensPerSecond(
|
||||
getMetricValue(metrics, "llamacpp:prompt_tokens_total"),
|
||||
getMetricValue(metrics, "llamacpp:prompt_seconds_total"),
|
||||
),
|
||||
requestsProcessing: getMetricValue(metrics, "llamacpp:requests_processing"),
|
||||
requestsDeferred: getMetricValue(metrics, "llamacpp:requests_deferred"),
|
||||
tokensProcessed: getMetricValue(metrics, "llamacpp:prompt_tokens_total"),
|
||||
tokensGenerated: getMetricValue(metrics, "llamacpp:tokens_predicted_total"),
|
||||
promptCacheHitRate,
|
||||
requestDecodedTokens: perRequest.decodedTokens,
|
||||
taskId: perRequest.taskId,
|
||||
specDraftTokens: getMetricValue(metrics, "llamacpp:spec_decode_num_draft_tokens_total"),
|
||||
specAcceptedTokens: getMetricValue(metrics, "llamacpp:spec_decode_num_accepted_tokens_total"),
|
||||
specDrafts: getMetricValue(metrics, "llamacpp:spec_decode_num_drafts_total"),
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
export const parseLlamacppHealthAsync = async (response: {
|
||||
json: () => Promise<unknown>;
|
||||
}): Promise<z.infer<typeof llamacppHealthSchema>> => {
|
||||
let json: unknown;
|
||||
try {
|
||||
json = await response.json();
|
||||
} catch (error) {
|
||||
throw new ParseError(LLMACPP_HEALTH_PARSE_ERROR_MESSAGE, {
|
||||
cause: error instanceof Error ? error : new Error(String(error)),
|
||||
});
|
||||
}
|
||||
|
||||
const result = await llamacppHealthSchema.safeParseAsync(json);
|
||||
if (!result.success) {
|
||||
throw new ParseError(LLMACPP_HEALTH_PARSE_ERROR_MESSAGE, { cause: result.error });
|
||||
}
|
||||
|
||||
return result.data;
|
||||
};
|
||||
|
||||
export const parseLlamacppModelsAsync = async (response: {
|
||||
json: () => Promise<unknown>;
|
||||
}): Promise<z.infer<typeof llamacppModelsSchema>> => {
|
||||
let json: unknown;
|
||||
try {
|
||||
json = await response.json();
|
||||
} catch (error) {
|
||||
throw new ParseError(LLMACPP_MODELS_PARSE_ERROR_MESSAGE, {
|
||||
cause: error instanceof Error ? error : new Error(String(error)),
|
||||
});
|
||||
}
|
||||
|
||||
const result = await llamacppModelsSchema.safeParseAsync(json);
|
||||
if (!result.success) {
|
||||
throw new ParseError(LLMACPP_MODELS_PARSE_ERROR_MESSAGE, { cause: result.error });
|
||||
}
|
||||
|
||||
return result.data;
|
||||
};
|
||||
|
||||
export const parseLlamacppMetricsAsync = async (response: {
|
||||
text: () => Promise<string>;
|
||||
}): Promise<{ name: string; value: number }[]> => {
|
||||
let text: string;
|
||||
try {
|
||||
text = await response.text();
|
||||
} catch (error) {
|
||||
throw new ParseError("Invalid llama.cpp /metrics response", {
|
||||
cause: error instanceof Error ? error : new Error(String(error)),
|
||||
});
|
||||
}
|
||||
|
||||
return parsePrometheusMetrics(text);
|
||||
};
|
||||
@@ -0,0 +1,489 @@
|
||||
// @vitest-environment node
|
||||
import { ParseError } from "@homarr/common/server";
|
||||
import { Response } from "undici";
|
||||
import { beforeEach, describe, expect, test, vi } from "vitest";
|
||||
|
||||
vi.hoisted(() => {
|
||||
process.env.SKIP_ENV_VALIDATION = "true";
|
||||
process.env.SECRET_ENCRYPTION_KEY = "ff3f4f7ce30e870c9630de9e5d244ffa81101a24ed0dfe5f064beb53a7e684f1";
|
||||
});
|
||||
|
||||
import { fetchWithTrustedCertificatesAsync } from "@homarr/core/infrastructure/http";
|
||||
|
||||
import type { IntegrationSecret } from "../../base/types";
|
||||
import { IntegrationParseError } from "../../base/errors/parse/integration-parse-error";
|
||||
import { LlamacppIntegration } from "../llamacpp-integration";
|
||||
import {
|
||||
avgTokensPerSecond,
|
||||
mapContextUsage,
|
||||
mapLlamacppModel,
|
||||
mapLlamacppPerRequest,
|
||||
mapSlotsSummary,
|
||||
parsePrometheusMetrics,
|
||||
requestSpeedTps,
|
||||
} from "../llamacpp-types";
|
||||
|
||||
vi.mock("@homarr/core/infrastructure/http", () => ({
|
||||
fetchWithTrustedCertificatesAsync: vi.fn(),
|
||||
}));
|
||||
|
||||
const TEST_URL = "http://llamacpp.example.com";
|
||||
const mockFetch = vi.mocked(fetchWithTrustedCertificatesAsync);
|
||||
|
||||
const sampleModel = {
|
||||
id: "/root/models/unsloth/Qwen3.8-27B-UD-Q4_K_XL.gguf",
|
||||
meta: {
|
||||
n_ctx: 131072,
|
||||
n_params: 27320697856,
|
||||
size: 17912397824,
|
||||
ftype: "Q4_K - Small",
|
||||
},
|
||||
};
|
||||
|
||||
const sampleModelsResponse = {
|
||||
data: [sampleModel],
|
||||
};
|
||||
|
||||
const sampleMetricsText = [
|
||||
"# HELP llamacpp:prompt_tokens_total Total number of tokens processed",
|
||||
"# TYPE llamacpp:prompt_tokens_total counter",
|
||||
"llamacpp:prompt_tokens_total 69834",
|
||||
"llamacpp:prompt_seconds_total 23278",
|
||||
"llamacpp:prompt_tokens_cached_total 12000",
|
||||
"llamacpp:tokens_predicted_total 12161",
|
||||
"llamacpp:tokens_predicted_seconds_total 380.03125",
|
||||
"llamacpp:requests_processing 1",
|
||||
"llamacpp:requests_deferred 0",
|
||||
"llamacpp:predicted_tokens_seconds 32.8472",
|
||||
"llamacpp:prompt_tokens_seconds 1500.5",
|
||||
"llamacpp:spec_decode_num_draft_tokens_total 128",
|
||||
"llamacpp:spec_decode_num_accepted_tokens_total 96",
|
||||
"llamacpp:spec_decode_num_drafts_total 16",
|
||||
].join("\n");
|
||||
|
||||
const sampleSlotsResponse = [
|
||||
{
|
||||
id: 0,
|
||||
n_ctx: 131072,
|
||||
n_prompt_tokens: 101076,
|
||||
is_processing: true,
|
||||
speculative: false,
|
||||
id_task: 7,
|
||||
next_token: [{ n_decoded: 42, n_remain: 100, has_next_token: true, has_new_line: false }],
|
||||
},
|
||||
];
|
||||
|
||||
const jsonResponse = (body: unknown, status = 200) =>
|
||||
new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { "content-type": "application/json" },
|
||||
}) as unknown as Awaited<ReturnType<typeof fetchWithTrustedCertificatesAsync>>;
|
||||
|
||||
const textResponse = (body: string, status = 200) =>
|
||||
new Response(body, {
|
||||
status,
|
||||
headers: { "content-type": "text/plain" },
|
||||
}) as unknown as Awaited<ReturnType<typeof fetchWithTrustedCertificatesAsync>>;
|
||||
|
||||
const createIntegration = (decryptedSecrets: IntegrationSecret[] = []) =>
|
||||
new LlamacppIntegration({
|
||||
id: "test-llamacpp",
|
||||
name: "Test llama.cpp",
|
||||
url: TEST_URL,
|
||||
externalUrl: null,
|
||||
decryptedSecrets,
|
||||
});
|
||||
|
||||
const mockAllEndpoints = () => {
|
||||
mockFetch.mockReset();
|
||||
mockFetch.mockImplementation((async (input) => {
|
||||
const url = String(input);
|
||||
if (url.includes("/health")) {
|
||||
return jsonResponse({ status: "ok" });
|
||||
}
|
||||
if (url.includes("/v1/models")) {
|
||||
return jsonResponse(sampleModelsResponse);
|
||||
}
|
||||
if (url.includes("/metrics")) {
|
||||
return textResponse(sampleMetricsText);
|
||||
}
|
||||
if (url.includes("/slots")) {
|
||||
return jsonResponse(sampleSlotsResponse);
|
||||
}
|
||||
return textResponse("not found", 404);
|
||||
}) as typeof fetchWithTrustedCertificatesAsync);
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
mockFetch.mockReset();
|
||||
});
|
||||
|
||||
describe("parsePrometheusMetrics", () => {
|
||||
test("parses metric lines and skips comments and HELP/TYPE lines", () => {
|
||||
const metrics = parsePrometheusMetrics(sampleMetricsText);
|
||||
|
||||
expect(metrics).toStrictEqual([
|
||||
{ name: "llamacpp:prompt_tokens_total", value: 69834 },
|
||||
{ name: "llamacpp:prompt_seconds_total", value: 23278 },
|
||||
{ name: "llamacpp:prompt_tokens_cached_total", value: 12000 },
|
||||
{ name: "llamacpp:tokens_predicted_total", value: 12161 },
|
||||
{ name: "llamacpp:tokens_predicted_seconds_total", value: 380.03125 },
|
||||
{ name: "llamacpp:requests_processing", value: 1 },
|
||||
{ name: "llamacpp:requests_deferred", value: 0 },
|
||||
{ name: "llamacpp:predicted_tokens_seconds", value: 32.8472 },
|
||||
{ name: "llamacpp:prompt_tokens_seconds", value: 1500.5 },
|
||||
{ name: "llamacpp:spec_decode_num_draft_tokens_total", value: 128 },
|
||||
{ name: "llamacpp:spec_decode_num_accepted_tokens_total", value: 96 },
|
||||
{ name: "llamacpp:spec_decode_num_drafts_total", value: 16 },
|
||||
]);
|
||||
});
|
||||
|
||||
test("parses metric lines with labels and skips non-numeric values", () => {
|
||||
const metrics = parsePrometheusMetrics(
|
||||
[
|
||||
'llamacpp:requests_by_status{status="ok"} 5',
|
||||
"llamacpp:uptime_seconds +Inf",
|
||||
"llamacpp:nan_metric NaN",
|
||||
"",
|
||||
].join("\n"),
|
||||
);
|
||||
|
||||
expect(metrics).toStrictEqual([{ name: "llamacpp:requests_by_status", value: 5 }]);
|
||||
});
|
||||
|
||||
test("returns an empty array for empty input", () => {
|
||||
expect(parsePrometheusMetrics("")).toStrictEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("mapLlamacppModel", () => {
|
||||
test("extracts a short display name and metadata fields", () => {
|
||||
const model = mapLlamacppModel(sampleModel);
|
||||
|
||||
expect(model).toStrictEqual({
|
||||
id: "/root/models/unsloth/Qwen3.8-27B-UD-Q4_K_XL.gguf",
|
||||
name: "Qwen3.8-27B-UD-Q4_K_XL",
|
||||
contextSize: 131072,
|
||||
parameterCount: 27320697856,
|
||||
fileSizeBytes: 17912397824,
|
||||
quantization: "Q4_K - Small",
|
||||
});
|
||||
});
|
||||
|
||||
test("returns null metadata when meta is missing", () => {
|
||||
const model = mapLlamacppModel({ id: "models/foo.gguf" });
|
||||
|
||||
expect(model).toStrictEqual({
|
||||
id: "models/foo.gguf",
|
||||
name: "foo",
|
||||
contextSize: null,
|
||||
parameterCount: null,
|
||||
fileSizeBytes: null,
|
||||
quantization: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("mapContextUsage", () => {
|
||||
test("computes percent from a slot with valid n_ctx and n_prompt_tokens", () => {
|
||||
expect(mapContextUsage(sampleSlotsResponse)).toStrictEqual({
|
||||
usedTokens: 101076,
|
||||
contextSize: 131072,
|
||||
percent: 77.1,
|
||||
});
|
||||
});
|
||||
|
||||
test("returns the slot with the greatest used tokens, not the first valid slot", () => {
|
||||
expect(
|
||||
mapContextUsage([
|
||||
{ n_ctx: 1000, n_prompt_tokens: 100 },
|
||||
{ n_ctx: 8192, n_prompt_tokens: 6000 },
|
||||
{ n_ctx: 4096, n_prompt_tokens: 3000 },
|
||||
]),
|
||||
).toStrictEqual({
|
||||
usedTokens: 6000,
|
||||
contextSize: 8192,
|
||||
percent: 73.2,
|
||||
});
|
||||
});
|
||||
|
||||
test("returns null for missing, non-array, or malformed slots", () => {
|
||||
expect(mapContextUsage(null)).toBeNull();
|
||||
expect(mapContextUsage([{ n_ctx: 0, n_prompt_tokens: 10 }])).toBeNull();
|
||||
expect(mapContextUsage([{ n_prompt_tokens: 10 }])).toBeNull();
|
||||
expect(mapContextUsage(["not-an-object", null])).toBeNull();
|
||||
});
|
||||
|
||||
test("clamps the percentage to 100", () => {
|
||||
expect(mapContextUsage([{ n_ctx: 10, n_prompt_tokens: 50 }])).toStrictEqual({
|
||||
usedTokens: 50,
|
||||
contextSize: 10,
|
||||
percent: 100,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("avgTokensPerSecond", () => {
|
||||
test("divides cumulative counters", () => {
|
||||
expect(avgTokensPerSecond(12161, 380.03125)).toBe(32);
|
||||
});
|
||||
|
||||
test("returns null when either counter is missing or seconds are zero", () => {
|
||||
expect(avgTokensPerSecond(null, 5)).toBeNull();
|
||||
expect(avgTokensPerSecond(10, null)).toBeNull();
|
||||
expect(avgTokensPerSecond(10, 0)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("mapLlamacppPerRequest", () => {
|
||||
test("returns the active request id and decoded tokens from the processing slot", () => {
|
||||
expect(mapLlamacppPerRequest(sampleSlotsResponse)).toStrictEqual({ taskId: 7, decodedTokens: 42 });
|
||||
});
|
||||
|
||||
test("skips idle slots and returns nulls", () => {
|
||||
expect(
|
||||
mapLlamacppPerRequest([{ is_processing: false, id_task: 3, next_token: [{ n_decoded: 10 }] }]),
|
||||
).toStrictEqual({ taskId: null, decodedTokens: null });
|
||||
});
|
||||
|
||||
test("returns nulls for missing, empty, or malformed slots", () => {
|
||||
expect(mapLlamacppPerRequest(null)).toStrictEqual({ taskId: null, decodedTokens: null });
|
||||
expect(mapLlamacppPerRequest([])).toStrictEqual({ taskId: null, decodedTokens: null });
|
||||
expect(mapLlamacppPerRequest(["nope", null])).toStrictEqual({ taskId: null, decodedTokens: null });
|
||||
});
|
||||
|
||||
test("handles a processing slot missing the per-request fields", () => {
|
||||
expect(mapLlamacppPerRequest([{ is_processing: true }])).toStrictEqual({
|
||||
taskId: null,
|
||||
decodedTokens: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("mapSlotsSummary", () => {
|
||||
test("counts slots, processing slots, decoding slots and speculative flags", () => {
|
||||
expect(
|
||||
mapSlotsSummary([
|
||||
{ is_processing: true, next_token: [{ n_decoded: 4 }], speculative: false },
|
||||
{ is_processing: false, speculative: true },
|
||||
{ is_processing: true, next_token: [], speculative: false },
|
||||
]),
|
||||
).toStrictEqual({ total: 3, processing: 2, decoding: 1, speculative: true });
|
||||
});
|
||||
|
||||
test("treats an empty next_token list as not decoding", () => {
|
||||
expect(mapSlotsSummary([{ is_processing: true, next_token: [] }, { is_processing: false }])).toStrictEqual({
|
||||
total: 2,
|
||||
processing: 1,
|
||||
decoding: 0,
|
||||
speculative: null,
|
||||
});
|
||||
});
|
||||
|
||||
test("returns null for missing, empty, or malformed slot lists", () => {
|
||||
expect(mapSlotsSummary(null)).toBeNull();
|
||||
expect(mapSlotsSummary([])).toBeNull();
|
||||
expect(mapSlotsSummary(["not-an-object", null])).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("requestSpeedTps", () => {
|
||||
test("divides token delta by elapsed seconds", () => {
|
||||
expect(requestSpeedTps(128, 4)).toBe(32);
|
||||
});
|
||||
|
||||
test("returns null for missing values, non-positive, or negative elapsed", () => {
|
||||
expect(requestSpeedTps(null, 4)).toBeNull();
|
||||
expect(requestSpeedTps(10, null)).toBeNull();
|
||||
expect(requestSpeedTps(10, 0)).toBeNull();
|
||||
expect(requestSpeedTps(10, -2)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("LlamacppIntegration getStatsAsync", () => {
|
||||
test("fetches health, models, metrics and slots and maps them into stats", async () => {
|
||||
mockAllEndpoints();
|
||||
|
||||
const stats = await createIntegration().getStatsAsync();
|
||||
|
||||
expect(stats).toStrictEqual({
|
||||
health: "ok",
|
||||
model: {
|
||||
id: "/root/models/unsloth/Qwen3.8-27B-UD-Q4_K_XL.gguf",
|
||||
name: "Qwen3.8-27B-UD-Q4_K_XL",
|
||||
contextSize: 131072,
|
||||
parameterCount: 27320697856,
|
||||
fileSizeBytes: 17912397824,
|
||||
quantization: "Q4_K - Small",
|
||||
},
|
||||
contextUsage: {
|
||||
usedTokens: 101076,
|
||||
contextSize: 131072,
|
||||
percent: 77.1,
|
||||
},
|
||||
slots: { total: 1, processing: 1, decoding: 1, speculative: false },
|
||||
metrics: {
|
||||
generationSpeedTps: 32.8472,
|
||||
promptSpeedTps: 1500.5,
|
||||
avgGenerationSpeedTps: 32,
|
||||
avgPromptSpeedTps: 3,
|
||||
requestsProcessing: 1,
|
||||
requestsDeferred: 0,
|
||||
tokensProcessed: 69834,
|
||||
tokensGenerated: 12161,
|
||||
promptCacheHitRate: 15,
|
||||
requestDecodedTokens: 42,
|
||||
taskId: 7,
|
||||
specDraftTokens: 128,
|
||||
specAcceptedTokens: 96,
|
||||
specDrafts: 16,
|
||||
},
|
||||
});
|
||||
|
||||
const requestedUrls = mockFetch.mock.calls.map((call) => String(call[0])).toSorted();
|
||||
expect(requestedUrls).toStrictEqual(
|
||||
[`${TEST_URL}/health`, `${TEST_URL}/metrics`, `${TEST_URL}/slots`, `${TEST_URL}/v1/models`].toSorted(),
|
||||
);
|
||||
});
|
||||
|
||||
test("returns null model when the server has no models loaded", async () => {
|
||||
mockAllEndpoints();
|
||||
mockFetch.mockImplementation((async (input) => {
|
||||
const url = String(input);
|
||||
if (url.includes("/v1/models")) {
|
||||
return jsonResponse({ data: [] });
|
||||
}
|
||||
if (url.includes("/health")) {
|
||||
return jsonResponse({ status: "ok" });
|
||||
}
|
||||
if (url.includes("/slots")) {
|
||||
return jsonResponse([]);
|
||||
}
|
||||
return textResponse(sampleMetricsText);
|
||||
}) as typeof fetchWithTrustedCertificatesAsync);
|
||||
|
||||
const stats = await createIntegration().getStatsAsync();
|
||||
|
||||
expect(stats.model).toBeNull();
|
||||
expect(stats.metrics.generationSpeedTps).toBe(32.8472);
|
||||
});
|
||||
|
||||
test("returns null for metrics that are absent from /metrics", async () => {
|
||||
mockAllEndpoints();
|
||||
mockFetch.mockImplementation((async (input) => {
|
||||
const url = String(input);
|
||||
if (url.includes("/health")) {
|
||||
return jsonResponse({ status: "ok" });
|
||||
}
|
||||
if (url.includes("/v1/models")) {
|
||||
return jsonResponse({ data: [] });
|
||||
}
|
||||
if (url.includes("/slots")) {
|
||||
return jsonResponse([]);
|
||||
}
|
||||
return textResponse("");
|
||||
}) as typeof fetchWithTrustedCertificatesAsync);
|
||||
|
||||
const stats = await createIntegration().getStatsAsync();
|
||||
|
||||
expect(stats.metrics).toStrictEqual({
|
||||
generationSpeedTps: null,
|
||||
promptSpeedTps: null,
|
||||
avgGenerationSpeedTps: null,
|
||||
avgPromptSpeedTps: null,
|
||||
requestsProcessing: null,
|
||||
requestsDeferred: null,
|
||||
tokensProcessed: null,
|
||||
tokensGenerated: null,
|
||||
promptCacheHitRate: null,
|
||||
requestDecodedTokens: null,
|
||||
taskId: null,
|
||||
specDraftTokens: null,
|
||||
specAcceptedTokens: null,
|
||||
specDrafts: null,
|
||||
});
|
||||
});
|
||||
|
||||
test("throws when the /health endpoint returns an error status", async () => {
|
||||
mockAllEndpoints();
|
||||
mockFetch.mockImplementation((async (input) => {
|
||||
const url = String(input);
|
||||
if (url.includes("/health")) {
|
||||
return textResponse("Service Unavailable", 503);
|
||||
}
|
||||
return jsonResponse({ status: "ok" });
|
||||
}) as typeof fetchWithTrustedCertificatesAsync);
|
||||
|
||||
await expect(createIntegration().getStatsAsync()).rejects.toThrow();
|
||||
});
|
||||
|
||||
test("degrades to null metrics when /metrics is not available (501) but keeps health and model data", async () => {
|
||||
mockAllEndpoints();
|
||||
mockFetch.mockImplementation((async (input) => {
|
||||
const url = String(input);
|
||||
if (url.includes("/metrics")) {
|
||||
return textResponse("Not Implemented", 501);
|
||||
}
|
||||
if (url.includes("/health")) {
|
||||
return jsonResponse({ status: "ok" });
|
||||
}
|
||||
if (url.includes("/v1/models")) {
|
||||
return jsonResponse(sampleModelsResponse);
|
||||
}
|
||||
if (url.includes("/slots")) {
|
||||
return jsonResponse(sampleSlotsResponse);
|
||||
}
|
||||
return textResponse("not found", 404);
|
||||
}) as typeof fetchWithTrustedCertificatesAsync);
|
||||
|
||||
const stats = await createIntegration().getStatsAsync();
|
||||
|
||||
expect(stats.health).toBe("ok");
|
||||
expect(stats.model?.name).toBe("Qwen3.8-27B-UD-Q4_K_XL");
|
||||
expect(stats.contextUsage).toStrictEqual({
|
||||
usedTokens: 101076,
|
||||
contextSize: 131072,
|
||||
percent: 77.1,
|
||||
});
|
||||
expect(stats.slots).toStrictEqual({ total: 1, processing: 1, decoding: 1, speculative: false });
|
||||
expect(stats.metrics).toStrictEqual({
|
||||
generationSpeedTps: null,
|
||||
promptSpeedTps: null,
|
||||
avgGenerationSpeedTps: null,
|
||||
avgPromptSpeedTps: null,
|
||||
requestsProcessing: null,
|
||||
requestsDeferred: null,
|
||||
tokensProcessed: null,
|
||||
tokensGenerated: null,
|
||||
promptCacheHitRate: null,
|
||||
requestDecodedTokens: 42,
|
||||
taskId: 7,
|
||||
specDraftTokens: null,
|
||||
specAcceptedTokens: null,
|
||||
specDrafts: null,
|
||||
});
|
||||
});
|
||||
|
||||
test("throws a parse error when /v1/models is not valid JSON", async () => {
|
||||
mockAllEndpoints();
|
||||
mockFetch.mockImplementation((async (input) => {
|
||||
const url = String(input);
|
||||
if (url.includes("/v1/models")) {
|
||||
return textResponse("not-json");
|
||||
}
|
||||
if (url.includes("/health")) {
|
||||
return jsonResponse({ status: "ok" });
|
||||
}
|
||||
if (url.includes("/slots")) {
|
||||
return jsonResponse([]);
|
||||
}
|
||||
return textResponse(sampleMetricsText);
|
||||
}) as typeof fetchWithTrustedCertificatesAsync);
|
||||
|
||||
await expect(createIntegration().getStatsAsync()).rejects.toSatisfy((error) => {
|
||||
if (!(error instanceof IntegrationParseError)) return false;
|
||||
const cause = error.cause;
|
||||
return cause instanceof ParseError && cause.message.includes("Invalid llama.cpp /v1/models response");
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -11,6 +11,7 @@ import type { TracearrDashboardData } from "../tracearr/tracearr-types";
|
||||
import type { TraefikDashboardData } from "../traefik/traefik-types";
|
||||
import type { UmamiEventSeries, UmamiMetricItem, UmamiVisitorStats, UmamiWebsite } from "../umami/umami-types";
|
||||
import type { UptimeKumaDashboardData } from "../uptime-kuma/uptime-kuma-types";
|
||||
import type { LlamacppStats } from "../llama-cpp/llamacpp-types";
|
||||
import type { WudStats } from "../wud/wud-types";
|
||||
import type {
|
||||
FirewallCpuSummary,
|
||||
@@ -352,4 +353,33 @@ export const mockWidgetData = {
|
||||
{ id: "redis", name: "Redis", currentVersion: "8.0.2", newVersion: "8.0.3", link: null },
|
||||
],
|
||||
} satisfies WudStats,
|
||||
llamacpp: {
|
||||
health: "ok",
|
||||
model: {
|
||||
id: "unsloth/Qwen3.8-27B-UD-Q4_K_XL.gguf",
|
||||
name: "Qwen3.8-27B-UD-Q4_K_XL",
|
||||
contextSize: 32768,
|
||||
parameterCount: 27_347_098_624,
|
||||
fileSizeBytes: 16_842_776_576,
|
||||
quantization: "Q4_K_XL",
|
||||
},
|
||||
contextUsage: { usedTokens: 8192, contextSize: 32768, percent: 25 },
|
||||
slots: { total: 2, processing: 1, decoding: 1, speculative: false },
|
||||
metrics: {
|
||||
generationSpeedTps: 42.5,
|
||||
promptSpeedTps: 1840.2,
|
||||
avgGenerationSpeedTps: 39.8,
|
||||
avgPromptSpeedTps: 2104.6,
|
||||
requestsProcessing: 1,
|
||||
requestsDeferred: 0,
|
||||
tokensProcessed: 12_483,
|
||||
tokensGenerated: 1_024,
|
||||
promptCacheHitRate: 62,
|
||||
requestDecodedTokens: 96,
|
||||
taskId: 1042,
|
||||
specDraftTokens: 48,
|
||||
specAcceptedTokens: 12,
|
||||
specDrafts: 8,
|
||||
},
|
||||
} satisfies LlamacppStats,
|
||||
} as const;
|
||||
@@ -0,0 +1,16 @@
|
||||
import { createIntegrationAsync } from "@homarr/integrations/factory";
|
||||
import type { LlamacppStats } from "@homarr/integrations";
|
||||
|
||||
import { createIntegrationRequestHandler } from "./lib/integration-request-handler";
|
||||
|
||||
export const llamacppStatsRequestHandler = createIntegrationRequestHandler<
|
||||
LlamacppStats,
|
||||
"llamacpp",
|
||||
Record<string, never>
|
||||
>({
|
||||
cacheNamespace: "llamacpp:stats",
|
||||
async requestAsync(integration) {
|
||||
const integrationInstance = await createIntegrationAsync(integration);
|
||||
return await integrationInstance.getStatsAsync();
|
||||
},
|
||||
});
|
||||
@@ -1027,7 +1027,8 @@
|
||||
"analytics": "分析",
|
||||
"vpn": "VPN",
|
||||
"mediaLibrary": "媒体库",
|
||||
"uptimeMonitoring": "运行时间监控"
|
||||
"uptimeMonitoring": "运行时间监控",
|
||||
"systemMonitoring": "系统监控"
|
||||
},
|
||||
"testConnection": {
|
||||
"action": {
|
||||
@@ -1355,6 +1356,10 @@
|
||||
"bazarr": {
|
||||
"name": "Bazarr",
|
||||
"description": "Sonarr 和 Radarr 的配套应用,用于管理和下载字幕"
|
||||
},
|
||||
"llamacpp": {
|
||||
"name": "llama.cpp",
|
||||
"description": "监控本地 llama.cpp llama-server:健康状态、模型与生成指标"
|
||||
}
|
||||
},
|
||||
"media": {
|
||||
@@ -4171,6 +4176,81 @@
|
||||
"subtitle": "Docker 容器的网络流量"
|
||||
}
|
||||
}
|
||||
},
|
||||
"llamacpp": {
|
||||
"name": "llama.cpp",
|
||||
"description": "显示本地 llama.cpp 服务器的健康状态、加载模型和生成速度。",
|
||||
"title": "llama.cpp",
|
||||
"status": {
|
||||
"unhealthy": "异常 ({status})"
|
||||
},
|
||||
"idle": "空闲",
|
||||
"speedTooltip": {
|
||||
"current": "当前生成速度",
|
||||
"average": "启动以来的平均生成速度",
|
||||
"request": "本次请求开始以来的平均速度"
|
||||
},
|
||||
"busy": "{count, plural, other {# 个请求处理中}}",
|
||||
"modelInfo": {
|
||||
"context": "上下文",
|
||||
"size": "文件大小",
|
||||
"parameters": "参数量",
|
||||
"model": "模型",
|
||||
"quantization": "量化"
|
||||
},
|
||||
"stats": {
|
||||
"tokensProcessed": "已处理 Token",
|
||||
"contextUsage": "上下文 / KV 占用",
|
||||
"cacheHitRate": "提示缓存命中率",
|
||||
"tokensGenerated": "已生成 Token"
|
||||
},
|
||||
"error": {
|
||||
"unreachable": "无法连接 llama-server"
|
||||
},
|
||||
"unit": {
|
||||
"speed": "{value} t/s",
|
||||
"parameters": "{value}B",
|
||||
"none": "—",
|
||||
"speedShort": "t/s"
|
||||
},
|
||||
"option": {
|
||||
"showTitle": {
|
||||
"label": "显示标题"
|
||||
},
|
||||
"showModelInfo": {
|
||||
"label": "显示模型信息"
|
||||
},
|
||||
"showContextUsage": {
|
||||
"label": "显示上下文占用"
|
||||
},
|
||||
"showCacheHitRate": {
|
||||
"label": "显示缓存命中率"
|
||||
},
|
||||
"showSpeedStats": {
|
||||
"label": "显示速度统计"
|
||||
},
|
||||
"showRequests": {
|
||||
"label": "显示请求负载"
|
||||
},
|
||||
"showTokenStats": {
|
||||
"label": "显示 Token 总量"
|
||||
},
|
||||
"showSpeculative": {
|
||||
"label": "显示投机解码"
|
||||
}
|
||||
},
|
||||
"advanced": {
|
||||
"promptSpeed": "Prompt 速度",
|
||||
"avgSpeed": "平均生成速度",
|
||||
"requests": "请求数",
|
||||
"deferred": "排队中",
|
||||
"slots": "Slot 占用 {processing}/{total}",
|
||||
"speculative": "投机解码",
|
||||
"avgPrompt": "平均 Prompt 速度",
|
||||
"specDrafts": "投机草稿数",
|
||||
"specAccepted": "投机接受 Token",
|
||||
"specDraftTokens": "投机草稿 Token"
|
||||
}
|
||||
}
|
||||
},
|
||||
"widgetPreview": {
|
||||
|
||||
@@ -1424,7 +1424,8 @@
|
||||
"vpn": "VPN",
|
||||
"mediaLibrary": "Media Library",
|
||||
"uptimeMonitoring": "Uptime Monitoring",
|
||||
"beszel": "Beszel"
|
||||
"beszel": "Beszel",
|
||||
"systemMonitoring": "System Monitoring"
|
||||
},
|
||||
"testConnection": {
|
||||
"action": {
|
||||
@@ -1716,6 +1717,10 @@
|
||||
"use": "Select integrations in items",
|
||||
"interact": "Interact with integrations",
|
||||
"full": "Full integration access"
|
||||
},
|
||||
"llamacpp": {
|
||||
"name": "llama.cpp",
|
||||
"description": "Monitor a local llama.cpp llama-server: health, model and generation metrics"
|
||||
}
|
||||
},
|
||||
"media": {
|
||||
@@ -5205,6 +5210,81 @@
|
||||
"uptime": "Uptime",
|
||||
"agent": "Agent"
|
||||
}
|
||||
},
|
||||
"llamacpp": {
|
||||
"name": "llama.cpp",
|
||||
"description": "Show the health, loaded model and generation speed of your local llama.cpp server.",
|
||||
"title": "llama.cpp",
|
||||
"status": {
|
||||
"unhealthy": "Unhealthy ({status})"
|
||||
},
|
||||
"idle": "Idle",
|
||||
"speedTooltip": {
|
||||
"current": "Current generation speed",
|
||||
"average": "Average generation speed since startup",
|
||||
"request": "Average speed since this request started"
|
||||
},
|
||||
"busy": "{count, plural, one {# request running} other {# requests running}}",
|
||||
"modelInfo": {
|
||||
"context": "Context",
|
||||
"size": "File size",
|
||||
"parameters": "Parameters",
|
||||
"model": "Model",
|
||||
"quantization": "Quantization"
|
||||
},
|
||||
"stats": {
|
||||
"tokensProcessed": "Tokens processed",
|
||||
"contextUsage": "Context / KV usage",
|
||||
"cacheHitRate": "Prompt cache hit rate",
|
||||
"tokensGenerated": "Tokens generated"
|
||||
},
|
||||
"error": {
|
||||
"unreachable": "llama-server is not reachable"
|
||||
},
|
||||
"unit": {
|
||||
"speed": "{value} t/s",
|
||||
"parameters": "{value}B",
|
||||
"none": "—",
|
||||
"speedShort": "t/s"
|
||||
},
|
||||
"option": {
|
||||
"showTitle": {
|
||||
"label": "Show title"
|
||||
},
|
||||
"showModelInfo": {
|
||||
"label": "Show model info"
|
||||
},
|
||||
"showContextUsage": {
|
||||
"label": "Show context usage"
|
||||
},
|
||||
"showCacheHitRate": {
|
||||
"label": "Show cache hit rate"
|
||||
},
|
||||
"showSpeedStats": {
|
||||
"label": "Show speed stats"
|
||||
},
|
||||
"showRequests": {
|
||||
"label": "Show request load"
|
||||
},
|
||||
"showTokenStats": {
|
||||
"label": "Show token totals"
|
||||
},
|
||||
"showSpeculative": {
|
||||
"label": "Show speculative decoding"
|
||||
}
|
||||
},
|
||||
"advanced": {
|
||||
"promptSpeed": "Prompt speed",
|
||||
"avgSpeed": "Avg generation",
|
||||
"requests": "Requests",
|
||||
"deferred": "Deferred",
|
||||
"slots": "{processing}/{total} slots busy",
|
||||
"speculative": "Speculative decoding",
|
||||
"avgPrompt": "Avg prompt",
|
||||
"specDrafts": "Spec. drafts",
|
||||
"specAccepted": "Spec. accepted tokens",
|
||||
"specDraftTokens": "Spec. draft tokens"
|
||||
}
|
||||
}
|
||||
},
|
||||
"widgetPreview": {
|
||||
@@ -6157,6 +6237,9 @@
|
||||
"lm-studio": {
|
||||
"description": "An LM Studio local server with OpenAI compatibility enabled."
|
||||
},
|
||||
"llama-cpp": {
|
||||
"description": "A local or network llama.cpp server using its OpenAI-compatible API."
|
||||
},
|
||||
"custom": {
|
||||
"label": "Custom endpoint",
|
||||
"description": "Any other endpoint compatible with OpenAI Chat Completions."
|
||||
|
||||
@@ -822,7 +822,8 @@
|
||||
"analytics": "分析",
|
||||
"vpn": "VPN",
|
||||
"mediaLibrary": "媒體庫",
|
||||
"uptimeMonitoring": "運作時間監控"
|
||||
"uptimeMonitoring": "運作時間監控",
|
||||
"systemMonitoring": "系統監控"
|
||||
},
|
||||
"testConnection": {
|
||||
"action": {
|
||||
@@ -1150,6 +1151,10 @@
|
||||
"bazarr": {
|
||||
"name": "Bazarr",
|
||||
"description": "Sonarr 和 Radarr 的配套應用程式,管理和下載字幕"
|
||||
},
|
||||
"llamacpp": {
|
||||
"name": "llama.cpp",
|
||||
"description": "監控本地 llama.cpp llama-server:健康狀態、模型與生成指標"
|
||||
}
|
||||
},
|
||||
"media": {
|
||||
@@ -3946,6 +3951,81 @@
|
||||
"error": {
|
||||
"internalServerError": "無法擷取 Traefik 資料"
|
||||
}
|
||||
},
|
||||
"llamacpp": {
|
||||
"name": "llama.cpp",
|
||||
"description": "顯示本地 llama.cpp 伺服器的健康狀態、載入模型與生成速度。",
|
||||
"title": "llama.cpp",
|
||||
"status": {
|
||||
"unhealthy": "異常 ({status})"
|
||||
},
|
||||
"idle": "閒置",
|
||||
"speedTooltip": {
|
||||
"current": "目前生成速度",
|
||||
"average": "啟動以來的平均生成速度",
|
||||
"request": "本次請求開始以來的平均速度"
|
||||
},
|
||||
"busy": "{count, plural, other {# 個請求處理中}}",
|
||||
"modelInfo": {
|
||||
"context": "上下文",
|
||||
"size": "檔案大小",
|
||||
"parameters": "參數量",
|
||||
"model": "模型",
|
||||
"quantization": "量化"
|
||||
},
|
||||
"stats": {
|
||||
"tokensProcessed": "已處理 Token",
|
||||
"contextUsage": "上下文 / KV 佔用",
|
||||
"cacheHitRate": "提示快取命中率",
|
||||
"tokensGenerated": "已生成 Token"
|
||||
},
|
||||
"error": {
|
||||
"unreachable": "無法連接 llama-server"
|
||||
},
|
||||
"unit": {
|
||||
"speed": "{value} t/s",
|
||||
"parameters": "{value}B",
|
||||
"none": "—",
|
||||
"speedShort": "t/s"
|
||||
},
|
||||
"option": {
|
||||
"showTitle": {
|
||||
"label": "顯示標題"
|
||||
},
|
||||
"showModelInfo": {
|
||||
"label": "顯示模型資訊"
|
||||
},
|
||||
"showContextUsage": {
|
||||
"label": "顯示上下文佔用"
|
||||
},
|
||||
"showCacheHitRate": {
|
||||
"label": "顯示快取命中率"
|
||||
},
|
||||
"showSpeedStats": {
|
||||
"label": "顯示速度統計"
|
||||
},
|
||||
"showRequests": {
|
||||
"label": "顯示請求負載"
|
||||
},
|
||||
"showTokenStats": {
|
||||
"label": "顯示 Token 總量"
|
||||
},
|
||||
"showSpeculative": {
|
||||
"label": "顯示投機解碼"
|
||||
}
|
||||
},
|
||||
"advanced": {
|
||||
"promptSpeed": "Prompt 速度",
|
||||
"avgSpeed": "平均生成速度",
|
||||
"requests": "請求數",
|
||||
"deferred": "排隊中",
|
||||
"slots": "Slot 佔用 {processing}/{total}",
|
||||
"speculative": "投機解碼",
|
||||
"avgPrompt": "平均 Prompt 速度",
|
||||
"specDrafts": "投機草稿數",
|
||||
"specAccepted": "投機接受 Token",
|
||||
"specDraftTokens": "投機草稿 Token"
|
||||
}
|
||||
}
|
||||
},
|
||||
"widgetPreview": {
|
||||
|
||||
@@ -20,6 +20,7 @@ import {
|
||||
IconClock,
|
||||
IconClockPlay,
|
||||
IconCloud,
|
||||
IconCpu,
|
||||
IconDeviceCctv,
|
||||
IconDeviceGamepad,
|
||||
IconDownload,
|
||||
@@ -116,4 +117,5 @@ export const widgetCatalogIcons: Record<WidgetKind, TablerIcon> = {
|
||||
traefik: IconRoute,
|
||||
customApi: IconApi,
|
||||
wud: IconBrandDocker,
|
||||
llamacpp: IconCpu,
|
||||
};
|
||||
@@ -0,0 +1,571 @@
|
||||
"use client";
|
||||
|
||||
import { useMemo, useRef } from "react";
|
||||
import type { ReactNode } from "react";
|
||||
|
||||
import { Box, Center, Group, Progress, RingProgress, ScrollArea, Stack, Text, Tooltip } from "@mantine/core";
|
||||
import {
|
||||
IconBrain,
|
||||
IconBolt,
|
||||
IconCircleX,
|
||||
IconDatabase,
|
||||
IconFileText,
|
||||
IconGauge,
|
||||
IconLayersIntersect,
|
||||
IconServer,
|
||||
} from "@tabler/icons-react";
|
||||
|
||||
import { clientApi } from "@homarr/api/client";
|
||||
import { formatBytes, formatNumber } from "@homarr/common";
|
||||
import { useI18n } from "@homarr/translation/client";
|
||||
|
||||
import { WidgetEmptyState } from "../common/empty-state";
|
||||
import type { WidgetComponentProps } from "../definition";
|
||||
import { NoIntegrationSelectedError } from "../errors/no-integration-selected";
|
||||
|
||||
// Board grid: a widget N cells wide is N*200 + (N-1)*12 px (1→200, 2→412, 3→624, 4→836).
|
||||
// Width drives how much info fits side-by-side; height only matters for the
|
||||
// single-row (short) case where the widget must drop down to essentials.
|
||||
const isNarrowWidth = (width: number) => width < 300; // 1 cell (200px)
|
||||
const isWideWidth = (width: number) => width >= 550; // 3+ cells (624px+)
|
||||
const isShortHeight = (height: number) => height < 300; // 1 row (200px)
|
||||
|
||||
const ringSizeByScale = [
|
||||
{ minWidth: 320, size: 108 },
|
||||
{ minWidth: 220, size: 92 },
|
||||
{ minWidth: 0, size: 68 },
|
||||
] as const;
|
||||
|
||||
const getContextColor = (percent: number) => (percent > 90 ? "red" : percent > 75 ? "yellow" : "blue");
|
||||
|
||||
export default function LlamacppWidget({
|
||||
integrationIds,
|
||||
options,
|
||||
width,
|
||||
height,
|
||||
displayMode = "compact",
|
||||
}: WidgetComponentProps<"llamacpp">) {
|
||||
const integrationId = integrationIds[0];
|
||||
if (!integrationId) {
|
||||
throw new NoIntegrationSelectedError();
|
||||
}
|
||||
|
||||
return (
|
||||
<LlamacppContent
|
||||
integrationId={integrationId}
|
||||
options={options}
|
||||
width={width}
|
||||
height={height}
|
||||
displayMode={displayMode}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
interface LlamacppContentProps {
|
||||
integrationId: string;
|
||||
options: WidgetComponentProps<"llamacpp">["options"];
|
||||
width: number;
|
||||
height: number;
|
||||
displayMode: "compact" | "advanced";
|
||||
}
|
||||
|
||||
function LlamacppContent({ integrationId, options, width, height, displayMode }: LlamacppContentProps) {
|
||||
const t = useI18n("widget.llamacpp");
|
||||
const { data, isError } = clientApi.widget.llamacpp.getStats.useQuery({ integrationId });
|
||||
|
||||
// Tracks the in-flight request across polls so the widget can show the average
|
||||
// speed since that request started: /slots reports the request id (taskId) and how
|
||||
// many tokens it has generated so far (requestDecodedTokens). The server resets that
|
||||
// counter to 0 when a new request starts, so a fresh taskId means a fresh request.
|
||||
const requestRef = useRef<{ taskId: number; baseTokens: number; baseTimeMs: number } | null>(null);
|
||||
const perRequestSpeedTps = useMemo(() => {
|
||||
const taskId = data?.stats.metrics.taskId ?? null;
|
||||
const decoded = data?.stats.metrics.requestDecodedTokens ?? null;
|
||||
if (taskId === null || decoded === null) {
|
||||
requestRef.current = null;
|
||||
return null;
|
||||
}
|
||||
|
||||
const nowMs = Date.now();
|
||||
const current = requestRef.current;
|
||||
if (current === null || current.taskId !== taskId) {
|
||||
requestRef.current = { taskId, baseTokens: decoded, baseTimeMs: nowMs };
|
||||
return null;
|
||||
}
|
||||
|
||||
const elapsedSeconds = (nowMs - current.baseTimeMs) / 1000;
|
||||
const deltaTokens = decoded - current.baseTokens;
|
||||
if (elapsedSeconds <= 0 || deltaTokens < 0) {
|
||||
return null;
|
||||
}
|
||||
const speed = deltaTokens / elapsedSeconds;
|
||||
return Number.isFinite(speed) && speed > 0 ? speed : null;
|
||||
}, [data]);
|
||||
|
||||
if (isError) {
|
||||
return (
|
||||
<Center h="100%" w="100%" p="sm">
|
||||
<Group gap="xs" wrap="nowrap" miw={0}>
|
||||
<IconCircleX size={20} color="red" />
|
||||
<Text size="sm" c="dimmed" lineClamp={1}>
|
||||
{t("error.unreachable")}
|
||||
</Text>
|
||||
</Group>
|
||||
</Center>
|
||||
);
|
||||
}
|
||||
|
||||
if (!data) {
|
||||
return <WidgetEmptyState />;
|
||||
}
|
||||
|
||||
const stats = data.stats;
|
||||
const isHealthy = stats.health === "ok";
|
||||
const model = stats.model;
|
||||
const contextUsage = stats.contextUsage;
|
||||
const slots = stats.slots;
|
||||
|
||||
const requestsProcessing = stats.metrics.requestsProcessing;
|
||||
const isBusy = requestsProcessing !== null && requestsProcessing > 0;
|
||||
|
||||
let displayedSpeed: number | null;
|
||||
let speedMode: "request" | "live" | "average";
|
||||
if (isBusy && perRequestSpeedTps !== null) {
|
||||
displayedSpeed = perRequestSpeedTps;
|
||||
speedMode = "request";
|
||||
} else if (isBusy && (stats.metrics.generationSpeedTps ?? 0) > 0) {
|
||||
displayedSpeed = stats.metrics.generationSpeedTps;
|
||||
speedMode = "live";
|
||||
} else {
|
||||
displayedSpeed = stats.metrics.avgGenerationSpeedTps;
|
||||
speedMode = "average";
|
||||
}
|
||||
const speedTooltip =
|
||||
speedMode === "request"
|
||||
? t("speedTooltip.request")
|
||||
: speedMode === "live"
|
||||
? t("speedTooltip.current")
|
||||
: t("speedTooltip.average");
|
||||
|
||||
const speedValue = displayedSpeed !== null && displayedSpeed > 0 ? formatNumber(displayedSpeed, 1) : t("unit.none");
|
||||
const statusColor = isHealthy ? "green" : "red";
|
||||
const statusLabel = isHealthy ? undefined : t("status.unhealthy", { status: stats.health });
|
||||
|
||||
const contextPercent = options.showContextUsage && contextUsage ? contextUsage.percent : null;
|
||||
const contextLabel = contextUsage
|
||||
? `${formatNumber(contextUsage.usedTokens, 0)} / ${formatNumber(contextUsage.contextSize, 0)} (${contextUsage.percent}%)`
|
||||
: null;
|
||||
|
||||
const ringSize = ringSizeByScale.find(({ minWidth }) => Math.min(width, height) >= minWidth)?.size ?? 68;
|
||||
const ring = (
|
||||
<Tooltip label={speedTooltip} withArrow position="bottom">
|
||||
<RingProgress
|
||||
size={ringSize}
|
||||
thickness={Math.max(6, Math.round(ringSize / 10))}
|
||||
roundCaps
|
||||
sections={[
|
||||
{ value: contextPercent ?? 0, color: contextPercent !== null ? getContextColor(contextPercent) : "gray" },
|
||||
]}
|
||||
label={
|
||||
<Center>
|
||||
<Stack gap={0} align="center" style={{ lineHeight: 1.15 }}>
|
||||
<Text size={ringSize >= 92 ? "lg" : "sm"} fw={800} lineClamp={1}>
|
||||
{speedValue}
|
||||
</Text>
|
||||
<Text size="xs" c="dimmed" lineClamp={1}>
|
||||
{t("unit.speedShort")}
|
||||
</Text>
|
||||
</Stack>
|
||||
</Center>
|
||||
}
|
||||
/>
|
||||
</Tooltip>
|
||||
);
|
||||
|
||||
const modelLine = model && (
|
||||
<Group gap={6} wrap="nowrap" miw={0}>
|
||||
<IconBrain size={14} color="dimmed" style={{ flexShrink: 0 }} />
|
||||
<Tooltip label={model.id} withArrow>
|
||||
<Text size="xs" lineClamp={1} style={{ maxWidth: "100%" }}>
|
||||
{model.name}
|
||||
</Text>
|
||||
</Tooltip>
|
||||
{model.quantization && (
|
||||
<Text size="xs" c="dimmed" fw={600} lineClamp={1} style={{ flexShrink: 0 }}>
|
||||
{model.quantization}
|
||||
</Text>
|
||||
)}
|
||||
</Group>
|
||||
);
|
||||
|
||||
const busyLabel =
|
||||
requestsProcessing !== null ? (isBusy ? t("busy", { count: requestsProcessing }) : t("idle")) : null;
|
||||
|
||||
const promptSpeed = stats.metrics.promptSpeedTps;
|
||||
const avgPromptSpeed = stats.metrics.avgPromptSpeedTps;
|
||||
const cacheHitRate = stats.metrics.promptCacheHitRate;
|
||||
const tokensProcessed = stats.metrics.tokensProcessed;
|
||||
const tokensGenerated = stats.metrics.tokensGenerated;
|
||||
const requestsDeferred = stats.metrics.requestsDeferred;
|
||||
const specDraftTokens = stats.metrics.specDraftTokens;
|
||||
const specAcceptedTokens = stats.metrics.specAcceptedTokens;
|
||||
const specDrafts = stats.metrics.specDrafts;
|
||||
const hasSpecStats = specDraftTokens !== null || specAcceptedTokens !== null || specDrafts !== null;
|
||||
|
||||
if (displayMode === "advanced") {
|
||||
return (
|
||||
<Box h="100%" w="100%">
|
||||
<ScrollArea h="100%">
|
||||
<Stack gap="md" p="md">
|
||||
<Group gap="xs" wrap="nowrap" miw={0}>
|
||||
<StatusDot color={statusColor} label={statusLabel} />
|
||||
<Text size="md" fw={700} lineClamp={1}>
|
||||
{t("title")}
|
||||
</Text>
|
||||
{options.showRequests && busyLabel && (
|
||||
<Text size="xs" c="dimmed" lineClamp={1}>
|
||||
{busyLabel}
|
||||
</Text>
|
||||
)}
|
||||
</Group>
|
||||
|
||||
<Group gap="lg" wrap="nowrap" miw={0}>
|
||||
{ring}
|
||||
<Stack gap="sm" style={{ flex: 1, minWidth: 0 }}>
|
||||
{options.showSpeedStats && (
|
||||
<AdvancedRow
|
||||
icon={<IconBolt size={14} color="dimmed" />}
|
||||
label={t("advanced.promptSpeed")}
|
||||
value={
|
||||
promptSpeed !== null && promptSpeed > 0
|
||||
? t("unit.speed", { value: formatNumber(promptSpeed, 1) })
|
||||
: t("unit.none")
|
||||
}
|
||||
/>
|
||||
)}
|
||||
{options.showSpeedStats && (
|
||||
<AdvancedRow
|
||||
icon={<IconBolt size={14} color="dimmed" />}
|
||||
label={t("advanced.avgPrompt")}
|
||||
value={
|
||||
avgPromptSpeed !== null && avgPromptSpeed > 0
|
||||
? t("unit.speed", { value: formatNumber(avgPromptSpeed, 1) })
|
||||
: t("unit.none")
|
||||
}
|
||||
/>
|
||||
)}
|
||||
{options.showSpeedStats && stats.metrics.avgGenerationSpeedTps !== null && (
|
||||
<AdvancedRow
|
||||
icon={<IconGauge size={14} color="dimmed" />}
|
||||
label={t("advanced.avgSpeed")}
|
||||
value={
|
||||
(stats.metrics.avgGenerationSpeedTps ?? 0) > 0
|
||||
? t("unit.speed", { value: formatNumber(stats.metrics.avgGenerationSpeedTps, 1) })
|
||||
: t("unit.none")
|
||||
}
|
||||
/>
|
||||
)}
|
||||
{options.showContextUsage && contextUsage && (
|
||||
<AdvancedRow
|
||||
icon={<IconDatabase size={14} color="dimmed" />}
|
||||
label={t("stats.contextUsage")}
|
||||
value={contextLabel ?? "—"}
|
||||
/>
|
||||
)}
|
||||
{options.showRequests && requestsProcessing !== null && (
|
||||
<AdvancedRow
|
||||
icon={<IconServer size={14} color="dimmed" />}
|
||||
label={t("advanced.requests")}
|
||||
value={formatNumber(requestsProcessing, 0)}
|
||||
/>
|
||||
)}
|
||||
{options.showRequests && requestsDeferred !== null && (
|
||||
<AdvancedRow
|
||||
icon={<IconLayersIntersect size={14} color="dimmed" />}
|
||||
label={t("advanced.deferred")}
|
||||
value={formatNumber(requestsDeferred, 0)}
|
||||
/>
|
||||
)}
|
||||
</Stack>
|
||||
</Group>
|
||||
|
||||
{(options.showRequests || (options.showSpeculative && slots?.speculative)) && slots && (
|
||||
<Group gap="sm" wrap="nowrap" miw={0}>
|
||||
{options.showRequests && (
|
||||
<Text size="xs" c="dimmed" lineClamp={1}>
|
||||
{t("advanced.slots", { processing: slots.processing, total: slots.total })}
|
||||
</Text>
|
||||
)}
|
||||
{options.showSpeculative && slots.speculative && (
|
||||
<Text size="xs" c="dimmed" lineClamp={1} fw={600}>
|
||||
{t("advanced.speculative")}
|
||||
</Text>
|
||||
)}
|
||||
</Group>
|
||||
)}
|
||||
|
||||
{options.showModelInfo && model && (
|
||||
<Stack gap="xs">
|
||||
<AdvancedRow
|
||||
icon={<IconBrain size={14} color="dimmed" />}
|
||||
label={t("modelInfo.model")}
|
||||
value={model.name}
|
||||
valueTooltip={model.id}
|
||||
/>
|
||||
{model.quantization && (
|
||||
<AdvancedRow
|
||||
icon={<IconFileText size={14} color="dimmed" />}
|
||||
label={t("modelInfo.quantization")}
|
||||
value={model.quantization}
|
||||
/>
|
||||
)}
|
||||
{model.contextSize !== null && (
|
||||
<AdvancedRow
|
||||
icon={<IconDatabase size={14} color="dimmed" />}
|
||||
label={t("modelInfo.context")}
|
||||
value={formatNumber(model.contextSize, 0)}
|
||||
/>
|
||||
)}
|
||||
{model.fileSizeBytes !== null && (
|
||||
<AdvancedRow
|
||||
icon={<IconFileText size={14} color="dimmed" />}
|
||||
label={t("modelInfo.size")}
|
||||
value={formatBytes(model.fileSizeBytes)}
|
||||
/>
|
||||
)}
|
||||
{model.parameterCount !== null && (
|
||||
<AdvancedRow
|
||||
icon={<IconBrain size={14} color="dimmed" />}
|
||||
label={t("modelInfo.parameters")}
|
||||
value={t("unit.parameters", { value: formatNumber(model.parameterCount / 1e9, 2) })}
|
||||
/>
|
||||
)}
|
||||
</Stack>
|
||||
)}
|
||||
|
||||
{options.showTokenStats && (tokensProcessed !== null || tokensGenerated !== null) && (
|
||||
<Group gap="md" wrap="nowrap" miw={0}>
|
||||
{tokensProcessed !== null && (
|
||||
<AdvancedRow
|
||||
icon={<IconFileText size={14} color="dimmed" />}
|
||||
label={t("stats.tokensProcessed")}
|
||||
value={formatNumber(tokensProcessed, 0)}
|
||||
/>
|
||||
)}
|
||||
{tokensGenerated !== null && (
|
||||
<AdvancedRow
|
||||
icon={<IconBolt size={14} color="dimmed" />}
|
||||
label={t("stats.tokensGenerated")}
|
||||
value={formatNumber(tokensGenerated, 0)}
|
||||
/>
|
||||
)}
|
||||
</Group>
|
||||
)}
|
||||
|
||||
{options.showSpeculative && hasSpecStats && (
|
||||
<Stack gap="xs">
|
||||
{specDrafts !== null && (
|
||||
<AdvancedRow
|
||||
icon={<IconLayersIntersect size={14} color="dimmed" />}
|
||||
label={t("advanced.specDrafts")}
|
||||
value={formatNumber(specDrafts, 0)}
|
||||
/>
|
||||
)}
|
||||
{specDraftTokens !== null && (
|
||||
<AdvancedRow
|
||||
icon={<IconFileText size={14} color="dimmed" />}
|
||||
label={t("advanced.specDraftTokens")}
|
||||
value={formatNumber(specDraftTokens, 0)}
|
||||
/>
|
||||
)}
|
||||
{specAcceptedTokens !== null && (
|
||||
<AdvancedRow
|
||||
icon={<IconBolt size={14} color="dimmed" />}
|
||||
label={t("advanced.specAccepted")}
|
||||
value={formatNumber(specAcceptedTokens, 0)}
|
||||
/>
|
||||
)}
|
||||
</Stack>
|
||||
)}
|
||||
|
||||
{options.showCacheHitRate && cacheHitRate !== null && (
|
||||
<AdvancedRow
|
||||
icon={<IconDatabase size={14} color="dimmed" />}
|
||||
label={t("stats.cacheHitRate")}
|
||||
value={`${cacheHitRate}%`}
|
||||
progress={{ value: cacheHitRate, color: cacheHitRate > 75 ? "green" : "blue" }}
|
||||
/>
|
||||
)}
|
||||
</Stack>
|
||||
</ScrollArea>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
// Compact (default) mode — density adapts to the widget's grid footprint.
|
||||
if (isNarrowWidth(width)) {
|
||||
return (
|
||||
<Center h="100%" w="100%">
|
||||
<Stack gap={isShortHeight(height) ? 0 : 4} align="center" p="xs" style={{ position: "relative" }}>
|
||||
<Center pos="relative">
|
||||
{ring}
|
||||
<Box pos="absolute" top={0} right={0} style={{ zIndex: 1 }}>
|
||||
<StatusDot color={statusColor} label={statusLabel} />
|
||||
</Box>
|
||||
</Center>
|
||||
{!isShortHeight(height) && options.showModelInfo && modelLine}
|
||||
</Stack>
|
||||
</Center>
|
||||
);
|
||||
}
|
||||
|
||||
const rightColumn = (
|
||||
<Stack gap="xs" style={{ flex: 1, minWidth: 0 }}>
|
||||
{options.showModelInfo && modelLine}
|
||||
{options.showRequests && busyLabel && (
|
||||
<Text size="xs" c="dimmed" lineClamp={1}>
|
||||
{busyLabel}
|
||||
</Text>
|
||||
)}
|
||||
{options.showContextUsage && contextUsage && (
|
||||
<Group gap={6} wrap="nowrap" miw={0} w="100%" align="center">
|
||||
<Progress
|
||||
value={contextUsage.percent}
|
||||
size="xs"
|
||||
radius="xs"
|
||||
color={getContextColor(contextUsage.percent)}
|
||||
style={{ flex: 1, minWidth: 0 }}
|
||||
/>
|
||||
<Text size="xs" c="dimmed" lineClamp={1} style={{ flexShrink: 0 }}>
|
||||
{contextUsage.percent}%
|
||||
</Text>
|
||||
</Group>
|
||||
)}
|
||||
{isWideWidth(width) && (
|
||||
<Stack gap="xs">
|
||||
{options.showSpeedStats && promptSpeed !== null && promptSpeed > 0 && (
|
||||
<InfoRow
|
||||
label={t("advanced.promptSpeed")}
|
||||
value={t("unit.speed", { value: formatNumber(promptSpeed, 1) })}
|
||||
/>
|
||||
)}
|
||||
{options.showCacheHitRate && cacheHitRate !== null && (
|
||||
<InfoRow label={t("stats.cacheHitRate")} value={`${cacheHitRate}%`} />
|
||||
)}
|
||||
{options.showTokenStats && (tokensProcessed !== null || tokensGenerated !== null) && (
|
||||
<InfoRow
|
||||
label={t("stats.tokensProcessed")}
|
||||
value={
|
||||
tokensProcessed !== null && tokensGenerated !== null
|
||||
? formatNumber(tokensProcessed + tokensGenerated, 0)
|
||||
: tokensProcessed !== null
|
||||
? formatNumber(tokensProcessed, 0)
|
||||
: "—"
|
||||
}
|
||||
/>
|
||||
)}
|
||||
</Stack>
|
||||
)}
|
||||
</Stack>
|
||||
);
|
||||
|
||||
if (isShortHeight(height)) {
|
||||
return (
|
||||
<Center h="100%" w="100%">
|
||||
<Group gap="md" w="100%" wrap="nowrap" miw={0} justify="center" p="xs">
|
||||
{ring}
|
||||
{!isNarrowWidth(width) && (
|
||||
<Stack gap={2} style={{ minWidth: 0 }}>
|
||||
{options.showModelInfo && modelLine}
|
||||
{options.showRequests && busyLabel && (
|
||||
<Text size="xs" c="dimmed" lineClamp={1}>
|
||||
{busyLabel}
|
||||
</Text>
|
||||
)}
|
||||
</Stack>
|
||||
)}
|
||||
</Group>
|
||||
</Center>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<Box h="100%" p="xs" pos="relative">
|
||||
<Group pos="absolute" top={6} right={10} gap={0} style={{ zIndex: 2 }}>
|
||||
<StatusDot color={statusColor} label={statusLabel} />
|
||||
</Group>
|
||||
<Center h="100%">
|
||||
<Group gap="md" w="100%" wrap="nowrap" miw={0}>
|
||||
{ring}
|
||||
{rightColumn}
|
||||
</Group>
|
||||
</Center>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
const StatusDot = ({ color, label }: { color: string; label?: string }) => {
|
||||
const dot = (
|
||||
<Box
|
||||
aria-label={label}
|
||||
style={{
|
||||
width: 9,
|
||||
height: 9,
|
||||
borderRadius: "50%",
|
||||
backgroundColor: `var(--mantine-color-${color}-6)`,
|
||||
flexShrink: 0,
|
||||
cursor: "default",
|
||||
}}
|
||||
/>
|
||||
);
|
||||
if (!label) return dot;
|
||||
return (
|
||||
<Tooltip label={label} withArrow position="left">
|
||||
{dot}
|
||||
</Tooltip>
|
||||
);
|
||||
};
|
||||
|
||||
interface AdvancedRowProps {
|
||||
icon: ReactNode;
|
||||
label: string;
|
||||
value: string;
|
||||
valueTooltip?: string;
|
||||
progress?: { value: number; color: string };
|
||||
}
|
||||
|
||||
const AdvancedRow = ({ icon, label, value, valueTooltip, progress }: AdvancedRowProps) => (
|
||||
<Stack gap={3} style={{ minWidth: 150 }}>
|
||||
<Group gap={6} wrap="nowrap" miw={0} justify="space-between" w="100%">
|
||||
<Group gap={6} wrap="nowrap" miw={0}>
|
||||
{icon}
|
||||
<Text size="xs" c="dimmed" lineClamp={1}>
|
||||
{label}
|
||||
</Text>
|
||||
</Group>
|
||||
{valueTooltip ? (
|
||||
<Tooltip label={valueTooltip} withArrow>
|
||||
<Text size="xs" fw={600} lineClamp={1} style={{ maxWidth: 240 }}>
|
||||
{value}
|
||||
</Text>
|
||||
</Tooltip>
|
||||
) : (
|
||||
<Text size="xs" fw={600} lineClamp={1} style={{ maxWidth: 240 }}>
|
||||
{value}
|
||||
</Text>
|
||||
)}
|
||||
</Group>
|
||||
{progress && <Progress value={progress.value} size="xs" radius="xs" color={progress.color} />}
|
||||
</Stack>
|
||||
);
|
||||
|
||||
const InfoRow = ({ label, value }: { label: string; value: string }) => (
|
||||
<Group justify="space-between" wrap="nowrap" miw={0}>
|
||||
<Text size="xs" c="dimmed" lineClamp={1}>
|
||||
{label}
|
||||
</Text>
|
||||
<Text size="xs" fw={600} lineClamp={1}>
|
||||
{value}
|
||||
</Text>
|
||||
</Group>
|
||||
);
|
||||
@@ -0,0 +1,24 @@
|
||||
import { IconCpu } from "@tabler/icons-react";
|
||||
|
||||
import { getWidgetIntegrationConfig } from "@homarr/definitions";
|
||||
|
||||
import { createWidgetDefinition } from "../definition";
|
||||
import { optionsBuilder } from "../options";
|
||||
|
||||
export const { definition, componentLoader } = createWidgetDefinition("llamacpp", {
|
||||
icon: IconCpu,
|
||||
refetchInterval: 5,
|
||||
supportsAdvancedFocus: true,
|
||||
...getWidgetIntegrationConfig("llamacpp"),
|
||||
createOptions() {
|
||||
return optionsBuilder.from((factory) => ({
|
||||
showModelInfo: factory.switch({ defaultValue: true }),
|
||||
showContextUsage: factory.switch({ defaultValue: true }),
|
||||
showCacheHitRate: factory.switch({ defaultValue: true }),
|
||||
showSpeedStats: factory.switch({ defaultValue: true }),
|
||||
showRequests: factory.switch({ defaultValue: true }),
|
||||
showTokenStats: factory.switch({ defaultValue: true }),
|
||||
showSpeculative: factory.switch({ defaultValue: true }),
|
||||
}));
|
||||
},
|
||||
}).withDynamicImport(() => import("./component"));
|
||||
@@ -14,6 +14,7 @@ export const widgetQueryRefetchIntervals = [
|
||||
},
|
||||
{ queryKey: [["widget", "firewall"]], intervalSeconds: 10 },
|
||||
{ queryKey: [["widget", "healthMonitoring"]], intervalSeconds: 10 },
|
||||
{ queryKey: [["widget", "llamacpp"]], intervalSeconds: 5 },
|
||||
{ queryKey: [["widget", "mediaServer", "getCurrentStreams"]], intervalSeconds: 10 },
|
||||
{ queryKey: [["widget", "tracearr"]], intervalSeconds: 10 },
|
||||
{
|
||||
|
||||
@@ -59,6 +59,7 @@ import type * as video from "./video";
|
||||
import type * as vpn from "./vpn";
|
||||
import type * as weather from "./weather";
|
||||
import type * as wud from "./wud";
|
||||
import type * as llamacpp from "./llama-cpp";
|
||||
|
||||
// Keep these imports explicit so Next.js and Turbopack can discover every widget
|
||||
// module without loading any widget definition or component eagerly.
|
||||
@@ -122,6 +123,7 @@ export const widgetModuleLoaders = {
|
||||
customApi: () => import("./custom-api"),
|
||||
weather: () => import("./weather"),
|
||||
wud: () => import("./wud"),
|
||||
llamacpp: () => import("./llama-cpp"),
|
||||
} satisfies { [TKind in WidgetKind]: () => Promise<WidgetImports[TKind]> };
|
||||
|
||||
export type WidgetImports = {
|
||||
@@ -184,6 +186,7 @@ export type WidgetImports = {
|
||||
customApi: typeof customApi;
|
||||
weather: typeof weather;
|
||||
wud: typeof wud;
|
||||
llamacpp: typeof llamacpp;
|
||||
};
|
||||
|
||||
type WidgetComponentLoaders = {
|
||||
@@ -253,6 +256,7 @@ export const widgetComponentLoaders = {
|
||||
customApi: () => import("./custom-api/component"),
|
||||
weather: () => import("./weather/component"),
|
||||
wud: () => import("./wud/component"),
|
||||
llamacpp: () => import("./llama-cpp/component"),
|
||||
} satisfies WidgetComponentLoaders;
|
||||
|
||||
export type WidgetImportKey = keyof WidgetImports;
|
||||
|
||||
Reference in new issue
Block a user