feat!: release v2 (#6545)
Co-authored-by: Artur <145776561+arturict@users.noreply.github.com> Co-authored-by: arturict <arturict@users.noreply.github.com> Co-authored-by: Thomas <thomas@ajnart.fr> Co-authored-by: Afonso Jorge Ramos <afonsojorgeramos@gmail.com> Co-authored-by: Danura30082 <danura30082+github@gmail.com> Co-authored-by: Kushie <122986913+kushiemoon-dev@users.noreply.github.com> Co-authored-by: ajnart <thomas@ajnart.fr> Co-authored-by: ajnart <thomas@ajnart.dev> Co-authored-by: chenjun <chenjun@foxmail.com> Co-authored-by: Wikibear <wikibear@users.noreply.github.com> Co-authored-by: Bergasha <bergasha@gmail.com> Co-authored-by: leafyelin <43306196+leafyelin@users.noreply.github.com> Co-authored-by: Lucas Araujo <lucas.lra@gmail.com> Co-authored-by: GreenMonito <86085570+GreenMonito@users.noreply.github.com> Co-authored-by: Nerdy-Technician <roffo@nerdy-technician.social> Co-authored-by: Jun <34545896+jcstrong@users.noreply.github.com> Co-authored-by: Edmond Abraham <54119952+gitedmond@users.noreply.github.com> Co-authored-by: thrawnx <tight.bulb9384@fastmail.com> Co-authored-by: Chris <58918656+nordada@users.noreply.github.com> Co-authored-by: yi111 <153097222+Yi-111-a@users.noreply.github.com> Co-authored-by: Yanis <naamaneyanis@gmail.com> Co-authored-by: Wikibear <me@sej.de> Co-authored-by: alessiottavo <93441886+alessiottavo@users.noreply.github.com>
This commit is contained in:
3286 files changed
+284386
-139176
No files matched your search
@@ -0,0 +1,83 @@
|
||||
import { ParseError, ResponseError } from "@homarr/common/server";
|
||||
import { fetchWithTrustedCertificatesAsync } from "@homarr/core/infrastructure/http";
|
||||
|
||||
import type { IntegrationTestingInput } from "../base/integration";
|
||||
import { Integration } from "../base/integration";
|
||||
import { TestConnectionError } from "../base/test-connection/test-connection-error";
|
||||
import type { TestingResult } from "../base/test-connection/test-connection-service";
|
||||
|
||||
import type { LlamacppStats } from "./llamacpp-types";
|
||||
import {
|
||||
mapContextUsage,
|
||||
mapLlamacppModel,
|
||||
mapLlamacppPerRequest,
|
||||
mapLlamacppStats,
|
||||
mapSlotsSummary,
|
||||
parseLlamacppHealthAsync,
|
||||
parseLlamacppMetricsAsync,
|
||||
parseLlamacppModelsAsync,
|
||||
parseLlamacppSlotsAsync,
|
||||
} from "./llamacpp-types";
|
||||
|
||||
const LLMACPP_REQUEST_TIMEOUT_MS = 10_000;
|
||||
|
||||
export class LlamacppIntegration extends Integration {
|
||||
protected async testingAsync(input: IntegrationTestingInput): Promise<TestingResult> {
|
||||
const response = await input.fetchAsync(this.url("/health"), {
|
||||
signal: AbortSignal.timeout(LLMACPP_REQUEST_TIMEOUT_MS),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
return TestConnectionError.StatusResult(response);
|
||||
}
|
||||
|
||||
// Validate the health payload shape, not just that it is parseable JSON: an
|
||||
// unrelated JSON body must not pass connection testing.
|
||||
try {
|
||||
await parseLlamacppHealthAsync(response);
|
||||
} catch (error) {
|
||||
if (error instanceof ParseError) {
|
||||
return TestConnectionError.ParseResult(error);
|
||||
}
|
||||
return TestConnectionError.UnknownResult(error);
|
||||
}
|
||||
|
||||
return { success: true };
|
||||
}
|
||||
|
||||
public async getStatsAsync(): Promise<LlamacppStats> {
|
||||
const [healthResponse, modelsResponse, metricsResponse, slotsResponse] = await Promise.all([
|
||||
fetchWithTrustedCertificatesAsync(this.url("/health"), { timeout: LLMACPP_REQUEST_TIMEOUT_MS }),
|
||||
fetchWithTrustedCertificatesAsync(this.url("/v1/models"), { timeout: LLMACPP_REQUEST_TIMEOUT_MS }),
|
||||
fetchWithTrustedCertificatesAsync(this.url("/metrics"), { timeout: LLMACPP_REQUEST_TIMEOUT_MS }),
|
||||
// /slots is not present on every build; a missing endpoint degrades to no context usage.
|
||||
fetchWithTrustedCertificatesAsync(this.url("/slots"), { timeout: LLMACPP_REQUEST_TIMEOUT_MS }).catch(() => null),
|
||||
]);
|
||||
|
||||
if (!healthResponse.ok) {
|
||||
throw new ResponseError(healthResponse);
|
||||
}
|
||||
if (!modelsResponse.ok) {
|
||||
throw new ResponseError(modelsResponse);
|
||||
}
|
||||
|
||||
// /metrics only exists when llama-server was started with --metrics; a non-OK
|
||||
// response (typically 501 Not Implemented) degrades to no rate counters instead
|
||||
// of dropping the health and model data that the other endpoints still provide.
|
||||
const metricsAvailable = metricsResponse.ok;
|
||||
|
||||
const [health, models, metrics, slots] = await Promise.all([
|
||||
parseLlamacppHealthAsync(healthResponse),
|
||||
parseLlamacppModelsAsync(modelsResponse),
|
||||
metricsAvailable ? parseLlamacppMetricsAsync(metricsResponse) : Promise.resolve([]),
|
||||
slotsResponse?.ok ? parseLlamacppSlotsAsync(slotsResponse) : Promise.resolve(null),
|
||||
]);
|
||||
|
||||
const firstModel = models.data[0] ? mapLlamacppModel(models.data[0]) : null;
|
||||
const contextUsage = mapContextUsage(slots);
|
||||
const slotsSummary = mapSlotsSummary(slots);
|
||||
const perRequest = mapLlamacppPerRequest(slots);
|
||||
|
||||
return mapLlamacppStats(health, firstModel, metrics, contextUsage, slotsSummary, perRequest);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,381 @@
|
||||
import { ParseError } from "@homarr/common/server";
|
||||
import { z } from "zod/v4";
|
||||
|
||||
export const LLMACPP_HEALTH_PARSE_ERROR_MESSAGE = "Invalid llama.cpp /health response";
|
||||
export const LLMACPP_MODELS_PARSE_ERROR_MESSAGE = "Invalid llama.cpp /v1/models response";
|
||||
|
||||
export const parsePrometheusMetrics = (text: string): { name: string; value: number }[] => {
|
||||
const metrics: { name: string; value: number }[] = [];
|
||||
for (const rawLine of text.split(/\r?\n/)) {
|
||||
const line = rawLine.trim();
|
||||
if (line.length === 0 || line.startsWith("#")) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const match = /^([a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{[^}]*\})?\s+(-?[0-9]+(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?|NaN)$/.exec(
|
||||
line,
|
||||
);
|
||||
if (!match) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const name = match[1];
|
||||
const rawValue = match[2];
|
||||
if (!name || rawValue === undefined) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const value = Number.parseFloat(rawValue);
|
||||
if (!Number.isFinite(value)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
metrics.push({ name, value });
|
||||
}
|
||||
|
||||
return metrics;
|
||||
};
|
||||
|
||||
export const llamacppHealthSchema = z.object({
|
||||
status: z.string(),
|
||||
});
|
||||
|
||||
const llamacppModelMetaSchema = z
|
||||
.object({
|
||||
n_ctx: z.number().optional(),
|
||||
n_params: z.number().optional(),
|
||||
size: z.number().optional(),
|
||||
ftype: z.string().optional(),
|
||||
})
|
||||
.optional();
|
||||
|
||||
const llamacppModelSchema = z.object({
|
||||
id: z.string(),
|
||||
meta: llamacppModelMetaSchema,
|
||||
});
|
||||
|
||||
export const llamacppModelsSchema = z.object({
|
||||
data: z.array(llamacppModelSchema),
|
||||
});
|
||||
|
||||
export interface LlamacppModel {
|
||||
id: string;
|
||||
name: string;
|
||||
contextSize: number | null;
|
||||
parameterCount: number | null;
|
||||
fileSizeBytes: number | null;
|
||||
quantization: string | null;
|
||||
}
|
||||
|
||||
export interface LlamacppPerRequest {
|
||||
taskId: number | null;
|
||||
decodedTokens: number | null;
|
||||
}
|
||||
|
||||
export interface LlamacppSlotsSummary {
|
||||
total: number;
|
||||
processing: number;
|
||||
decoding: number;
|
||||
speculative: boolean | null;
|
||||
}
|
||||
|
||||
export interface LlamacppStats {
|
||||
health: string;
|
||||
model: LlamacppModel | null;
|
||||
contextUsage: LlamacppContextUsage | null;
|
||||
slots: LlamacppSlotsSummary | null;
|
||||
metrics: {
|
||||
generationSpeedTps: number | null;
|
||||
promptSpeedTps: number | null;
|
||||
avgGenerationSpeedTps: number | null;
|
||||
avgPromptSpeedTps: number | null;
|
||||
requestsProcessing: number | null;
|
||||
requestsDeferred: number | null;
|
||||
tokensProcessed: number | null;
|
||||
tokensGenerated: number | null;
|
||||
promptCacheHitRate: number | null;
|
||||
requestDecodedTokens: number | null;
|
||||
taskId: number | null;
|
||||
specDraftTokens: number | null;
|
||||
specAcceptedTokens: number | null;
|
||||
specDrafts: number | null;
|
||||
};
|
||||
}
|
||||
|
||||
const shortModelName = (id: string): string =>
|
||||
id
|
||||
.split("/")
|
||||
.pop()
|
||||
?.replace(/\.gguf$/i, "") ?? id;
|
||||
|
||||
export const mapLlamacppModel = (model: z.infer<typeof llamacppModelSchema>): LlamacppModel => ({
|
||||
id: model.id,
|
||||
name: shortModelName(model.id),
|
||||
contextSize: model.meta?.n_ctx ?? null,
|
||||
parameterCount: model.meta?.n_params ?? null,
|
||||
fileSizeBytes: model.meta?.size ?? null,
|
||||
quantization: model.meta?.ftype ?? null,
|
||||
});
|
||||
|
||||
export const getMetricValue = (metrics: readonly { name: string; value: number }[], name: string): number | null =>
|
||||
metrics.find((metric) => metric.name === name)?.value ?? null;
|
||||
|
||||
/**
|
||||
* Averages cumulative counters (tokens ÷ seconds) so a stable number is available while
|
||||
* the server is idle, mirroring how llama-monitor derives its always-on speeds.
|
||||
*/
|
||||
export const avgTokensPerSecond = (tokens: number | null, seconds: number | null): number | null =>
|
||||
tokens !== null && seconds !== null && seconds > 0 ? tokens / seconds : null;
|
||||
|
||||
interface LlamacppSlot {
|
||||
n_ctx?: number;
|
||||
n_prompt_tokens?: number;
|
||||
id_task?: number;
|
||||
is_processing?: boolean;
|
||||
speculative?: boolean;
|
||||
next_token?: { n_decoded?: number }[];
|
||||
}
|
||||
|
||||
export interface LlamacppContextUsage {
|
||||
usedTokens: number;
|
||||
contextSize: number;
|
||||
percent: number;
|
||||
}
|
||||
|
||||
const clampPercent = (value: number): number => Math.min(100, Math.max(0, value));
|
||||
|
||||
/**
|
||||
* Computes context (KV) usage from the /slots endpoint. The /metrics endpoint has no
|
||||
* KV metrics, so the largest used prompt tokens over the slot context size is used.
|
||||
*/
|
||||
export const mapContextUsage = (slots: readonly unknown[] | null): LlamacppContextUsage | null => {
|
||||
if (!Array.isArray(slots)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let largest: LlamacppContextUsage | null = null;
|
||||
for (const slot of slots) {
|
||||
if (typeof slot !== "object" || slot === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const record = slot as Partial<LlamacppSlot>;
|
||||
const contextSize = record.n_ctx;
|
||||
const usedTokens = record.n_prompt_tokens;
|
||||
|
||||
if (typeof contextSize === "number" && contextSize > 0 && typeof usedTokens === "number" && usedTokens >= 0) {
|
||||
const usage: LlamacppContextUsage = {
|
||||
usedTokens,
|
||||
contextSize,
|
||||
percent: clampPercent(Math.round((usedTokens / contextSize) * 1000) / 10),
|
||||
};
|
||||
if (largest === null || usage.usedTokens > largest.usedTokens) {
|
||||
largest = usage;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return largest;
|
||||
};
|
||||
|
||||
/**
|
||||
* Summarizes the /slots endpoint: total slot count, how many are currently
|
||||
* processing, how many are actively decoding, and whether any slot has
|
||||
* speculative decoding enabled. Null when /slots is unavailable.
|
||||
*/
|
||||
export const mapSlotsSummary = (slots: readonly unknown[] | null): LlamacppSlotsSummary | null => {
|
||||
if (!Array.isArray(slots) || slots.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let total = 0;
|
||||
let processing = 0;
|
||||
let decoding = 0;
|
||||
let speculative: boolean | null = null;
|
||||
|
||||
for (const slot of slots) {
|
||||
if (typeof slot !== "object" || slot === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
total += 1;
|
||||
|
||||
const record = slot as Partial<LlamacppSlot>;
|
||||
if (record.is_processing === true) {
|
||||
processing += 1;
|
||||
if (Array.isArray(record.next_token) && record.next_token.length > 0) {
|
||||
decoding += 1;
|
||||
}
|
||||
}
|
||||
if (record.speculative === true) {
|
||||
speculative = true;
|
||||
} else if (typeof record.speculative === "boolean" && speculative !== true) {
|
||||
speculative = record.speculative;
|
||||
}
|
||||
}
|
||||
|
||||
return total > 0
|
||||
? {
|
||||
total,
|
||||
processing,
|
||||
decoding,
|
||||
speculative,
|
||||
}
|
||||
: null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Extracts the active request's identity and generated-token count from /slots. `id_task`
|
||||
* identifies the in-flight request (llama.cpp task IDs are monotonically increasing and
|
||||
* never reused) and `next_token[0].n_decoded` is `stats.n_gen`, the tokens this request has
|
||||
* generated so far, which the server resets to 0 whenever a new generation starts. The widget
|
||||
* takes deltas between polls to derive the per-request average speed.
|
||||
*/
|
||||
export const mapLlamacppPerRequest = (slots: readonly unknown[] | null): LlamacppPerRequest => {
|
||||
if (!Array.isArray(slots)) {
|
||||
return { taskId: null, decodedTokens: null };
|
||||
}
|
||||
|
||||
for (const slot of slots) {
|
||||
if (typeof slot !== "object" || slot === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const record = slot as LlamacppSlot;
|
||||
if (record.is_processing !== true) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const taskId = typeof record.id_task === "number" ? record.id_task : null;
|
||||
const nextToken = record.next_token?.[0];
|
||||
const decodedTokens = typeof nextToken?.n_decoded === "number" ? nextToken.n_decoded : null;
|
||||
|
||||
return { taskId, decodedTokens };
|
||||
}
|
||||
|
||||
return { taskId: null, decodedTokens: null };
|
||||
};
|
||||
|
||||
export const requestSpeedTps = (tokens: number | null, elapsedSeconds: number | null): number | null => {
|
||||
if (tokens === null || elapsedSeconds === null || elapsedSeconds <= 0) {
|
||||
return null;
|
||||
}
|
||||
const speed = tokens / elapsedSeconds;
|
||||
return Number.isFinite(speed) && speed > 0 ? speed : null;
|
||||
};
|
||||
|
||||
export const parseLlamacppSlotsAsync = async (response: {
|
||||
json: () => Promise<unknown>;
|
||||
}): Promise<unknown[] | null> => {
|
||||
let json: unknown;
|
||||
try {
|
||||
json = await response.json();
|
||||
} catch (error) {
|
||||
throw new ParseError("Invalid llama.cpp /slots response", {
|
||||
cause: error instanceof Error ? error : new Error(String(error)),
|
||||
});
|
||||
}
|
||||
|
||||
return Array.isArray(json) ? json : null;
|
||||
};
|
||||
|
||||
export const mapLlamacppStats = (
|
||||
health: z.infer<typeof llamacppHealthSchema>,
|
||||
model: LlamacppModel | null,
|
||||
metrics: readonly { name: string; value: number }[],
|
||||
contextUsage: LlamacppContextUsage | null,
|
||||
slotsSummary: LlamacppSlotsSummary | null,
|
||||
perRequest: LlamacppPerRequest,
|
||||
): LlamacppStats => {
|
||||
const promptTokens = getMetricValue(metrics, "llamacpp:prompt_tokens_total");
|
||||
const cachedTokens = getMetricValue(metrics, "llamacpp:prompt_tokens_cached_total");
|
||||
|
||||
const promptCacheHitRate =
|
||||
promptTokens !== null && cachedTokens !== null && promptTokens + cachedTokens > 0
|
||||
? Math.round((cachedTokens / (promptTokens + cachedTokens)) * 100)
|
||||
: null;
|
||||
|
||||
return {
|
||||
health: health.status,
|
||||
model,
|
||||
contextUsage,
|
||||
slots: slotsSummary,
|
||||
metrics: {
|
||||
generationSpeedTps: getMetricValue(metrics, "llamacpp:predicted_tokens_seconds"),
|
||||
promptSpeedTps: getMetricValue(metrics, "llamacpp:prompt_tokens_seconds"),
|
||||
avgGenerationSpeedTps: avgTokensPerSecond(
|
||||
getMetricValue(metrics, "llamacpp:tokens_predicted_total"),
|
||||
getMetricValue(metrics, "llamacpp:tokens_predicted_seconds_total"),
|
||||
),
|
||||
avgPromptSpeedTps: avgTokensPerSecond(
|
||||
getMetricValue(metrics, "llamacpp:prompt_tokens_total"),
|
||||
getMetricValue(metrics, "llamacpp:prompt_seconds_total"),
|
||||
),
|
||||
requestsProcessing: getMetricValue(metrics, "llamacpp:requests_processing"),
|
||||
requestsDeferred: getMetricValue(metrics, "llamacpp:requests_deferred"),
|
||||
tokensProcessed: getMetricValue(metrics, "llamacpp:prompt_tokens_total"),
|
||||
tokensGenerated: getMetricValue(metrics, "llamacpp:tokens_predicted_total"),
|
||||
promptCacheHitRate,
|
||||
requestDecodedTokens: perRequest.decodedTokens,
|
||||
taskId: perRequest.taskId,
|
||||
specDraftTokens: getMetricValue(metrics, "llamacpp:spec_decode_num_draft_tokens_total"),
|
||||
specAcceptedTokens: getMetricValue(metrics, "llamacpp:spec_decode_num_accepted_tokens_total"),
|
||||
specDrafts: getMetricValue(metrics, "llamacpp:spec_decode_num_drafts_total"),
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
export const parseLlamacppHealthAsync = async (response: {
|
||||
json: () => Promise<unknown>;
|
||||
}): Promise<z.infer<typeof llamacppHealthSchema>> => {
|
||||
let json: unknown;
|
||||
try {
|
||||
json = await response.json();
|
||||
} catch (error) {
|
||||
throw new ParseError(LLMACPP_HEALTH_PARSE_ERROR_MESSAGE, {
|
||||
cause: error instanceof Error ? error : new Error(String(error)),
|
||||
});
|
||||
}
|
||||
|
||||
const result = await llamacppHealthSchema.safeParseAsync(json);
|
||||
if (!result.success) {
|
||||
throw new ParseError(LLMACPP_HEALTH_PARSE_ERROR_MESSAGE, { cause: result.error });
|
||||
}
|
||||
|
||||
return result.data;
|
||||
};
|
||||
|
||||
export const parseLlamacppModelsAsync = async (response: {
|
||||
json: () => Promise<unknown>;
|
||||
}): Promise<z.infer<typeof llamacppModelsSchema>> => {
|
||||
let json: unknown;
|
||||
try {
|
||||
json = await response.json();
|
||||
} catch (error) {
|
||||
throw new ParseError(LLMACPP_MODELS_PARSE_ERROR_MESSAGE, {
|
||||
cause: error instanceof Error ? error : new Error(String(error)),
|
||||
});
|
||||
}
|
||||
|
||||
const result = await llamacppModelsSchema.safeParseAsync(json);
|
||||
if (!result.success) {
|
||||
throw new ParseError(LLMACPP_MODELS_PARSE_ERROR_MESSAGE, { cause: result.error });
|
||||
}
|
||||
|
||||
return result.data;
|
||||
};
|
||||
|
||||
export const parseLlamacppMetricsAsync = async (response: {
|
||||
text: () => Promise<string>;
|
||||
}): Promise<{ name: string; value: number }[]> => {
|
||||
let text: string;
|
||||
try {
|
||||
text = await response.text();
|
||||
} catch (error) {
|
||||
throw new ParseError("Invalid llama.cpp /metrics response", {
|
||||
cause: error instanceof Error ? error : new Error(String(error)),
|
||||
});
|
||||
}
|
||||
|
||||
return parsePrometheusMetrics(text);
|
||||
};
|
||||
@@ -0,0 +1,489 @@
|
||||
// @vitest-environment node
|
||||
import { ParseError } from "@homarr/common/server";
|
||||
import { Response } from "undici";
|
||||
import { beforeEach, describe, expect, test, vi } from "vitest";
|
||||
|
||||
vi.hoisted(() => {
|
||||
process.env.SKIP_ENV_VALIDATION = "true";
|
||||
process.env.SECRET_ENCRYPTION_KEY = "ff3f4f7ce30e870c9630de9e5d244ffa81101a24ed0dfe5f064beb53a7e684f1";
|
||||
});
|
||||
|
||||
import { fetchWithTrustedCertificatesAsync } from "@homarr/core/infrastructure/http";
|
||||
|
||||
import type { IntegrationSecret } from "../../base/types";
|
||||
import { IntegrationParseError } from "../../base/errors/parse/integration-parse-error";
|
||||
import { LlamacppIntegration } from "../llamacpp-integration";
|
||||
import {
|
||||
avgTokensPerSecond,
|
||||
mapContextUsage,
|
||||
mapLlamacppModel,
|
||||
mapLlamacppPerRequest,
|
||||
mapSlotsSummary,
|
||||
parsePrometheusMetrics,
|
||||
requestSpeedTps,
|
||||
} from "../llamacpp-types";
|
||||
|
||||
vi.mock("@homarr/core/infrastructure/http", () => ({
|
||||
fetchWithTrustedCertificatesAsync: vi.fn(),
|
||||
}));
|
||||
|
||||
const TEST_URL = "http://llamacpp.example.com";
|
||||
const mockFetch = vi.mocked(fetchWithTrustedCertificatesAsync);
|
||||
|
||||
const sampleModel = {
|
||||
id: "/root/models/unsloth/Qwen3.8-27B-UD-Q4_K_XL.gguf",
|
||||
meta: {
|
||||
n_ctx: 131072,
|
||||
n_params: 27320697856,
|
||||
size: 17912397824,
|
||||
ftype: "Q4_K - Small",
|
||||
},
|
||||
};
|
||||
|
||||
const sampleModelsResponse = {
|
||||
data: [sampleModel],
|
||||
};
|
||||
|
||||
const sampleMetricsText = [
|
||||
"# HELP llamacpp:prompt_tokens_total Total number of tokens processed",
|
||||
"# TYPE llamacpp:prompt_tokens_total counter",
|
||||
"llamacpp:prompt_tokens_total 69834",
|
||||
"llamacpp:prompt_seconds_total 23278",
|
||||
"llamacpp:prompt_tokens_cached_total 12000",
|
||||
"llamacpp:tokens_predicted_total 12161",
|
||||
"llamacpp:tokens_predicted_seconds_total 380.03125",
|
||||
"llamacpp:requests_processing 1",
|
||||
"llamacpp:requests_deferred 0",
|
||||
"llamacpp:predicted_tokens_seconds 32.8472",
|
||||
"llamacpp:prompt_tokens_seconds 1500.5",
|
||||
"llamacpp:spec_decode_num_draft_tokens_total 128",
|
||||
"llamacpp:spec_decode_num_accepted_tokens_total 96",
|
||||
"llamacpp:spec_decode_num_drafts_total 16",
|
||||
].join("\n");
|
||||
|
||||
const sampleSlotsResponse = [
|
||||
{
|
||||
id: 0,
|
||||
n_ctx: 131072,
|
||||
n_prompt_tokens: 101076,
|
||||
is_processing: true,
|
||||
speculative: false,
|
||||
id_task: 7,
|
||||
next_token: [{ n_decoded: 42, n_remain: 100, has_next_token: true, has_new_line: false }],
|
||||
},
|
||||
];
|
||||
|
||||
const jsonResponse = (body: unknown, status = 200) =>
|
||||
new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { "content-type": "application/json" },
|
||||
}) as unknown as Awaited<ReturnType<typeof fetchWithTrustedCertificatesAsync>>;
|
||||
|
||||
const textResponse = (body: string, status = 200) =>
|
||||
new Response(body, {
|
||||
status,
|
||||
headers: { "content-type": "text/plain" },
|
||||
}) as unknown as Awaited<ReturnType<typeof fetchWithTrustedCertificatesAsync>>;
|
||||
|
||||
const createIntegration = (decryptedSecrets: IntegrationSecret[] = []) =>
|
||||
new LlamacppIntegration({
|
||||
id: "test-llamacpp",
|
||||
name: "Test llama.cpp",
|
||||
url: TEST_URL,
|
||||
externalUrl: null,
|
||||
decryptedSecrets,
|
||||
});
|
||||
|
||||
const mockAllEndpoints = () => {
|
||||
mockFetch.mockReset();
|
||||
mockFetch.mockImplementation((async (input) => {
|
||||
const url = String(input);
|
||||
if (url.includes("/health")) {
|
||||
return jsonResponse({ status: "ok" });
|
||||
}
|
||||
if (url.includes("/v1/models")) {
|
||||
return jsonResponse(sampleModelsResponse);
|
||||
}
|
||||
if (url.includes("/metrics")) {
|
||||
return textResponse(sampleMetricsText);
|
||||
}
|
||||
if (url.includes("/slots")) {
|
||||
return jsonResponse(sampleSlotsResponse);
|
||||
}
|
||||
return textResponse("not found", 404);
|
||||
}) as typeof fetchWithTrustedCertificatesAsync);
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
mockFetch.mockReset();
|
||||
});
|
||||
|
||||
describe("parsePrometheusMetrics", () => {
|
||||
test("parses metric lines and skips comments and HELP/TYPE lines", () => {
|
||||
const metrics = parsePrometheusMetrics(sampleMetricsText);
|
||||
|
||||
expect(metrics).toStrictEqual([
|
||||
{ name: "llamacpp:prompt_tokens_total", value: 69834 },
|
||||
{ name: "llamacpp:prompt_seconds_total", value: 23278 },
|
||||
{ name: "llamacpp:prompt_tokens_cached_total", value: 12000 },
|
||||
{ name: "llamacpp:tokens_predicted_total", value: 12161 },
|
||||
{ name: "llamacpp:tokens_predicted_seconds_total", value: 380.03125 },
|
||||
{ name: "llamacpp:requests_processing", value: 1 },
|
||||
{ name: "llamacpp:requests_deferred", value: 0 },
|
||||
{ name: "llamacpp:predicted_tokens_seconds", value: 32.8472 },
|
||||
{ name: "llamacpp:prompt_tokens_seconds", value: 1500.5 },
|
||||
{ name: "llamacpp:spec_decode_num_draft_tokens_total", value: 128 },
|
||||
{ name: "llamacpp:spec_decode_num_accepted_tokens_total", value: 96 },
|
||||
{ name: "llamacpp:spec_decode_num_drafts_total", value: 16 },
|
||||
]);
|
||||
});
|
||||
|
||||
test("parses metric lines with labels and skips non-numeric values", () => {
|
||||
const metrics = parsePrometheusMetrics(
|
||||
[
|
||||
'llamacpp:requests_by_status{status="ok"} 5',
|
||||
"llamacpp:uptime_seconds +Inf",
|
||||
"llamacpp:nan_metric NaN",
|
||||
"",
|
||||
].join("\n"),
|
||||
);
|
||||
|
||||
expect(metrics).toStrictEqual([{ name: "llamacpp:requests_by_status", value: 5 }]);
|
||||
});
|
||||
|
||||
test("returns an empty array for empty input", () => {
|
||||
expect(parsePrometheusMetrics("")).toStrictEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("mapLlamacppModel", () => {
|
||||
test("extracts a short display name and metadata fields", () => {
|
||||
const model = mapLlamacppModel(sampleModel);
|
||||
|
||||
expect(model).toStrictEqual({
|
||||
id: "/root/models/unsloth/Qwen3.8-27B-UD-Q4_K_XL.gguf",
|
||||
name: "Qwen3.8-27B-UD-Q4_K_XL",
|
||||
contextSize: 131072,
|
||||
parameterCount: 27320697856,
|
||||
fileSizeBytes: 17912397824,
|
||||
quantization: "Q4_K - Small",
|
||||
});
|
||||
});
|
||||
|
||||
test("returns null metadata when meta is missing", () => {
|
||||
const model = mapLlamacppModel({ id: "models/foo.gguf" });
|
||||
|
||||
expect(model).toStrictEqual({
|
||||
id: "models/foo.gguf",
|
||||
name: "foo",
|
||||
contextSize: null,
|
||||
parameterCount: null,
|
||||
fileSizeBytes: null,
|
||||
quantization: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("mapContextUsage", () => {
|
||||
test("computes percent from a slot with valid n_ctx and n_prompt_tokens", () => {
|
||||
expect(mapContextUsage(sampleSlotsResponse)).toStrictEqual({
|
||||
usedTokens: 101076,
|
||||
contextSize: 131072,
|
||||
percent: 77.1,
|
||||
});
|
||||
});
|
||||
|
||||
test("returns the slot with the greatest used tokens, not the first valid slot", () => {
|
||||
expect(
|
||||
mapContextUsage([
|
||||
{ n_ctx: 1000, n_prompt_tokens: 100 },
|
||||
{ n_ctx: 8192, n_prompt_tokens: 6000 },
|
||||
{ n_ctx: 4096, n_prompt_tokens: 3000 },
|
||||
]),
|
||||
).toStrictEqual({
|
||||
usedTokens: 6000,
|
||||
contextSize: 8192,
|
||||
percent: 73.2,
|
||||
});
|
||||
});
|
||||
|
||||
test("returns null for missing, non-array, or malformed slots", () => {
|
||||
expect(mapContextUsage(null)).toBeNull();
|
||||
expect(mapContextUsage([{ n_ctx: 0, n_prompt_tokens: 10 }])).toBeNull();
|
||||
expect(mapContextUsage([{ n_prompt_tokens: 10 }])).toBeNull();
|
||||
expect(mapContextUsage(["not-an-object", null])).toBeNull();
|
||||
});
|
||||
|
||||
test("clamps the percentage to 100", () => {
|
||||
expect(mapContextUsage([{ n_ctx: 10, n_prompt_tokens: 50 }])).toStrictEqual({
|
||||
usedTokens: 50,
|
||||
contextSize: 10,
|
||||
percent: 100,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("avgTokensPerSecond", () => {
|
||||
test("divides cumulative counters", () => {
|
||||
expect(avgTokensPerSecond(12161, 380.03125)).toBe(32);
|
||||
});
|
||||
|
||||
test("returns null when either counter is missing or seconds are zero", () => {
|
||||
expect(avgTokensPerSecond(null, 5)).toBeNull();
|
||||
expect(avgTokensPerSecond(10, null)).toBeNull();
|
||||
expect(avgTokensPerSecond(10, 0)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("mapLlamacppPerRequest", () => {
|
||||
test("returns the active request id and decoded tokens from the processing slot", () => {
|
||||
expect(mapLlamacppPerRequest(sampleSlotsResponse)).toStrictEqual({ taskId: 7, decodedTokens: 42 });
|
||||
});
|
||||
|
||||
test("skips idle slots and returns nulls", () => {
|
||||
expect(
|
||||
mapLlamacppPerRequest([{ is_processing: false, id_task: 3, next_token: [{ n_decoded: 10 }] }]),
|
||||
).toStrictEqual({ taskId: null, decodedTokens: null });
|
||||
});
|
||||
|
||||
test("returns nulls for missing, empty, or malformed slots", () => {
|
||||
expect(mapLlamacppPerRequest(null)).toStrictEqual({ taskId: null, decodedTokens: null });
|
||||
expect(mapLlamacppPerRequest([])).toStrictEqual({ taskId: null, decodedTokens: null });
|
||||
expect(mapLlamacppPerRequest(["nope", null])).toStrictEqual({ taskId: null, decodedTokens: null });
|
||||
});
|
||||
|
||||
test("handles a processing slot missing the per-request fields", () => {
|
||||
expect(mapLlamacppPerRequest([{ is_processing: true }])).toStrictEqual({
|
||||
taskId: null,
|
||||
decodedTokens: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("mapSlotsSummary", () => {
|
||||
test("counts slots, processing slots, decoding slots and speculative flags", () => {
|
||||
expect(
|
||||
mapSlotsSummary([
|
||||
{ is_processing: true, next_token: [{ n_decoded: 4 }], speculative: false },
|
||||
{ is_processing: false, speculative: true },
|
||||
{ is_processing: true, next_token: [], speculative: false },
|
||||
]),
|
||||
).toStrictEqual({ total: 3, processing: 2, decoding: 1, speculative: true });
|
||||
});
|
||||
|
||||
test("treats an empty next_token list as not decoding", () => {
|
||||
expect(mapSlotsSummary([{ is_processing: true, next_token: [] }, { is_processing: false }])).toStrictEqual({
|
||||
total: 2,
|
||||
processing: 1,
|
||||
decoding: 0,
|
||||
speculative: null,
|
||||
});
|
||||
});
|
||||
|
||||
test("returns null for missing, empty, or malformed slot lists", () => {
|
||||
expect(mapSlotsSummary(null)).toBeNull();
|
||||
expect(mapSlotsSummary([])).toBeNull();
|
||||
expect(mapSlotsSummary(["not-an-object", null])).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("requestSpeedTps", () => {
|
||||
test("divides token delta by elapsed seconds", () => {
|
||||
expect(requestSpeedTps(128, 4)).toBe(32);
|
||||
});
|
||||
|
||||
test("returns null for missing values, non-positive, or negative elapsed", () => {
|
||||
expect(requestSpeedTps(null, 4)).toBeNull();
|
||||
expect(requestSpeedTps(10, null)).toBeNull();
|
||||
expect(requestSpeedTps(10, 0)).toBeNull();
|
||||
expect(requestSpeedTps(10, -2)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("LlamacppIntegration getStatsAsync", () => {
|
||||
test("fetches health, models, metrics and slots and maps them into stats", async () => {
|
||||
mockAllEndpoints();
|
||||
|
||||
const stats = await createIntegration().getStatsAsync();
|
||||
|
||||
expect(stats).toStrictEqual({
|
||||
health: "ok",
|
||||
model: {
|
||||
id: "/root/models/unsloth/Qwen3.8-27B-UD-Q4_K_XL.gguf",
|
||||
name: "Qwen3.8-27B-UD-Q4_K_XL",
|
||||
contextSize: 131072,
|
||||
parameterCount: 27320697856,
|
||||
fileSizeBytes: 17912397824,
|
||||
quantization: "Q4_K - Small",
|
||||
},
|
||||
contextUsage: {
|
||||
usedTokens: 101076,
|
||||
contextSize: 131072,
|
||||
percent: 77.1,
|
||||
},
|
||||
slots: { total: 1, processing: 1, decoding: 1, speculative: false },
|
||||
metrics: {
|
||||
generationSpeedTps: 32.8472,
|
||||
promptSpeedTps: 1500.5,
|
||||
avgGenerationSpeedTps: 32,
|
||||
avgPromptSpeedTps: 3,
|
||||
requestsProcessing: 1,
|
||||
requestsDeferred: 0,
|
||||
tokensProcessed: 69834,
|
||||
tokensGenerated: 12161,
|
||||
promptCacheHitRate: 15,
|
||||
requestDecodedTokens: 42,
|
||||
taskId: 7,
|
||||
specDraftTokens: 128,
|
||||
specAcceptedTokens: 96,
|
||||
specDrafts: 16,
|
||||
},
|
||||
});
|
||||
|
||||
const requestedUrls = mockFetch.mock.calls.map((call) => String(call[0])).toSorted();
|
||||
expect(requestedUrls).toStrictEqual(
|
||||
[`${TEST_URL}/health`, `${TEST_URL}/metrics`, `${TEST_URL}/slots`, `${TEST_URL}/v1/models`].toSorted(),
|
||||
);
|
||||
});
|
||||
|
||||
test("returns null model when the server has no models loaded", async () => {
|
||||
mockAllEndpoints();
|
||||
mockFetch.mockImplementation((async (input) => {
|
||||
const url = String(input);
|
||||
if (url.includes("/v1/models")) {
|
||||
return jsonResponse({ data: [] });
|
||||
}
|
||||
if (url.includes("/health")) {
|
||||
return jsonResponse({ status: "ok" });
|
||||
}
|
||||
if (url.includes("/slots")) {
|
||||
return jsonResponse([]);
|
||||
}
|
||||
return textResponse(sampleMetricsText);
|
||||
}) as typeof fetchWithTrustedCertificatesAsync);
|
||||
|
||||
const stats = await createIntegration().getStatsAsync();
|
||||
|
||||
expect(stats.model).toBeNull();
|
||||
expect(stats.metrics.generationSpeedTps).toBe(32.8472);
|
||||
});
|
||||
|
||||
test("returns null for metrics that are absent from /metrics", async () => {
|
||||
mockAllEndpoints();
|
||||
mockFetch.mockImplementation((async (input) => {
|
||||
const url = String(input);
|
||||
if (url.includes("/health")) {
|
||||
return jsonResponse({ status: "ok" });
|
||||
}
|
||||
if (url.includes("/v1/models")) {
|
||||
return jsonResponse({ data: [] });
|
||||
}
|
||||
if (url.includes("/slots")) {
|
||||
return jsonResponse([]);
|
||||
}
|
||||
return textResponse("");
|
||||
}) as typeof fetchWithTrustedCertificatesAsync);
|
||||
|
||||
const stats = await createIntegration().getStatsAsync();
|
||||
|
||||
expect(stats.metrics).toStrictEqual({
|
||||
generationSpeedTps: null,
|
||||
promptSpeedTps: null,
|
||||
avgGenerationSpeedTps: null,
|
||||
avgPromptSpeedTps: null,
|
||||
requestsProcessing: null,
|
||||
requestsDeferred: null,
|
||||
tokensProcessed: null,
|
||||
tokensGenerated: null,
|
||||
promptCacheHitRate: null,
|
||||
requestDecodedTokens: null,
|
||||
taskId: null,
|
||||
specDraftTokens: null,
|
||||
specAcceptedTokens: null,
|
||||
specDrafts: null,
|
||||
});
|
||||
});
|
||||
|
||||
test("throws when the /health endpoint returns an error status", async () => {
|
||||
mockAllEndpoints();
|
||||
mockFetch.mockImplementation((async (input) => {
|
||||
const url = String(input);
|
||||
if (url.includes("/health")) {
|
||||
return textResponse("Service Unavailable", 503);
|
||||
}
|
||||
return jsonResponse({ status: "ok" });
|
||||
}) as typeof fetchWithTrustedCertificatesAsync);
|
||||
|
||||
await expect(createIntegration().getStatsAsync()).rejects.toThrow();
|
||||
});
|
||||
|
||||
test("degrades to null metrics when /metrics is not available (501) but keeps health and model data", async () => {
|
||||
mockAllEndpoints();
|
||||
mockFetch.mockImplementation((async (input) => {
|
||||
const url = String(input);
|
||||
if (url.includes("/metrics")) {
|
||||
return textResponse("Not Implemented", 501);
|
||||
}
|
||||
if (url.includes("/health")) {
|
||||
return jsonResponse({ status: "ok" });
|
||||
}
|
||||
if (url.includes("/v1/models")) {
|
||||
return jsonResponse(sampleModelsResponse);
|
||||
}
|
||||
if (url.includes("/slots")) {
|
||||
return jsonResponse(sampleSlotsResponse);
|
||||
}
|
||||
return textResponse("not found", 404);
|
||||
}) as typeof fetchWithTrustedCertificatesAsync);
|
||||
|
||||
const stats = await createIntegration().getStatsAsync();
|
||||
|
||||
expect(stats.health).toBe("ok");
|
||||
expect(stats.model?.name).toBe("Qwen3.8-27B-UD-Q4_K_XL");
|
||||
expect(stats.contextUsage).toStrictEqual({
|
||||
usedTokens: 101076,
|
||||
contextSize: 131072,
|
||||
percent: 77.1,
|
||||
});
|
||||
expect(stats.slots).toStrictEqual({ total: 1, processing: 1, decoding: 1, speculative: false });
|
||||
expect(stats.metrics).toStrictEqual({
|
||||
generationSpeedTps: null,
|
||||
promptSpeedTps: null,
|
||||
avgGenerationSpeedTps: null,
|
||||
avgPromptSpeedTps: null,
|
||||
requestsProcessing: null,
|
||||
requestsDeferred: null,
|
||||
tokensProcessed: null,
|
||||
tokensGenerated: null,
|
||||
promptCacheHitRate: null,
|
||||
requestDecodedTokens: 42,
|
||||
taskId: 7,
|
||||
specDraftTokens: null,
|
||||
specAcceptedTokens: null,
|
||||
specDrafts: null,
|
||||
});
|
||||
});
|
||||
|
||||
test("throws a parse error when /v1/models is not valid JSON", async () => {
|
||||
mockAllEndpoints();
|
||||
mockFetch.mockImplementation((async (input) => {
|
||||
const url = String(input);
|
||||
if (url.includes("/v1/models")) {
|
||||
return textResponse("not-json");
|
||||
}
|
||||
if (url.includes("/health")) {
|
||||
return jsonResponse({ status: "ok" });
|
||||
}
|
||||
if (url.includes("/slots")) {
|
||||
return jsonResponse([]);
|
||||
}
|
||||
return textResponse(sampleMetricsText);
|
||||
}) as typeof fetchWithTrustedCertificatesAsync);
|
||||
|
||||
await expect(createIntegration().getStatsAsync()).rejects.toSatisfy((error) => {
|
||||
if (!(error instanceof IntegrationParseError)) return false;
|
||||
const cause = error.cause;
|
||||
return cause instanceof ParseError && cause.message.includes("Invalid llama.cpp /v1/models response");
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user