feat!: release v2 (#6545)
Co-authored-by: Artur <145776561+arturict@users.noreply.github.com> Co-authored-by: arturict <arturict@users.noreply.github.com> Co-authored-by: Thomas <thomas@ajnart.fr> Co-authored-by: Afonso Jorge Ramos <afonsojorgeramos@gmail.com> Co-authored-by: Danura30082 <danura30082+github@gmail.com> Co-authored-by: Kushie <122986913+kushiemoon-dev@users.noreply.github.com> Co-authored-by: ajnart <thomas@ajnart.fr> Co-authored-by: ajnart <thomas@ajnart.dev> Co-authored-by: chenjun <chenjun@foxmail.com> Co-authored-by: Wikibear <wikibear@users.noreply.github.com> Co-authored-by: Bergasha <bergasha@gmail.com> Co-authored-by: leafyelin <43306196+leafyelin@users.noreply.github.com> Co-authored-by: Lucas Araujo <lucas.lra@gmail.com> Co-authored-by: GreenMonito <86085570+GreenMonito@users.noreply.github.com> Co-authored-by: Nerdy-Technician <roffo@nerdy-technician.social> Co-authored-by: Jun <34545896+jcstrong@users.noreply.github.com> Co-authored-by: Edmond Abraham <54119952+gitedmond@users.noreply.github.com> Co-authored-by: thrawnx <tight.bulb9384@fastmail.com> Co-authored-by: Chris <58918656+nordada@users.noreply.github.com> Co-authored-by: yi111 <153097222+Yi-111-a@users.noreply.github.com> Co-authored-by: Yanis <naamaneyanis@gmail.com> Co-authored-by: Wikibear <me@sej.de> Co-authored-by: alessiottavo <93441886+alessiottavo@users.noreply.github.com>
This commit is contained in:
3286 files changed
+284386
-139176
No files matched your search
@@ -0,0 +1,489 @@
|
||||
// @vitest-environment node
|
||||
import { ParseError } from "@homarr/common/server";
|
||||
import { Response } from "undici";
|
||||
import { beforeEach, describe, expect, test, vi } from "vitest";
|
||||
|
||||
vi.hoisted(() => {
|
||||
process.env.SKIP_ENV_VALIDATION = "true";
|
||||
process.env.SECRET_ENCRYPTION_KEY = "ff3f4f7ce30e870c9630de9e5d244ffa81101a24ed0dfe5f064beb53a7e684f1";
|
||||
});
|
||||
|
||||
import { fetchWithTrustedCertificatesAsync } from "@homarr/core/infrastructure/http";
|
||||
|
||||
import type { IntegrationSecret } from "../../base/types";
|
||||
import { IntegrationParseError } from "../../base/errors/parse/integration-parse-error";
|
||||
import { LlamacppIntegration } from "../llamacpp-integration";
|
||||
import {
|
||||
avgTokensPerSecond,
|
||||
mapContextUsage,
|
||||
mapLlamacppModel,
|
||||
mapLlamacppPerRequest,
|
||||
mapSlotsSummary,
|
||||
parsePrometheusMetrics,
|
||||
requestSpeedTps,
|
||||
} from "../llamacpp-types";
|
||||
|
||||
vi.mock("@homarr/core/infrastructure/http", () => ({
|
||||
fetchWithTrustedCertificatesAsync: vi.fn(),
|
||||
}));
|
||||
|
||||
const TEST_URL = "http://llamacpp.example.com";
|
||||
const mockFetch = vi.mocked(fetchWithTrustedCertificatesAsync);
|
||||
|
||||
const sampleModel = {
|
||||
id: "/root/models/unsloth/Qwen3.8-27B-UD-Q4_K_XL.gguf",
|
||||
meta: {
|
||||
n_ctx: 131072,
|
||||
n_params: 27320697856,
|
||||
size: 17912397824,
|
||||
ftype: "Q4_K - Small",
|
||||
},
|
||||
};
|
||||
|
||||
const sampleModelsResponse = {
|
||||
data: [sampleModel],
|
||||
};
|
||||
|
||||
const sampleMetricsText = [
|
||||
"# HELP llamacpp:prompt_tokens_total Total number of tokens processed",
|
||||
"# TYPE llamacpp:prompt_tokens_total counter",
|
||||
"llamacpp:prompt_tokens_total 69834",
|
||||
"llamacpp:prompt_seconds_total 23278",
|
||||
"llamacpp:prompt_tokens_cached_total 12000",
|
||||
"llamacpp:tokens_predicted_total 12161",
|
||||
"llamacpp:tokens_predicted_seconds_total 380.03125",
|
||||
"llamacpp:requests_processing 1",
|
||||
"llamacpp:requests_deferred 0",
|
||||
"llamacpp:predicted_tokens_seconds 32.8472",
|
||||
"llamacpp:prompt_tokens_seconds 1500.5",
|
||||
"llamacpp:spec_decode_num_draft_tokens_total 128",
|
||||
"llamacpp:spec_decode_num_accepted_tokens_total 96",
|
||||
"llamacpp:spec_decode_num_drafts_total 16",
|
||||
].join("\n");
|
||||
|
||||
const sampleSlotsResponse = [
|
||||
{
|
||||
id: 0,
|
||||
n_ctx: 131072,
|
||||
n_prompt_tokens: 101076,
|
||||
is_processing: true,
|
||||
speculative: false,
|
||||
id_task: 7,
|
||||
next_token: [{ n_decoded: 42, n_remain: 100, has_next_token: true, has_new_line: false }],
|
||||
},
|
||||
];
|
||||
|
||||
const jsonResponse = (body: unknown, status = 200) =>
|
||||
new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { "content-type": "application/json" },
|
||||
}) as unknown as Awaited<ReturnType<typeof fetchWithTrustedCertificatesAsync>>;
|
||||
|
||||
const textResponse = (body: string, status = 200) =>
|
||||
new Response(body, {
|
||||
status,
|
||||
headers: { "content-type": "text/plain" },
|
||||
}) as unknown as Awaited<ReturnType<typeof fetchWithTrustedCertificatesAsync>>;
|
||||
|
||||
const createIntegration = (decryptedSecrets: IntegrationSecret[] = []) =>
|
||||
new LlamacppIntegration({
|
||||
id: "test-llamacpp",
|
||||
name: "Test llama.cpp",
|
||||
url: TEST_URL,
|
||||
externalUrl: null,
|
||||
decryptedSecrets,
|
||||
});
|
||||
|
||||
const mockAllEndpoints = () => {
|
||||
mockFetch.mockReset();
|
||||
mockFetch.mockImplementation((async (input) => {
|
||||
const url = String(input);
|
||||
if (url.includes("/health")) {
|
||||
return jsonResponse({ status: "ok" });
|
||||
}
|
||||
if (url.includes("/v1/models")) {
|
||||
return jsonResponse(sampleModelsResponse);
|
||||
}
|
||||
if (url.includes("/metrics")) {
|
||||
return textResponse(sampleMetricsText);
|
||||
}
|
||||
if (url.includes("/slots")) {
|
||||
return jsonResponse(sampleSlotsResponse);
|
||||
}
|
||||
return textResponse("not found", 404);
|
||||
}) as typeof fetchWithTrustedCertificatesAsync);
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
mockFetch.mockReset();
|
||||
});
|
||||
|
||||
describe("parsePrometheusMetrics", () => {
|
||||
test("parses metric lines and skips comments and HELP/TYPE lines", () => {
|
||||
const metrics = parsePrometheusMetrics(sampleMetricsText);
|
||||
|
||||
expect(metrics).toStrictEqual([
|
||||
{ name: "llamacpp:prompt_tokens_total", value: 69834 },
|
||||
{ name: "llamacpp:prompt_seconds_total", value: 23278 },
|
||||
{ name: "llamacpp:prompt_tokens_cached_total", value: 12000 },
|
||||
{ name: "llamacpp:tokens_predicted_total", value: 12161 },
|
||||
{ name: "llamacpp:tokens_predicted_seconds_total", value: 380.03125 },
|
||||
{ name: "llamacpp:requests_processing", value: 1 },
|
||||
{ name: "llamacpp:requests_deferred", value: 0 },
|
||||
{ name: "llamacpp:predicted_tokens_seconds", value: 32.8472 },
|
||||
{ name: "llamacpp:prompt_tokens_seconds", value: 1500.5 },
|
||||
{ name: "llamacpp:spec_decode_num_draft_tokens_total", value: 128 },
|
||||
{ name: "llamacpp:spec_decode_num_accepted_tokens_total", value: 96 },
|
||||
{ name: "llamacpp:spec_decode_num_drafts_total", value: 16 },
|
||||
]);
|
||||
});
|
||||
|
||||
test("parses metric lines with labels and skips non-numeric values", () => {
|
||||
const metrics = parsePrometheusMetrics(
|
||||
[
|
||||
'llamacpp:requests_by_status{status="ok"} 5',
|
||||
"llamacpp:uptime_seconds +Inf",
|
||||
"llamacpp:nan_metric NaN",
|
||||
"",
|
||||
].join("\n"),
|
||||
);
|
||||
|
||||
expect(metrics).toStrictEqual([{ name: "llamacpp:requests_by_status", value: 5 }]);
|
||||
});
|
||||
|
||||
test("returns an empty array for empty input", () => {
|
||||
expect(parsePrometheusMetrics("")).toStrictEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("mapLlamacppModel", () => {
|
||||
test("extracts a short display name and metadata fields", () => {
|
||||
const model = mapLlamacppModel(sampleModel);
|
||||
|
||||
expect(model).toStrictEqual({
|
||||
id: "/root/models/unsloth/Qwen3.8-27B-UD-Q4_K_XL.gguf",
|
||||
name: "Qwen3.8-27B-UD-Q4_K_XL",
|
||||
contextSize: 131072,
|
||||
parameterCount: 27320697856,
|
||||
fileSizeBytes: 17912397824,
|
||||
quantization: "Q4_K - Small",
|
||||
});
|
||||
});
|
||||
|
||||
test("returns null metadata when meta is missing", () => {
|
||||
const model = mapLlamacppModel({ id: "models/foo.gguf" });
|
||||
|
||||
expect(model).toStrictEqual({
|
||||
id: "models/foo.gguf",
|
||||
name: "foo",
|
||||
contextSize: null,
|
||||
parameterCount: null,
|
||||
fileSizeBytes: null,
|
||||
quantization: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("mapContextUsage", () => {
|
||||
test("computes percent from a slot with valid n_ctx and n_prompt_tokens", () => {
|
||||
expect(mapContextUsage(sampleSlotsResponse)).toStrictEqual({
|
||||
usedTokens: 101076,
|
||||
contextSize: 131072,
|
||||
percent: 77.1,
|
||||
});
|
||||
});
|
||||
|
||||
test("returns the slot with the greatest used tokens, not the first valid slot", () => {
|
||||
expect(
|
||||
mapContextUsage([
|
||||
{ n_ctx: 1000, n_prompt_tokens: 100 },
|
||||
{ n_ctx: 8192, n_prompt_tokens: 6000 },
|
||||
{ n_ctx: 4096, n_prompt_tokens: 3000 },
|
||||
]),
|
||||
).toStrictEqual({
|
||||
usedTokens: 6000,
|
||||
contextSize: 8192,
|
||||
percent: 73.2,
|
||||
});
|
||||
});
|
||||
|
||||
test("returns null for missing, non-array, or malformed slots", () => {
|
||||
expect(mapContextUsage(null)).toBeNull();
|
||||
expect(mapContextUsage([{ n_ctx: 0, n_prompt_tokens: 10 }])).toBeNull();
|
||||
expect(mapContextUsage([{ n_prompt_tokens: 10 }])).toBeNull();
|
||||
expect(mapContextUsage(["not-an-object", null])).toBeNull();
|
||||
});
|
||||
|
||||
test("clamps the percentage to 100", () => {
|
||||
expect(mapContextUsage([{ n_ctx: 10, n_prompt_tokens: 50 }])).toStrictEqual({
|
||||
usedTokens: 50,
|
||||
contextSize: 10,
|
||||
percent: 100,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("avgTokensPerSecond", () => {
|
||||
test("divides cumulative counters", () => {
|
||||
expect(avgTokensPerSecond(12161, 380.03125)).toBe(32);
|
||||
});
|
||||
|
||||
test("returns null when either counter is missing or seconds are zero", () => {
|
||||
expect(avgTokensPerSecond(null, 5)).toBeNull();
|
||||
expect(avgTokensPerSecond(10, null)).toBeNull();
|
||||
expect(avgTokensPerSecond(10, 0)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("mapLlamacppPerRequest", () => {
|
||||
test("returns the active request id and decoded tokens from the processing slot", () => {
|
||||
expect(mapLlamacppPerRequest(sampleSlotsResponse)).toStrictEqual({ taskId: 7, decodedTokens: 42 });
|
||||
});
|
||||
|
||||
test("skips idle slots and returns nulls", () => {
|
||||
expect(
|
||||
mapLlamacppPerRequest([{ is_processing: false, id_task: 3, next_token: [{ n_decoded: 10 }] }]),
|
||||
).toStrictEqual({ taskId: null, decodedTokens: null });
|
||||
});
|
||||
|
||||
test("returns nulls for missing, empty, or malformed slots", () => {
|
||||
expect(mapLlamacppPerRequest(null)).toStrictEqual({ taskId: null, decodedTokens: null });
|
||||
expect(mapLlamacppPerRequest([])).toStrictEqual({ taskId: null, decodedTokens: null });
|
||||
expect(mapLlamacppPerRequest(["nope", null])).toStrictEqual({ taskId: null, decodedTokens: null });
|
||||
});
|
||||
|
||||
test("handles a processing slot missing the per-request fields", () => {
|
||||
expect(mapLlamacppPerRequest([{ is_processing: true }])).toStrictEqual({
|
||||
taskId: null,
|
||||
decodedTokens: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("mapSlotsSummary", () => {
|
||||
test("counts slots, processing slots, decoding slots and speculative flags", () => {
|
||||
expect(
|
||||
mapSlotsSummary([
|
||||
{ is_processing: true, next_token: [{ n_decoded: 4 }], speculative: false },
|
||||
{ is_processing: false, speculative: true },
|
||||
{ is_processing: true, next_token: [], speculative: false },
|
||||
]),
|
||||
).toStrictEqual({ total: 3, processing: 2, decoding: 1, speculative: true });
|
||||
});
|
||||
|
||||
test("treats an empty next_token list as not decoding", () => {
|
||||
expect(mapSlotsSummary([{ is_processing: true, next_token: [] }, { is_processing: false }])).toStrictEqual({
|
||||
total: 2,
|
||||
processing: 1,
|
||||
decoding: 0,
|
||||
speculative: null,
|
||||
});
|
||||
});
|
||||
|
||||
test("returns null for missing, empty, or malformed slot lists", () => {
|
||||
expect(mapSlotsSummary(null)).toBeNull();
|
||||
expect(mapSlotsSummary([])).toBeNull();
|
||||
expect(mapSlotsSummary(["not-an-object", null])).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("requestSpeedTps", () => {
|
||||
test("divides token delta by elapsed seconds", () => {
|
||||
expect(requestSpeedTps(128, 4)).toBe(32);
|
||||
});
|
||||
|
||||
test("returns null for missing values, non-positive, or negative elapsed", () => {
|
||||
expect(requestSpeedTps(null, 4)).toBeNull();
|
||||
expect(requestSpeedTps(10, null)).toBeNull();
|
||||
expect(requestSpeedTps(10, 0)).toBeNull();
|
||||
expect(requestSpeedTps(10, -2)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("LlamacppIntegration getStatsAsync", () => {
|
||||
test("fetches health, models, metrics and slots and maps them into stats", async () => {
|
||||
mockAllEndpoints();
|
||||
|
||||
const stats = await createIntegration().getStatsAsync();
|
||||
|
||||
expect(stats).toStrictEqual({
|
||||
health: "ok",
|
||||
model: {
|
||||
id: "/root/models/unsloth/Qwen3.8-27B-UD-Q4_K_XL.gguf",
|
||||
name: "Qwen3.8-27B-UD-Q4_K_XL",
|
||||
contextSize: 131072,
|
||||
parameterCount: 27320697856,
|
||||
fileSizeBytes: 17912397824,
|
||||
quantization: "Q4_K - Small",
|
||||
},
|
||||
contextUsage: {
|
||||
usedTokens: 101076,
|
||||
contextSize: 131072,
|
||||
percent: 77.1,
|
||||
},
|
||||
slots: { total: 1, processing: 1, decoding: 1, speculative: false },
|
||||
metrics: {
|
||||
generationSpeedTps: 32.8472,
|
||||
promptSpeedTps: 1500.5,
|
||||
avgGenerationSpeedTps: 32,
|
||||
avgPromptSpeedTps: 3,
|
||||
requestsProcessing: 1,
|
||||
requestsDeferred: 0,
|
||||
tokensProcessed: 69834,
|
||||
tokensGenerated: 12161,
|
||||
promptCacheHitRate: 15,
|
||||
requestDecodedTokens: 42,
|
||||
taskId: 7,
|
||||
specDraftTokens: 128,
|
||||
specAcceptedTokens: 96,
|
||||
specDrafts: 16,
|
||||
},
|
||||
});
|
||||
|
||||
const requestedUrls = mockFetch.mock.calls.map((call) => String(call[0])).toSorted();
|
||||
expect(requestedUrls).toStrictEqual(
|
||||
[`${TEST_URL}/health`, `${TEST_URL}/metrics`, `${TEST_URL}/slots`, `${TEST_URL}/v1/models`].toSorted(),
|
||||
);
|
||||
});
|
||||
|
||||
test("returns null model when the server has no models loaded", async () => {
|
||||
mockAllEndpoints();
|
||||
mockFetch.mockImplementation((async (input) => {
|
||||
const url = String(input);
|
||||
if (url.includes("/v1/models")) {
|
||||
return jsonResponse({ data: [] });
|
||||
}
|
||||
if (url.includes("/health")) {
|
||||
return jsonResponse({ status: "ok" });
|
||||
}
|
||||
if (url.includes("/slots")) {
|
||||
return jsonResponse([]);
|
||||
}
|
||||
return textResponse(sampleMetricsText);
|
||||
}) as typeof fetchWithTrustedCertificatesAsync);
|
||||
|
||||
const stats = await createIntegration().getStatsAsync();
|
||||
|
||||
expect(stats.model).toBeNull();
|
||||
expect(stats.metrics.generationSpeedTps).toBe(32.8472);
|
||||
});
|
||||
|
||||
test("returns null for metrics that are absent from /metrics", async () => {
|
||||
mockAllEndpoints();
|
||||
mockFetch.mockImplementation((async (input) => {
|
||||
const url = String(input);
|
||||
if (url.includes("/health")) {
|
||||
return jsonResponse({ status: "ok" });
|
||||
}
|
||||
if (url.includes("/v1/models")) {
|
||||
return jsonResponse({ data: [] });
|
||||
}
|
||||
if (url.includes("/slots")) {
|
||||
return jsonResponse([]);
|
||||
}
|
||||
return textResponse("");
|
||||
}) as typeof fetchWithTrustedCertificatesAsync);
|
||||
|
||||
const stats = await createIntegration().getStatsAsync();
|
||||
|
||||
expect(stats.metrics).toStrictEqual({
|
||||
generationSpeedTps: null,
|
||||
promptSpeedTps: null,
|
||||
avgGenerationSpeedTps: null,
|
||||
avgPromptSpeedTps: null,
|
||||
requestsProcessing: null,
|
||||
requestsDeferred: null,
|
||||
tokensProcessed: null,
|
||||
tokensGenerated: null,
|
||||
promptCacheHitRate: null,
|
||||
requestDecodedTokens: null,
|
||||
taskId: null,
|
||||
specDraftTokens: null,
|
||||
specAcceptedTokens: null,
|
||||
specDrafts: null,
|
||||
});
|
||||
});
|
||||
|
||||
test("throws when the /health endpoint returns an error status", async () => {
|
||||
mockAllEndpoints();
|
||||
mockFetch.mockImplementation((async (input) => {
|
||||
const url = String(input);
|
||||
if (url.includes("/health")) {
|
||||
return textResponse("Service Unavailable", 503);
|
||||
}
|
||||
return jsonResponse({ status: "ok" });
|
||||
}) as typeof fetchWithTrustedCertificatesAsync);
|
||||
|
||||
await expect(createIntegration().getStatsAsync()).rejects.toThrow();
|
||||
});
|
||||
|
||||
test("degrades to null metrics when /metrics is not available (501) but keeps health and model data", async () => {
|
||||
mockAllEndpoints();
|
||||
mockFetch.mockImplementation((async (input) => {
|
||||
const url = String(input);
|
||||
if (url.includes("/metrics")) {
|
||||
return textResponse("Not Implemented", 501);
|
||||
}
|
||||
if (url.includes("/health")) {
|
||||
return jsonResponse({ status: "ok" });
|
||||
}
|
||||
if (url.includes("/v1/models")) {
|
||||
return jsonResponse(sampleModelsResponse);
|
||||
}
|
||||
if (url.includes("/slots")) {
|
||||
return jsonResponse(sampleSlotsResponse);
|
||||
}
|
||||
return textResponse("not found", 404);
|
||||
}) as typeof fetchWithTrustedCertificatesAsync);
|
||||
|
||||
const stats = await createIntegration().getStatsAsync();
|
||||
|
||||
expect(stats.health).toBe("ok");
|
||||
expect(stats.model?.name).toBe("Qwen3.8-27B-UD-Q4_K_XL");
|
||||
expect(stats.contextUsage).toStrictEqual({
|
||||
usedTokens: 101076,
|
||||
contextSize: 131072,
|
||||
percent: 77.1,
|
||||
});
|
||||
expect(stats.slots).toStrictEqual({ total: 1, processing: 1, decoding: 1, speculative: false });
|
||||
expect(stats.metrics).toStrictEqual({
|
||||
generationSpeedTps: null,
|
||||
promptSpeedTps: null,
|
||||
avgGenerationSpeedTps: null,
|
||||
avgPromptSpeedTps: null,
|
||||
requestsProcessing: null,
|
||||
requestsDeferred: null,
|
||||
tokensProcessed: null,
|
||||
tokensGenerated: null,
|
||||
promptCacheHitRate: null,
|
||||
requestDecodedTokens: 42,
|
||||
taskId: 7,
|
||||
specDraftTokens: null,
|
||||
specAcceptedTokens: null,
|
||||
specDrafts: null,
|
||||
});
|
||||
});
|
||||
|
||||
test("throws a parse error when /v1/models is not valid JSON", async () => {
|
||||
mockAllEndpoints();
|
||||
mockFetch.mockImplementation((async (input) => {
|
||||
const url = String(input);
|
||||
if (url.includes("/v1/models")) {
|
||||
return textResponse("not-json");
|
||||
}
|
||||
if (url.includes("/health")) {
|
||||
return jsonResponse({ status: "ok" });
|
||||
}
|
||||
if (url.includes("/slots")) {
|
||||
return jsonResponse([]);
|
||||
}
|
||||
return textResponse(sampleMetricsText);
|
||||
}) as typeof fetchWithTrustedCertificatesAsync);
|
||||
|
||||
await expect(createIntegration().getStatsAsync()).rejects.toSatisfy((error) => {
|
||||
if (!(error instanceof IntegrationParseError)) return false;
|
||||
const cause = error.cause;
|
||||
return cause instanceof ParseError && cause.message.includes("Invalid llama.cpp /v1/models response");
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user