fix: harden the Homarr provider
This commit is contained in:
1 parent
0774b02878
commit
bffa3e723c
43 files changed
+515
-999
No files matched your search
@@ -1,6 +1,7 @@
|
||||
import { createServer } from "node:http";
|
||||
|
||||
const port = Number(process.env.PORT ?? 18091);
|
||||
const expectedModel = process.env.EXPECTED_MODEL ?? "mock/team-selected-model";
|
||||
|
||||
const json = (response, status, body) => {
|
||||
response.writeHead(status, { "content-type": "application/json" });
|
||||
@@ -24,16 +25,16 @@ const server = createServer(async (request, response) => {
|
||||
const chunks = [];
|
||||
for await (const chunk of request) chunks.push(chunk);
|
||||
const body = JSON.parse(Buffer.concat(chunks).toString("utf8"));
|
||||
if (body.model !== "~deepseek/deepseek-v4-flash-latest") {
|
||||
if (body.model !== expectedModel) {
|
||||
json(response, 400, { error: { message: "Unexpected upstream model" } });
|
||||
return;
|
||||
}
|
||||
if (body.max_tokens !== 32_768 || body.n !== 1 || body.parallel_tool_calls !== false) {
|
||||
if (body.max_completion_tokens !== 32_768 || body.n !== 1 || body.parallel_tool_calls !== false) {
|
||||
json(response, 400, { error: { message: "Unsafe generation controls reached the upstream" } });
|
||||
return;
|
||||
}
|
||||
if (
|
||||
"max_completion_tokens" in body ||
|
||||
"max_tokens" in body ||
|
||||
"user" in body ||
|
||||
"metadata" in body ||
|
||||
"extra_body" in body ||
|
||||
@@ -43,11 +44,22 @@ const server = createServer(async (request, response) => {
|
||||
json(response, 400, { error: { message: "Private or unbounded request fields reached the upstream" } });
|
||||
return;
|
||||
}
|
||||
const forbiddenRoutingField = ["models", "provider", "route", "plugins", "transforms"].find((field) => field in body);
|
||||
const forbiddenCostField = ["audio", "modalities", "logprobs", "top_logprobs", "prediction", "service_tier"].find(
|
||||
(field) => field in body,
|
||||
);
|
||||
if (forbiddenCostField) {
|
||||
json(response, 400, { error: { message: `Client cost field was forwarded: ${forbiddenCostField}` } });
|
||||
return;
|
||||
}
|
||||
const forbiddenRoutingField = ["models", "route", "plugins", "transforms"].find((field) => field in body);
|
||||
if (forbiddenRoutingField) {
|
||||
json(response, 400, { error: { message: `Client routing field was forwarded: ${forbiddenRoutingField}` } });
|
||||
return;
|
||||
}
|
||||
if (body.provider?.zdr !== true || body.provider?.data_collection !== "deny") {
|
||||
json(response, 400, { error: { message: "Required upstream privacy controls are missing" } });
|
||||
return;
|
||||
}
|
||||
const webSearchTools = body.tools?.filter((tool) => tool.type === "openrouter:web_search") ?? [];
|
||||
if (
|
||||
webSearchTools.length > 1 ||
|
||||
|
||||
@@ -75,30 +75,28 @@ for (const removedField of ["displayName", "avatarUrl", "githubProfileUrl", "git
|
||||
}
|
||||
|
||||
const providerMigration = await read("apps/workshop/pb_migrations/1786500000_homarr_provider.js");
|
||||
for (const required of [
|
||||
'name: "assistant_quotas"',
|
||||
'name: "assistant_requests"',
|
||||
'name: "assistant_activity"',
|
||||
'listRule: ""',
|
||||
"idx_assistant_quotas_user",
|
||||
]) {
|
||||
for (const required of ['name: "assistant_quotas"', 'name: "assistant_global_quota"', "idx_assistant_quotas_user"]) {
|
||||
if (!providerMigration.includes(required)) throw new Error(`Homarr provider migration is missing: ${required}`);
|
||||
}
|
||||
if (/assistant_activity[\s\S]*name: "user"/u.test(providerMigration.split("const requests")[0])) {
|
||||
throw new Error("The public assistant activity collection must not contain user identity");
|
||||
for (const forbidden of ['name: "assistant_requests"', 'name: "assistant_activity"', 'name: "dailyLimit"']) {
|
||||
if (providerMigration.includes(forbidden)) throw new Error(`Homarr provider must not persist ${forbidden}`);
|
||||
}
|
||||
|
||||
const providerBackend = await read("apps/workshop/homarr_provider.go");
|
||||
for (const required of [
|
||||
'"homarr/deepseek-v4-flash-latest"',
|
||||
'"homarr/model"',
|
||||
'apis.RequireAuth("users")',
|
||||
"DefaultActivityLoggerMiddlewareId",
|
||||
'os.Getenv("OPENROUTER_API_KEY")',
|
||||
'os.Getenv("HOMARR_AI_DAILY_REQUEST_LIMIT")',
|
||||
'os.Getenv("HOMARR_AI_GLOBAL_DAILY_REQUEST_LIMIT")',
|
||||
'os.Getenv("HOMARR_AI_OPENROUTER_MODEL")',
|
||||
'"data_collection": "deny"',
|
||||
'"zdr": true',
|
||||
]) {
|
||||
if (!providerBackend.includes(required)) throw new Error(`Homarr provider backend is missing: ${required}`);
|
||||
}
|
||||
if (/logger\(\)|\.Logger\(/u.test(providerBackend)) {
|
||||
if (/logger\(\)|\.Logger\(|log\.Print/u.test(providerBackend)) {
|
||||
throw new Error("The Homarr provider must not log request content or user tokens");
|
||||
}
|
||||
for (const required of [
|
||||
|
||||
@@ -1,34 +1,5 @@
|
||||
const baseUrl = process.env.WORKSHOP_TEST_URL ?? "http://127.0.0.1:18090";
|
||||
|
||||
const createSseReader = (body) => {
|
||||
const reader = body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = "";
|
||||
return {
|
||||
cancel: () => reader.cancel(),
|
||||
async next() {
|
||||
while (true) {
|
||||
const boundary = buffer.indexOf("\n\n");
|
||||
if (boundary >= 0) {
|
||||
const block = buffer.slice(0, boundary);
|
||||
buffer = buffer.slice(boundary + 2);
|
||||
const result = { event: "message", id: "", data: "" };
|
||||
for (const line of block.split("\n")) {
|
||||
if (line.startsWith("event:")) result.event = line.slice(6).trim();
|
||||
if (line.startsWith("id:")) result.id = line.slice(3).trim();
|
||||
if (line.startsWith("data:")) result.data += line.slice(5).trim();
|
||||
}
|
||||
if (result.data || result.id) return result;
|
||||
continue;
|
||||
}
|
||||
const chunk = await reader.read();
|
||||
if (chunk.done) throw new Error("PocketBase realtime stream ended unexpectedly");
|
||||
buffer += decoder.decode(chunk.value, { stream: true }).replaceAll("\r\n", "\n");
|
||||
}
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
const request = async (path, init = {}) => {
|
||||
const response = await fetch(`${baseUrl}${path}`, {
|
||||
...init,
|
||||
@@ -65,12 +36,11 @@ for (const required of [
|
||||
"workshop_listings",
|
||||
"workshop_report_summaries",
|
||||
"assistant_quotas",
|
||||
"assistant_requests",
|
||||
"assistant_activity",
|
||||
"assistant_global_quota",
|
||||
]) {
|
||||
if (!collectionNames.has(required)) throw new Error(`Missing Workshop collection: ${required}`);
|
||||
}
|
||||
for (const removed of ["workshop_admin_actions", "workshop_admins"]) {
|
||||
for (const removed of ["workshop_admin_actions", "workshop_admins", "assistant_requests", "assistant_activity"]) {
|
||||
if (collectionNames.has(removed)) throw new Error(`Removed Workshop collection still exists: ${removed}`);
|
||||
}
|
||||
|
||||
@@ -136,7 +106,7 @@ const modelList = await request("/api/ai/v1/models");
|
||||
if (
|
||||
modelList.object !== "list" ||
|
||||
modelList.data.length !== 1 ||
|
||||
modelList.data[0].id !== "homarr/deepseek-v4-flash-latest" ||
|
||||
modelList.data[0].id !== "homarr/model" ||
|
||||
modelList.data[0].context_length !== 256 * 1024
|
||||
) {
|
||||
throw new Error("The Homarr provider must advertise exactly one model");
|
||||
@@ -147,7 +117,7 @@ await expectStatus(
|
||||
{
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
model: "homarr/deepseek-v4-flash-latest",
|
||||
model: "homarr/model",
|
||||
messages: [{ role: "user", content: "unauthenticated" }],
|
||||
}),
|
||||
},
|
||||
@@ -172,7 +142,7 @@ await expectStatus(
|
||||
method: "POST",
|
||||
headers: authorSession.headers,
|
||||
body: JSON.stringify({
|
||||
model: "homarr/deepseek-v4-flash-latest",
|
||||
model: "homarr/model",
|
||||
messages: [
|
||||
{ role: "user", content: "too many tools" },
|
||||
...Array.from({ length: 1000 }, (_, index) => ({
|
||||
@@ -192,8 +162,8 @@ await expectStatus(
|
||||
method: "POST",
|
||||
headers: authorSession.headers,
|
||||
body: JSON.stringify({
|
||||
model: "homarr/deepseek-v4-flash-latest",
|
||||
messages: [{ role: "user", content: "x".repeat(256 * 1024) }],
|
||||
model: "homarr/model",
|
||||
messages: [{ role: "user", content: "x".repeat(4 * 256 * 1024 + 1) }],
|
||||
}),
|
||||
},
|
||||
413,
|
||||
@@ -213,31 +183,12 @@ await expectStatus(
|
||||
400,
|
||||
);
|
||||
|
||||
const realtimeResponse = await fetch(`${baseUrl}/api/realtime`);
|
||||
if (!realtimeResponse.ok || !realtimeResponse.body) throw new Error("PocketBase realtime connection failed");
|
||||
const realtime = createSseReader(realtimeResponse.body);
|
||||
const connectedEvent = await realtime.next();
|
||||
if (connectedEvent.event !== "PB_CONNECT" || !connectedEvent.id) {
|
||||
throw new Error("PocketBase realtime did not provide a client ID");
|
||||
}
|
||||
await request("/api/realtime", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ clientId: connectedEvent.id, subscriptions: ["assistant_activity/*"] }),
|
||||
});
|
||||
const activityEvent = (async () => {
|
||||
while (true) {
|
||||
const event = await realtime.next();
|
||||
const payload = JSON.parse(event.data);
|
||||
if (payload.action === "create") return payload.record;
|
||||
}
|
||||
})();
|
||||
|
||||
const streamStartedAt = performance.now();
|
||||
const streamResponse = await fetch(`${baseUrl}/api/ai/v1/chat/completions`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", ...authorSession.headers },
|
||||
body: JSON.stringify({
|
||||
model: "homarr/deepseek-v4-flash-latest",
|
||||
model: "homarr/model",
|
||||
messages: [{ role: "user", content: "hello" }],
|
||||
stream: true,
|
||||
}),
|
||||
@@ -265,20 +216,11 @@ if (
|
||||
) {
|
||||
throw new Error("The Homarr provider did not preserve the upstream SSE stream");
|
||||
}
|
||||
const liveActivity = await Promise.race([
|
||||
activityEvent,
|
||||
new Promise((_, reject) => setTimeout(() => reject(new Error("Assistant activity realtime event timed out")), 2_000)),
|
||||
]);
|
||||
await realtime.cancel();
|
||||
if (liveActivity.model !== "homarr/deepseek-v4-flash-latest" || liveActivity.requestUnits !== 1) {
|
||||
throw new Error("The public assistant activity event is invalid");
|
||||
}
|
||||
|
||||
const toolResponse = await fetch(`${baseUrl}/api/ai/v1/chat/completions`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", ...authorSession.headers },
|
||||
body: JSON.stringify({
|
||||
model: "homarr/deepseek-v4-flash-latest",
|
||||
model: "homarr/model",
|
||||
messages: [
|
||||
{ role: "user", content: "inspect my dashboard" },
|
||||
{ role: "assistant", content: null, tool_calls: [{ id: "one" }, { id: "two" }] },
|
||||
@@ -298,6 +240,12 @@ const toolResponse = await fetch(`${baseUrl}/api/ai/v1/chat/completions`, {
|
||||
extra_body: { provider: { order: ["attacker"] } },
|
||||
extra_headers: { Authorization: "Bearer attacker" },
|
||||
metadata: { user: "must-not-reach-openrouter" },
|
||||
audio: { format: "wav", voice: "alloy" },
|
||||
modalities: ["text", "audio"],
|
||||
logprobs: true,
|
||||
top_logprobs: 20,
|
||||
prediction: { content: "attacker-controlled prediction" },
|
||||
service_tier: "priority",
|
||||
reasoning: { effort: "high", max_tokens: 1_000_000 },
|
||||
user: "must-not-reach-openrouter",
|
||||
tools: [
|
||||
@@ -307,12 +255,12 @@ const toolResponse = await fetch(`${baseUrl}/api/ai/v1/chat/completions`, {
|
||||
],
|
||||
}),
|
||||
});
|
||||
if (!toolResponse.ok || toolResponse.headers.get("x-homarr-quota-remaining") !== "46") {
|
||||
throw new Error("Tool results must each consume one Homarr provider request unit");
|
||||
if (!toolResponse.ok || toolResponse.headers.get("x-homarr-quota-remaining") !== "48") {
|
||||
throw new Error("Every upstream inference must consume one Homarr provider request unit");
|
||||
}
|
||||
|
||||
const afterToolsUsage = await request("/api/ai/usage", { headers: authorSession.headers });
|
||||
if (afterToolsUsage.used !== 4 || afterToolsUsage.remaining !== 46) {
|
||||
if (afterToolsUsage.used !== 2 || afterToolsUsage.remaining !== 48) {
|
||||
throw new Error(`Tool request accounting is incorrect: ${JSON.stringify(afterToolsUsage)}`);
|
||||
}
|
||||
|
||||
@@ -320,12 +268,12 @@ const upstreamFailure = await fetch(`${baseUrl}/api/ai/v1/chat/completions`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", ...authorSession.headers },
|
||||
body: JSON.stringify({
|
||||
model: "homarr/deepseek-v4-flash-latest",
|
||||
model: "homarr/model",
|
||||
messages: [{ role: "user", content: "upstream failure" }],
|
||||
stream: false,
|
||||
}),
|
||||
});
|
||||
if (upstreamFailure.status !== 503 || upstreamFailure.headers.get("x-homarr-quota-remaining") !== "45") {
|
||||
if (upstreamFailure.status !== 503 || upstreamFailure.headers.get("x-homarr-quota-remaining") !== "47") {
|
||||
throw new Error("Every request forwarded upstream must consume Homarr provider allowance");
|
||||
}
|
||||
const upstreamFailureBody = await upstreamFailure.text();
|
||||
@@ -333,7 +281,7 @@ if (upstreamFailureBody.includes("Simulated upstream failure")) {
|
||||
throw new Error("Upstream error details must not be exposed to clients");
|
||||
}
|
||||
const afterFailureUsage = await request("/api/ai/usage", { headers: authorSession.headers });
|
||||
if (afterFailureUsage.used !== 5 || afterFailureUsage.remaining !== 45) {
|
||||
if (afterFailureUsage.used !== 3 || afterFailureUsage.remaining !== 47) {
|
||||
throw new Error(`Forwarded upstream failure escaped quota accounting: ${JSON.stringify(afterFailureUsage)}`);
|
||||
}
|
||||
const visitorUsage = await request("/api/ai/usage", { headers: visitorSession.headers });
|
||||
@@ -341,75 +289,42 @@ if (visitorUsage.used !== 0 || visitorUsage.remaining !== 50) {
|
||||
throw new Error(`Provider allowance leaked across users: ${JSON.stringify(visitorUsage)}`);
|
||||
}
|
||||
|
||||
const publicActivities = await request("/api/collections/assistant_activity/records?sort=-created&perPage=10");
|
||||
if (
|
||||
publicActivities.items.length < 2 ||
|
||||
publicActivities.items.some((item) => "user" in item) ||
|
||||
!publicActivities.items.some(
|
||||
(item) => item.status === "completed" && item.inputTokens === 12 && item.outputTokens === 7,
|
||||
)
|
||||
) {
|
||||
throw new Error("Public assistant activity must be anonymous and include completed token totals");
|
||||
}
|
||||
await expectStatus(
|
||||
"/api/collections/assistant_activity/records",
|
||||
{
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
status: "completed",
|
||||
model: "forged",
|
||||
requestUnits: 1,
|
||||
}),
|
||||
},
|
||||
403,
|
||||
);
|
||||
await expectStatus("/api/collections/assistant_requests/records", { headers: authorSession.headers }, 403);
|
||||
await expectStatus("/api/collections/assistant_quotas/records", { headers: authorSession.headers }, 403);
|
||||
|
||||
const privateRequests = await request("/api/collections/assistant_requests/records?perPage=20", {
|
||||
headers: rootHeaders,
|
||||
});
|
||||
if (
|
||||
privateRequests.items.length < 2 ||
|
||||
privateRequests.items.some((item) => item.user !== author.id) ||
|
||||
privateRequests.items.some((item) => "messages" in item || "prompt" in item || "body" in item)
|
||||
) {
|
||||
throw new Error("Private provider accounting must identify the user without storing request content");
|
||||
}
|
||||
const requestToRetain = privateRequests.items.find((item) => item.publicActivity);
|
||||
if (!requestToRetain) throw new Error("Expected a private request linked to public activity");
|
||||
await request(`/api/collections/assistant_activity/records/${requestToRetain.publicActivity}`, {
|
||||
method: "DELETE",
|
||||
headers: rootHeaders,
|
||||
});
|
||||
const retainedRequest = await request(`/api/collections/assistant_requests/records/${requestToRetain.id}`, {
|
||||
headers: rootHeaders,
|
||||
});
|
||||
if (retainedRequest.id !== requestToRetain.id || retainedRequest.publicActivity) {
|
||||
throw new Error("Pruning public activity must retain private accounting records");
|
||||
}
|
||||
await expectStatus("/api/collections/assistant_global_quota/records", { headers: authorSession.headers }, 403);
|
||||
const quotas = await request("/api/collections/assistant_quotas/records?perPage=20", { headers: rootHeaders });
|
||||
const authorQuota = quotas.items.find((item) => item.user === author.id);
|
||||
if (!authorQuota || authorQuota.dailyLimit !== 50 || authorQuota.used !== 5 || authorQuota.totalTokens !== 38) {
|
||||
if (
|
||||
!authorQuota ||
|
||||
authorQuota.used !== 3 ||
|
||||
["dailyLimit", "inputTokens", "outputTokens", "totalTokens", "model", "cost"].some((field) => field in authorQuota)
|
||||
) {
|
||||
throw new Error(`Private provider quota is incorrect: ${JSON.stringify(authorQuota)}`);
|
||||
}
|
||||
|
||||
await request(`/api/collections/assistant_quotas/records/${authorQuota.id}`, {
|
||||
method: "PATCH",
|
||||
headers: rootHeaders,
|
||||
body: JSON.stringify({ day: "2000-01-01", used: 49, dailyLimit: 2 }),
|
||||
body: JSON.stringify({ day: "2000-01-01", used: 49 }),
|
||||
});
|
||||
const resetUsage = await request("/api/ai/usage", { headers: authorSession.headers });
|
||||
if (resetUsage.limit !== 2 || resetUsage.used !== 0 || resetUsage.remaining !== 2) {
|
||||
if (resetUsage.limit !== 50 || resetUsage.used !== 0 || resetUsage.remaining !== 50) {
|
||||
throw new Error(`Quota did not reset at the UTC day boundary: ${JSON.stringify(resetUsage)}`);
|
||||
}
|
||||
const resetQuota = (
|
||||
await request("/api/collections/assistant_quotas/records?perPage=20", { headers: rootHeaders })
|
||||
).items.find((item) => item.user === author.id);
|
||||
await request(`/api/collections/assistant_quotas/records/${resetQuota.id}`, {
|
||||
method: "PATCH",
|
||||
headers: rootHeaders,
|
||||
body: JSON.stringify({ used: 48 }),
|
||||
});
|
||||
|
||||
const allowanceProbe = () =>
|
||||
fetch(`${baseUrl}/api/ai/v1/chat/completions`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", ...authorSession.headers },
|
||||
body: JSON.stringify({
|
||||
model: "homarr/deepseek-v4-flash-latest",
|
||||
model: "homarr/model",
|
||||
messages: [{ role: "user", content: "allowance probe" }],
|
||||
stream: false,
|
||||
}),
|
||||
@@ -425,6 +340,35 @@ if (
|
||||
throw new Error("The per-user allowance was not enforced atomically");
|
||||
}
|
||||
|
||||
const globalQuota = (await request("/api/collections/assistant_global_quota/records", { headers: rootHeaders }))
|
||||
.items[0];
|
||||
await request(`/api/collections/assistant_global_quota/records/${globalQuota.id}`, {
|
||||
method: "PATCH",
|
||||
headers: rootHeaders,
|
||||
body: JSON.stringify({ day: new Date().toISOString().slice(0, 10), used: 9_999 }),
|
||||
});
|
||||
await request(`/api/collections/assistant_quotas/records/${resetQuota.id}`, {
|
||||
method: "PATCH",
|
||||
headers: rootHeaders,
|
||||
body: JSON.stringify({ used: 0 }),
|
||||
});
|
||||
const globalResponses = await Promise.all([
|
||||
allowanceProbe(),
|
||||
fetch(`${baseUrl}/api/ai/v1/chat/completions`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", ...visitorSession.headers },
|
||||
body: JSON.stringify({ model: "homarr/model", messages: [{ role: "user", content: "global allowance probe" }] }),
|
||||
}),
|
||||
]);
|
||||
if (
|
||||
globalResponses
|
||||
.map((response) => response.status)
|
||||
.toSorted()
|
||||
.join(",") !== "200,429"
|
||||
) {
|
||||
throw new Error("The shared upstream allowance was not enforced atomically across users");
|
||||
}
|
||||
|
||||
const widget = {
|
||||
$schema: "homarr-custom-widget-v2",
|
||||
name: "Workshop runtime probe",
|
||||
|
||||
Reference in new issue
Block a user