feat!: release v2 (#6545)

Co-authored-by: Artur <145776561+arturict@users.noreply.github.com>
Co-authored-by: arturict <arturict@users.noreply.github.com>
Co-authored-by: Thomas <thomas@ajnart.fr>
Co-authored-by: Afonso Jorge Ramos <afonsojorgeramos@gmail.com>
Co-authored-by: Danura30082 <danura30082+github@gmail.com>
Co-authored-by: Kushie <122986913+kushiemoon-dev@users.noreply.github.com>
Co-authored-by: ajnart <thomas@ajnart.fr>
Co-authored-by: ajnart <thomas@ajnart.dev>
Co-authored-by: chenjun <chenjun@foxmail.com>
Co-authored-by: Wikibear <wikibear@users.noreply.github.com>
Co-authored-by: Bergasha <bergasha@gmail.com>
Co-authored-by: leafyelin <43306196+leafyelin@users.noreply.github.com>
Co-authored-by: Lucas Araujo <lucas.lra@gmail.com>
Co-authored-by: GreenMonito <86085570+GreenMonito@users.noreply.github.com>
Co-authored-by: Nerdy-Technician <roffo@nerdy-technician.social>
Co-authored-by: Jun <34545896+jcstrong@users.noreply.github.com>
Co-authored-by: Edmond Abraham <54119952+gitedmond@users.noreply.github.com>
Co-authored-by: thrawnx <tight.bulb9384@fastmail.com>
Co-authored-by: Chris <58918656+nordada@users.noreply.github.com>
Co-authored-by: yi111 <153097222+Yi-111-a@users.noreply.github.com>
Co-authored-by: Yanis <naamaneyanis@gmail.com>
Co-authored-by: Wikibear <me@sej.de>
Co-authored-by: alessiottavo <93441886+alessiottavo@users.noreply.github.com>
This commit is contained in:
3286 files changed
+284386 -139176

No files matched your search

@@ -0,0 +1,176 @@
import assert from "node:assert/strict";
import { execFile } from "node:child_process";
import { mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { DatabaseSync } from "node:sqlite";
import { promisify } from "node:util";
import { test } from "node:test";
import { setTimeout } from "node:timers/promises";
import { convert } from "../src/convert.mjs";
import { migrations, quote, schema } from "../src/legacy.mjs";
import { createFixture, fixtureCiphertext, fixtureDate, fixtureMedia, fixturePasswordHash } from "./fixture.mjs";
const exec = promisify(execFile);
test("real v1.77.1 MySQL converts losslessly and rejects unsafe sources", { timeout: 300_000 }, async (t) => {
const fixture = await createFixture();
const directory = await mkdtemp(join(tmpdir(), "homarr-conversion-"));
t.after(async () => {
try {
await fixture.cleanup();
} finally {
await rm(directory, { recursive: true, force: true });
}
});
const { connection, connectionOptions } = fixture;
const output = join(directory, "db.sqlite");
const sourceSnapshot = async () =>
Object.fromEntries(
await Promise.all(
schema.tables.map(async ({ name }) => {
const [rows] = await connection.query(`SELECT * FROM ${quote(name)}`);
return [name, rows];
}),
),
);
const before = await sourceSnapshot();
await assert.rejects(convert({ connectionOptions, output }), /Stop every Homarr/);
const result = await convert({ connectionOptions, output, homarrStopped: true });
assert.equal(Object.keys(result.counts).length, 35);
assert.equal(result.counts.serverSetting, 252);
assert.deepEqual(await sourceSnapshot(), before);
const database = new DatabaseSync(output, { readOnly: true });
try {
assert.deepEqual(database.prepare("PRAGMA foreign_key_check").all(), []);
assert.equal(database.prepare("PRAGMA integrity_check").get().integrity_check, "ok");
assert.equal(database.prepare("SELECT password FROM user").get().password, fixturePasswordHash);
assert.equal(database.prepare("SELECT value FROM integrationSecret").get().value, fixtureCiphertext);
assert.deepEqual(Buffer.from(database.prepare("SELECT content FROM media").get().content), fixtureMedia);
assert.equal(
database.prepare("SELECT expires FROM session").get().expires,
Date.parse(fixtureDate.replace(" ", "T") + "Z"),
);
assert.equal(
database.prepare("SELECT updated_at FROM integrationSecret").get().updated_at,
Date.parse(fixtureDate.replace(" ", "T") + "Z") / 1000,
);
assert.equal(database.prepare("SELECT image FROM user").get().image, null);
const [sourceBoards] = await connection.query("SELECT custom_css, is_public FROM board");
const convertedBoard = database.prepare("SELECT custom_css, is_public FROM board").get();
assert.equal(convertedBoard.custom_css, sourceBoards[0].custom_css);
assert.ok(convertedBoard.custom_css.length > 50_000);
assert.ok(convertedBoard.custom_css.includes("café 🏡"));
assert.equal(convertedBoard.is_public, 0);
assert.equal(database.prepare("SELECT completed_manage_tour FROM user").get().completed_manage_tour, 1);
assert.equal(database.prepare("SELECT is_enabled FROM cron_job_configuration").get().is_enabled, 0);
assert.equal(database.prepare("SELECT parent_section_id FROM section_layout").get().parent_section_id, null);
assert.equal(
database.prepare("SELECT COUNT(*) AS n FROM __drizzle_migrations").get().n,
migrations("sqlite").length,
);
} finally {
database.close();
}
const original = await readFile(output);
await assert.rejects(convert({ connectionOptions, output, homarrStopped: true }), /Destination already exists/);
assert.deepEqual(await readFile(output), original);
await connection.query("ALTER TABLE user ADD COLUMN unsupported_schema TEXT");
await assert.rejects(
convert({ connectionOptions, output: join(directory, "unsupported.sqlite"), homarrStopped: true }),
/Unsupported or partially migrated/,
);
await connection.query("ALTER TABLE user DROP COLUMN unsupported_schema");
await connection.query("UPDATE __drizzle_migrations SET hash = 'unknown' WHERE id = 1");
await assert.rejects(
convert({ connectionOptions, output: join(directory, "journal.sqlite"), homarrStopped: true }),
/Unsupported migration journal/,
);
await connection.query("UPDATE __drizzle_migrations SET hash = ? WHERE id = 1", [migrations("mysql")[0].hash]);
await connection.query("SET FOREIGN_KEY_CHECKS = 0");
await connection.query("UPDATE groupMember SET user_id = 'missing-user'");
await assert.rejects(
convert({ connectionOptions, output: join(directory, "invalid.sqlite"), homarrStopped: true }),
/broken relationships/,
);
await connection.query("UPDATE groupMember SET user_id = 'migration-user'");
await connection.query("SET FOREIGN_KEY_CHECKS = 1");
assert.deepEqual(await readdir(directory), ["db.sqlite"]);
assert.deepEqual(await sourceSnapshot(), before);
const controller = new AbortController();
const interrupted = convert({
connectionOptions,
output: join(directory, "interrupted.sqlite"),
homarrStopped: true,
signal: controller.signal,
});
// Observe a started copy rather than relying on a sleep duration to interrupt it.
const rejection = assert.rejects(interrupted, /aborted/);
const deadline = Date.now() + 10_000;
while (!(await readdir(directory)).some((name) => name.startsWith("interrupted.sqlite.partial-"))) {
if (Date.now() > deadline) throw new Error("Conversion did not start in time");
await setTimeout(1);
}
controller.abort();
await rejection;
assert.deepEqual(await readdir(directory), ["db.sqlite"]);
assert.deepEqual(await sourceSnapshot(), before);
// Create a competing destination after conversion passes its initial existence check.
const racedOutput = join(directory, "raced.sqlite");
const racedConversion = convert({ connectionOptions, output: racedOutput, homarrStopped: true });
const racedRejection = assert.rejects(racedConversion, { code: "EEXIST" });
const raceDeadline = Date.now() + 10_000;
while (!(await readdir(directory)).some((name) => name.startsWith("raced.sqlite.partial-"))) {
if (Date.now() > raceDeadline) throw new Error("Conversion did not start before the destination race");
await setTimeout(1);
}
const competingContent = Buffer.from("An independently created file must never be overwritten.");
await writeFile(racedOutput, competingContent, { flag: "wx", mode: 0o600 });
await racedRejection;
assert.deepEqual(await readFile(racedOutput), competingContent);
assert.deepEqual((await readdir(directory)).sort(), ["db.sqlite", "raced.sqlite"]);
await rm(racedOutput);
assert.deepEqual(await sourceSnapshot(), before);
// Exercise the delivered Docker entrypoint against the same real database.
const image = `homarr-mysql-converter-test:${process.pid}`;
await exec("docker", ["build", "-t", image, "."], {
cwd: new URL("../", import.meta.url),
maxBuffer: 8 * 1024 * 1024,
});
t.after(() => exec("docker", ["image", "rm", image]));
const environmentFile = join(directory, "connection.env");
await writeFile(
environmentFile,
`MYSQL_HOST=127.0.0.1\nMYSQL_PORT=${connectionOptions.port}\nMYSQL_USER=${connectionOptions.user}\nMYSQL_PASSWORD=${connectionOptions.password}\nMYSQL_DATABASE=homarr\n`,
{ mode: 0o600 },
);
await exec("docker", [
"run",
"--rm",
"--user",
`${process.getuid()}:${process.getgid()}`,
"--network",
"host",
"--env-file",
environmentFile,
"-v",
`${directory}:/output`,
image,
"--output",
"/output/packaged.sqlite",
"--homarr-stopped",
]);
const packaged = new DatabaseSync(join(directory, "packaged.sqlite"), { readOnly: true });
try {
assert.equal(packaged.prepare("SELECT password FROM user").get().password, fixturePasswordHash);
} finally {
packaged.close();
}
assert.deepEqual(await sourceSnapshot(), before);
});
+16
View File
@@ -0,0 +1,16 @@
import type { Connection, ConnectionOptions } from "mysql2/promise";
export const fixtureUser: string;
export const fixturePassword: string;
export const fixtureBoard: string;
export const fixtureSecret: string;
export const fixtureMedia: Buffer;
export const fixtureDate: string;
export const fixturePasswordHash: string;
export const fixtureCiphertext: string;
export function createFixture(): Promise<{
connectionOptions: ConnectionOptions;
connection: Connection;
container: string;
cleanup(): Promise<void>;
}>;
+266
View File
@@ -0,0 +1,266 @@
import { execFile } from "node:child_process";
import { createCipheriv, randomBytes } from "node:crypto";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { promisify } from "node:util";
import { setTimeout } from "node:timers/promises";
import mysql from "mysql2/promise";
import { quote, schema } from "../src/legacy.mjs";
const exec = promisify(execFile);
// Published v1.77.1 multi-platform image. Its bundled migrator, not our frozen
// SQL helper, must produce the source schema and journal used by these tests.
const sourceImage =
"ghcr.io/homarr-labs/homarr:v1.77.1@sha256:1f5b892aeef4ad0a4907f075777bbd0ce7120f4cdc53b9f3f028519250421aed";
export const fixtureUser = "migration-user";
export const fixturePassword = "migration-password";
export const fixtureBoard = "migration-board";
export const fixtureSecret = "migration-secret";
export const fixtureMedia = Buffer.from([0, 1, 2, 127, 128, 255]);
export const fixtureDate = "2025-03-14 15:09:26";
export const fixturePasswordHash = "$2b$10$RPaXzPcun3JX2n8.A138S.m.AVTueysEPV77vJf/pKGI5DdCzVW0y";
const iv = Buffer.alloc(16, 1);
const cipher = createCipheriv("aes-256-cbc", Buffer.alloc(32), iv);
export const fixtureCiphertext = `${Buffer.concat([cipher.update(fixtureSecret), cipher.final()]).toString("hex")}.${iv.toString("hex")}`;
const ids = {
user: fixtureUser,
board: fixtureBoard,
group: "migration-group",
app: "migration-app",
custom_widget_definition: "migration-custom",
iconRepository: "migration-repository",
icon: "migration-icon",
integration: "migration-integration",
item: "migration-item",
layout: "migration-layout",
media: "migration-media",
section: "migration-section",
search_engine: "migration-search",
apiKey: "migration-api-key",
invite: "migration-invite",
};
const references = {
user_id: ids.user,
creator_id: ids.user,
owner_id: ids.user,
board_id: ids.board,
home_board_id: ids.board,
group_id: ids.group,
app_id: ids.app,
definition_id: ids.custom_widget_definition,
icon_repository_id: ids.iconRepository,
integration_id: ids.integration,
item_id: ids.item,
layout_id: ids.layout,
section_id: ids.section,
};
const overrides = {
user: {
name: fixtureUser,
email: "migration@example.test",
email_verified: fixtureDate,
password: fixturePasswordHash,
provider: "credentials",
home_board_id: ids.board,
color_scheme: "dark",
first_day_of_week: 1,
completed_manage_tour: 1,
completed_board_tour: 1,
},
group: { name: "credentials-admin", position: 0 },
groupPermission: { permission: "admin" },
boardGroupPermission: { permission: "full" },
boardUserPermission: { permission: "full" },
integrationGroupPermissions: { permission: "full" },
integrationUserPermission: { permission: "full" },
board: {
name: "converted-board",
page_title: "Converted board",
primary_color: "blue",
secondary_color: "violet",
background_image_attachment: "fixed",
background_image_repeat: "no-repeat",
background_image_size: "cover",
item_radius: "md",
opacity: 100,
is_public: 0,
custom_css: "/* Unicode: café 🏡 */\n" + "x".repeat(60_000),
},
onboarding: { id: "onboarding", step: "finish", previous_step: "setup" },
integration: { name: "Converted Pi-hole", kind: "piHole", url: "http://127.0.0.1:9" },
integrationSecret: { kind: "apiKey", value: fixtureCiphertext },
custom_widget_definition: {
name: "Legacy weather",
url: "https://example.test/weather",
enabled: 0,
auth_type: "bearer",
method: "GET",
display_type: "singleValue",
display_config: '{"type":"singleValue","jsonPath":"$.temperature"}',
},
custom_widget_secret: { kind: "apiKey", value: fixtureCiphertext },
widget_secret: { kind: "apiKey", value: fixtureCiphertext },
app: { name: "Converted app", icon_url: "/favicon.ico", href: "https://example.test" },
iconRepository: { slug: "migration/repository" },
icon: { name: "fixture", url: "https://example.test/icon.png", checksum: "fixture" },
item: { kind: "clock", options: '{"json":{}}', advanced_options: '{"json":{}}' },
layout: { name: "Base", column_count: 12, breakpoint: 768 },
section: { kind: "empty", x_offset: 0, y_offset: 0, options: '{"json":{}}' },
section_layout: { parent_section_id: null, x_offset: 0, y_offset: 0, width: 12, height: 3 },
item_layout: { x_offset: 0, y_offset: 0, width: 3, height: 2 },
media: {
name: "Binary fixture",
content: fixtureMedia,
content_type: "application/octet-stream",
size: fixtureMedia.length,
},
search_engine: {
name: "Fixture search",
short: "fx",
type: "generic",
icon_url: "/favicon.ico",
url_template: "https://example.test/?q={query}",
},
cron_job_configuration: { name: "ping", cron_expression: "* * * * *", is_enabled: 0 },
serverSetting: { setting_key: "migration-fixture", value: '{"json":{"unicode":"café 🏡"}}' },
account: { type: "oauth", provider: "oidc", provider_account_id: "migration-external-user" },
apiKey: { api_key: fixturePasswordHash },
trusted_certificate_hostname: { hostname: "example.test", thumbprint: "fixture", certificate: "fixture certificate" },
};
export async function createFixture() {
const password = randomBytes(16).toString("hex");
const { stdout } = await exec("docker", [
"run",
"-d",
"--rm",
"-p",
"127.0.0.1::3306",
"-e",
`MYSQL_ROOT_PASSWORD=${password}`,
"-e",
"MYSQL_DATABASE=homarr",
"mysql:8.4.6",
]);
const container = stdout.trim();
let connection;
const cleanup = async () => {
try {
await connection?.end();
} finally {
await exec("docker", ["rm", "-f", container]);
}
};
try {
const { stdout: published } = await exec("docker", ["port", container, "3306/tcp"]);
const port = Number(published.trim().split(":").at(-1));
const connectionOptions = {
host: "127.0.0.1",
port,
user: "root",
password,
database: "homarr",
timezone: "Z",
dateStrings: true,
};
const deadline = Date.now() + 120_000;
while (!connection) {
try {
connection = await mysql.createConnection(connectionOptions);
} catch (error) {
if (Date.now() >= deadline) throw error;
await setTimeout(250);
}
}
await connection.query("SET SESSION time_zone = '+00:00'");
const environmentDirectory = await mkdtemp(join(tmpdir(), "homarr-release-source-"));
try {
const environmentFile = join(environmentDirectory, "source.env");
await writeFile(
environmentFile,
[
"DB_DRIVER=mysql2",
"DB_DIALECT=mysql",
"DB_HOST=127.0.0.1",
`DB_PORT=${port}`,
"DB_USER=root",
`DB_PASSWORD=${password}`,
"DB_NAME=homarr",
"SKIP_ENV_VALIDATION=true",
"NO_EXTERNAL_CONNECTION=true",
`SECRET_ENCRYPTION_KEY=${"0".repeat(64)}`,
].join("\n") + "\n",
{ mode: 0o600 },
);
await exec(
"docker",
[
"run",
"--rm",
"--network",
"host",
"--env-file",
environmentFile,
"--entrypoint",
"node",
sourceImage,
"/app/db/migrations/mysql/migrate.cjs",
"/app/db/migrations/mysql",
],
{ timeout: 120_000, maxBuffer: 8 * 1024 * 1024 },
);
} finally {
await rm(environmentDirectory, { recursive: true, force: true });
}
await connection.query("SET FOREIGN_KEY_CHECKS = 0");
// Replace release seed rows with deterministic user data, leaving the real
// release's schema and migration bookkeeping intact.
const [tables] = await connection.query("SHOW TABLES");
for (const table of tables) {
const name = Object.values(table)[0];
if (name !== "__drizzle_migrations") await connection.query(`DELETE FROM ${quote(name)}`);
}
for (const table of schema.tables) {
const values = Object.fromEntries(
table.columns.map((column) => {
let value = !column.notNull
? null
: column.encoding === "boolean"
? 0
: column.encoding === "integer"
? 1
: ["seconds", "milliseconds"].includes(column.encoding)
? fixtureDate
: "fixture";
if (column.name === "id") value = ids[table.name] ?? `migration-${table.name}`;
if (references[column.name] && column.notNull) value = references[column.name];
return [column.name, value];
}),
);
Object.assign(values, overrides[table.name]);
await connection.query(
`INSERT INTO ${quote(table.name)} (${Object.keys(values).map(quote).join(", ")}) VALUES (${Object.keys(values)
.map(() => "?")
.join(", ")})`,
Object.values(values),
);
}
// More than one copy batch, including duplicate non-key values and Unicode.
for (let i = 0; i < 251; i++)
await connection.query("INSERT INTO serverSetting (setting_key, value) VALUES (?, ?)", [
`migration-${i}`,
"café 🏡",
]);
await connection.query("SET FOREIGN_KEY_CHECKS = 1");
await connection.query("CREATE USER 'migration_reader'@'%' IDENTIFIED BY ?", [password]);
await connection.query("GRANT SELECT ON homarr.* TO 'migration_reader'@'%'");
return { connectionOptions: { ...connectionOptions, user: "migration_reader" }, connection, container, cleanup };
} catch (error) {
await cleanup();
throw error;
}
}