feat!: release v2 (#6545)

Co-authored-by: Artur <145776561+arturict@users.noreply.github.com>
Co-authored-by: arturict <arturict@users.noreply.github.com>
Co-authored-by: Thomas <thomas@ajnart.fr>
Co-authored-by: Afonso Jorge Ramos <afonsojorgeramos@gmail.com>
Co-authored-by: Danura30082 <danura30082+github@gmail.com>
Co-authored-by: Kushie <122986913+kushiemoon-dev@users.noreply.github.com>
Co-authored-by: ajnart <thomas@ajnart.fr>
Co-authored-by: ajnart <thomas@ajnart.dev>
Co-authored-by: chenjun <chenjun@foxmail.com>
Co-authored-by: Wikibear <wikibear@users.noreply.github.com>
Co-authored-by: Bergasha <bergasha@gmail.com>
Co-authored-by: leafyelin <43306196+leafyelin@users.noreply.github.com>
Co-authored-by: Lucas Araujo <lucas.lra@gmail.com>
Co-authored-by: GreenMonito <86085570+GreenMonito@users.noreply.github.com>
Co-authored-by: Nerdy-Technician <roffo@nerdy-technician.social>
Co-authored-by: Jun <34545896+jcstrong@users.noreply.github.com>
Co-authored-by: Edmond Abraham <54119952+gitedmond@users.noreply.github.com>
Co-authored-by: thrawnx <tight.bulb9384@fastmail.com>
Co-authored-by: Chris <58918656+nordada@users.noreply.github.com>
Co-authored-by: yi111 <153097222+Yi-111-a@users.noreply.github.com>
Co-authored-by: Yanis <naamaneyanis@gmail.com>
Co-authored-by: Wikibear <me@sej.de>
Co-authored-by: alessiottavo <93441886+alessiottavo@users.noreply.github.com>

tools/mysql-to-sqlite/src/cli.mjs

@@ -0,0 +1,57 @@
+import { readFileSync } from "node:fs";
+import { parseArgs } from "node:util";
+
+import { convert } from "./convert.mjs";
+
+const controller = new AbortController();
+const abort = () => controller.abort(new Error("Conversion interrupted; no output was published."));
+process.once("SIGTERM", abort);
+process.once("SIGINT", abort);
+
+try {
+  const { values } = parseArgs({
+    options: { output: { type: "string" }, "homarr-stopped": { type: "boolean" }, help: { type: "boolean" } },
+  });
+  if (values.help) {
+    console.log(
+      "Usage: node src/cli.mjs --output /output/db.sqlite --homarr-stopped\nSet MYSQL_HOST, MYSQL_USER, MYSQL_PASSWORD, MYSQL_DATABASE; optional MYSQL_PORT (3306), MYSQL_SSL_CA (PEM file). Supports Homarr v1.77.1 only.",
+    );
+  } else {
+    for (const key of ["MYSQL_HOST", "MYSQL_USER", "MYSQL_PASSWORD", "MYSQL_DATABASE"]) {
+      if (!process.env[key]) throw new Error(`Missing ${key}`);
+    }
+    const port = Number(process.env.MYSQL_PORT ?? 3306);
+    if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error("Invalid MYSQL_PORT");
+    const result = await convert({
+      connectionOptions: {
+        host: process.env.MYSQL_HOST,
+        port,
+        user: process.env.MYSQL_USER,
+        password: process.env.MYSQL_PASSWORD,
+        database: process.env.MYSQL_DATABASE,
+        ...(process.env.MYSQL_SSL_CA ? { ssl: { ca: readFileSync(process.env.MYSQL_SSL_CA) } } : {}),
+      },
+      output: values.output,
+      homarrStopped: values["homarr-stopped"],
+      signal: controller.signal,
+    });
+    console.log(
+      `Verified ${Object.keys(result.counts).length} tables (${Object.values(result.counts).reduce((a, b) => a + b, 0)} rows). Created ${result.output}.`,
+    );
+    console.log(
+      "Start v2 with this SQLite file, the original appdata, and the same SECRET_ENCRYPTION_KEY. Keep MySQL unchanged for rollback.",
+    );
+  }
+} catch (error) {
+  // Driver errors may embed query values. Only print our validation messages or
+  // the driver's error code, never its SQL, connection object, or credentials.
+  console.error(
+    error.code
+      ? `Conversion failed (${error.code}). Source unchanged; check connectivity and the supported source schema.`
+      : error.message,
+  );
+  process.exitCode = 1;
+} finally {
+  process.removeListener("SIGTERM", abort);
+  process.removeListener("SIGINT", abort);
+}

tools/mysql-to-sqlite/src/convert.d.mts

@@ -0,0 +1,8 @@
+import type { ConnectionOptions } from "mysql2/promise";
+
+export function convert(options: {
+  connectionOptions: ConnectionOptions;
+  output: string;
+  homarrStopped: boolean;
+  signal?: AbortSignal;
+}): Promise<{ release: string; output: string; counts: Record<string, number> }>;

tools/mysql-to-sqlite/src/convert.mjs

@@ -0,0 +1,210 @@
+import { randomUUID } from "node:crypto";
+import { closeSync, constants, fsyncSync, openSync } from "node:fs";
+import { link, lstat, unlink } from "node:fs/promises";
+import { resolve } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+import { isDeepStrictEqual } from "node:util";
+import mysql from "mysql2/promise";
+
+import { initializeSqlite, migrations, quote, schema } from "./legacy.mjs";
+
+function normalizeType(type) {
+  return type
+    .toLowerCase()
+    .replace(/^boolean$/, "tinyint")
+    .replace(/^(tinyint|smallint|int)\(\d+\)$/, "$1");
+}
+
+async function validateSource(connection) {
+  const [tables] = await connection.query(
+    "SELECT TABLE_NAME AS name, ENGINE AS engine FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE()",
+  );
+  const expected = [...schema.tables.map(({ name }) => name), "__drizzle_migrations"].toSorted();
+  if (
+    !isDeepStrictEqual(tables.map(({ name }) => name).toSorted(), expected) ||
+    tables.some(({ engine }) => engine !== "InnoDB")
+  ) {
+    throw new Error("Unsupported source schema: expected the complete v1.77.1 InnoDB database, without extra tables.");
+  }
+  const [columns] = await connection.query(
+    "SELECT TABLE_NAME AS tableName, COLUMN_NAME AS name, COLUMN_TYPE AS type, IS_NULLABLE AS nullable FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE()",
+  );
+  for (const table of schema.tables) {
+    const actual = columns.filter(({ tableName }) => tableName === table.name);
+    if (
+      actual.length !== table.columns.length ||
+      table.columns.some((column) => {
+        const found = actual.find(({ name }) => name === column.name);
+        return (
+          !found ||
+          normalizeType(found.type) !== normalizeType(column.mysqlType) ||
+          (found.nullable === "NO") !== column.notNull
+        );
+      })
+    ) {
+      throw new Error(`Unsupported or partially migrated source schema: ${table.name}. Expected v1.77.1.`);
+    }
+  }
+  const [journal] = await connection.query("SELECT hash, created_at FROM __drizzle_migrations ORDER BY created_at");
+  const expectedJournal = migrations("mysql");
+  if (
+    journal.length !== expectedJournal.length ||
+    journal.some(
+      (row, index) =>
+        row.hash !== expectedJournal[index].hash || Number(row.created_at) !== expectedJournal[index].when,
+    )
+  ) {
+    throw new Error("Unsupported migration journal. Upgrade Homarr to v1.77.1 and complete all migrations first.");
+  }
+}
+
+function encode(value, column) {
+  if (value === null) return null;
+  switch (column.encoding) {
+    case "milliseconds":
+    case "seconds": {
+      if (typeof value !== "string" || !/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/.test(value)) {
+        throw new Error(`Invalid timestamp in ${column.name}`);
+      }
+      const milliseconds = Date.parse(`${value.replace(" ", "T")}Z`);
+      if (
+        !Number.isFinite(milliseconds) ||
+        new Date(milliseconds).toISOString().slice(0, 19).replace("T", " ") !== value
+      ) {
+        throw new Error(`Invalid timestamp in ${column.name}`);
+      }
+      return column.encoding === "milliseconds" ? milliseconds : milliseconds / 1000;
+    }
+    case "boolean":
+      if (value !== 0 && value !== 1) throw new Error(`Invalid boolean in ${column.name}`);
+      return value;
+    case "integer":
+      if (!Number.isSafeInteger(value)) throw new Error(`Invalid integer in ${column.name}`);
+      return value;
+    case "blob":
+      if (!Buffer.isBuffer(value)) throw new Error(`Invalid binary value in ${column.name}`);
+      return value;
+    case "text":
+      if (typeof value !== "string") throw new Error(`Invalid text in ${column.name}`);
+      return value;
+    default:
+      throw new Error(`Unknown encoding in ${column.name}`);
+  }
+}
+
+export async function convert({ connectionOptions, output, homarrStopped, signal }) {
+  signal?.throwIfAborted();
+  if (!homarrStopped) throw new Error("Stop every Homarr process using this database, then pass --homarr-stopped.");
+  if (!output) throw new Error("An --output file is required.");
+  const destination = resolve(output);
+  try {
+    await lstat(destination);
+    throw new Error("Destination already exists; choose a new output file.");
+  } catch (error) {
+    if (error.code !== "ENOENT") throw error;
+  }
+  const temporary = `${destination}.partial-${randomUUID()}`;
+  let connection;
+  let database;
+  let created = false;
+  let published = false;
+  try {
+    connection = await mysql.createConnection({
+      ...connectionOptions,
+      timezone: "Z",
+      dateStrings: true,
+      supportBigNumbers: true,
+      bigNumberStrings: true,
+      multipleStatements: false,
+      charset: "utf8mb4",
+    });
+    await connection.query("SET SESSION time_zone = '+00:00'");
+    await connection.query("SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ");
+    await connection.query("START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY");
+    await validateSource(connection);
+    signal?.throwIfAborted();
+
+    closeSync(openSync(temporary, "wx", 0o600));
+    created = true;
+    database = new DatabaseSync(temporary);
+    database.exec("PRAGMA foreign_keys = OFF");
+    initializeSqlite(database);
+    // Historical migrations contain their own FK pragmas. Disable only for the
+    // complete copy, then check every relationship before publishing the file.
+    database.exec("PRAGMA foreign_keys = OFF; BEGIN IMMEDIATE");
+    const counts = {};
+    for (const table of schema.tables) {
+      const names = table.columns.map(({ name }) => quote(name)).join(", ");
+      const insert = database.prepare(
+        `INSERT INTO ${quote(table.name)} (${names}) VALUES (${table.columns.map(() => "?").join(", ")})`,
+      );
+      const read = database.prepare(`SELECT ${names} FROM ${quote(table.name)} WHERE rowid = ?`);
+      let count = 0;
+      while (true) {
+        signal?.throwIfAborted();
+        const [rows] = await connection.query(
+          `SELECT ${names} FROM ${quote(table.name)} ORDER BY ${table.orderBy.map(quote).join(", ")} LIMIT 250 OFFSET ?`,
+          [count],
+        );
+        for (const row of rows) {
+          const values = table.columns.map((column) => encode(row[column.name], column));
+          const inserted = insert.run(...values);
+          const stored = read.get(inserted.lastInsertRowid);
+          for (const [index, column] of table.columns.entries()) {
+            const value = stored[column.name];
+            const normalized = value instanceof Uint8Array ? Buffer.from(value) : value;
+            if (!isDeepStrictEqual(normalized, values[index]))
+              throw new Error(`Value verification failed in ${table.name}.${column.name}`);
+          }
+        }
+        count += rows.length;
+        if (rows.length < 250) break;
+      }
+      const [[sourceCount]] = await connection.query(`SELECT COUNT(*) AS count FROM ${quote(table.name)}`);
+      const targetCount = database.prepare(`SELECT COUNT(*) AS count FROM ${quote(table.name)}`).get().count;
+      if (Number(sourceCount.count) !== count || targetCount !== count)
+        throw new Error(`Row count mismatch in ${table.name}`);
+      counts[table.name] = count;
+    }
+    if (database.prepare("PRAGMA foreign_key_check").all().length)
+      throw new Error("Source contains broken relationships; output was not published.");
+    const integrity = database.prepare("PRAGMA integrity_check").all();
+    if (integrity.length !== 1 || integrity[0].integrity_check !== "ok")
+      throw new Error("SQLite integrity check failed.");
+    database.exec("COMMIT; PRAGMA foreign_keys = ON");
+    database.close();
+    database = undefined;
+    await connection.rollback();
+    await connection.end();
+    connection = undefined;
+    const descriptor = openSync(temporary, constants.O_RDONLY);
+    try {
+      fsyncSync(descriptor);
+    } finally {
+      closeSync(descriptor);
+    }
+    signal?.throwIfAborted();
+    // Hard-link publication is atomic and fails if another process created the
+    // destination. rename() would silently overwrite that file on Unix.
+    await link(temporary, destination);
+    published = true;
+    return { release: schema.release, output: destination, counts };
+  } finally {
+    try {
+      database?.close();
+    } finally {
+      try {
+        await connection?.end();
+      } finally {
+        if (created)
+          await unlink(temporary).catch((error) => {
+            if (error.code === "ENOENT") return;
+            if (!published) throw error;
+            // The verified output already exists. A leftover hard link must not
+            // turn a successful conversion into a misleading failure report.
+            console.warn("Conversion succeeded, but its temporary .partial file could not be removed.");
+          });
+      }
+    }
+  }
+}

tools/mysql-to-sqlite/src/legacy.mjs

@@ -0,0 +1,29 @@
+import { createHash } from "node:crypto";
+import { readFileSync } from "node:fs";
+
+const root = new URL("../legacy/v1.77.1/", import.meta.url);
+export const schema = JSON.parse(readFileSync(new URL("schema.json", root), "utf8"));
+
+export function migrations(dialect) {
+  const journal = JSON.parse(readFileSync(new URL(`${dialect}/journal.json`, root), "utf8"));
+  return journal.entries.map((entry) => {
+    const sql = readFileSync(new URL(`${dialect}/${entry.tag}.sql`, root), "utf8");
+    return { ...entry, sql, hash: createHash("sha256").update(sql).digest("hex") };
+  });
+}
+
+export const quote = (identifier) => `\`${identifier.replaceAll("`", "``")}\``;
+
+// Use the release's SQL, not its seeding entrypoint: source data owns every ID.
+export function initializeSqlite(database) {
+  database.exec('CREATE TABLE "__drizzle_migrations" (id SERIAL PRIMARY KEY, hash text NOT NULL, created_at numeric)');
+  const record = database.prepare('INSERT INTO "__drizzle_migrations" (hash, created_at) VALUES (?, ?)');
+  database.exec("BEGIN");
+  for (const migration of migrations("sqlite")) {
+    for (const statement of migration.sql.split("--> statement-breakpoint")) {
+      if (statement.trim()) database.exec(statement);
+    }
+    record.run(migration.hash, migration.when);
+  }
+  database.exec("COMMIT");
+}