fix(assistant): mysql message storage, migration metadata and per-page bundle cost (#6530)

This commit is contained in:
Thomas authored and GitHub committed 2026-08-08 09:34:16 +02:00
1 parent 0e3461c6d7
commit 20fd49f082
20 files changed
+6417 -167

No files matched your search

@@ -0,0 +1,98 @@
import fs from "fs";
import path from "path";
import { describe, expect, test } from "vitest";
/**
* Guards the migration folders against the failure modes that have actually shipped:
*
* - v1.70.0 shipped `0042_add_widget_secret.sql` for MySQL without a `_journal.json` entry, so the
* migrator never ran it and MySQL instances upgraded from that release had no `widget_secret`
* table.
* - The follow-up fix hand-wrote `0042_snapshot.json` with a `checkConstraints` key instead of
* `checkConstraint`, which made `drizzle-kit generate` reject the folder and silently exit 0 for
* every later MySQL migration.
*/
const dialects = ["mysql", "postgresql", "sqlite"] as const;
const migrationsRoot = path.join(__dirname, "..", "migrations");
interface Journal {
entries: { idx: number; tag: string }[];
}
const readDialect = (dialect: (typeof dialects)[number]) => {
const folder = path.join(migrationsRoot, dialect);
const journal = JSON.parse(fs.readFileSync(path.join(folder, "meta", "_journal.json"), "utf8")) as Journal;
const sqlFiles = fs
.readdirSync(folder)
.filter((file) => file.endsWith(".sql"))
.map((file) => file.replace(/\.sql$/u, ""));
const snapshots = fs
.readdirSync(path.join(folder, "meta"))
.filter((file) => /^\d+_snapshot\.json$/u.test(file))
.map((file) => file.split("_")[0]);
return { folder, journal, sqlFiles, snapshots };
};
describe.each(dialects)("%s migrations", (dialect) => {
test("every journal entry has a matching .sql file", () => {
const { journal, sqlFiles } = readDialect(dialect);
const missing = journal.entries.filter((entry) => !sqlFiles.includes(entry.tag)).map((entry) => entry.tag);
expect(missing).toEqual([]);
});
test("every .sql file is registered in the journal", () => {
const { journal, sqlFiles } = readDialect(dialect);
const tags = new Set(journal.entries.map((entry) => entry.tag));
const orphans = sqlFiles.filter((file) => !tags.has(file));
expect(orphans).toEqual([]);
});
test("every journal entry has a matching snapshot", () => {
const { journal, snapshots } = readDialect(dialect);
const missing = journal.entries
.filter((entry) => !snapshots.includes(String(entry.idx).padStart(4, "0")))
.map((entry) => entry.tag);
expect(missing).toEqual([]);
});
test("journal indexes are unique and gap-free", () => {
const { journal } = readDialect(dialect);
const indexes = journal.entries.map((entry) => entry.idx);
expect(indexes).toEqual(indexes.map((_, position) => position));
});
// The check-constraint key is spelled differently per snapshot format version (MySQL's v5 uses
// `checkConstraint`, the newer SQLite/Postgres formats use `checkConstraints`), so rather than
// hard-coding a spelling this asserts every snapshot in a dialect agrees with the others. A
// hand-written snapshot that gets it wrong stands out as the odd one and fails drizzle-kit.
test("all snapshots agree on the check-constraint key spelling", () => {
const { folder, snapshots } = readDialect(dialect);
const spellingsPerSnapshot = snapshots.map((snapshot) => {
const file = path.join(folder, "meta", `${snapshot}_snapshot.json`);
const parsed = JSON.parse(fs.readFileSync(file, "utf8")) as { tables?: Record<string, object> };
const spellings = new Set<string>();
for (const table of Object.values(parsed.tables ?? {})) {
if ("checkConstraint" in table) spellings.add("checkConstraint");
if ("checkConstraints" in table) spellings.add("checkConstraints");
}
return { snapshot, spellings: [...spellings].toSorted().join("+") };
});
const used = [...new Set(spellingsPerSnapshot.map((entry) => entry.spellings))].filter(
(spelling) => spelling.length > 0,
);
expect(used, `snapshots disagree: ${JSON.stringify(spellingsPerSnapshot.filter((e) => e.spellings))}`).toHaveLength(
1,
);
});
});
+18 -1
View File
@@ -1,9 +1,10 @@
import path from "path";
import { MySqlContainer } from "@testcontainers/mysql";
import { eq } from "drizzle-orm";
import { drizzle } from "drizzle-orm/mysql2";
import { migrate } from "drizzle-orm/mysql2/migrator";
import mysql from "mysql2";
import { describe, test } from "vitest";
import { describe, expect, test } from "vitest";
import { DB_CASING } from "@homarr/core/infrastructure/db/constants";
@@ -35,6 +36,22 @@ describe("Mysql Migration", () => {
// Check if users table exists
await database.query.users.findMany();
// A single base64 image attachment already exceeds the 64KB that `text` holds on MySQL, so
// `assistant_message.content` has to be `mediumtext`. Everything below 64KB passes either way.
const largeContent = "x".repeat(200_000);
await database.insert(mysqlSchema.users).values({ id: "size-check-user", name: "size-check" });
await database.insert(mysqlSchema.assistantThreads).values({ id: "size-check-thread", userId: "size-check-user" });
await database.insert(mysqlSchema.assistantMessages).values({
id: "size-check-message",
threadId: "size-check-thread",
parentId: null,
content: largeContent,
});
const [storedMessage] = await database.query.assistantMessages.findMany({
where: eq(mysqlSchema.assistantMessages.id, "size-check-message"),
});
expect(storedMessage?.content).toHaveLength(largeContent.length);
connection.end();
await mysqlContainer.stop();
}, 120_000);
+58 -32
View File
@@ -1,7 +1,10 @@
/* eslint-disable @typescript-eslint/no-non-null-assertion */
import { is } from "drizzle-orm";
import type { Column, InferSelectModel } from "drizzle-orm";
import { MySqlTable, getTableConfig as getMysqlTableConfig } from "drizzle-orm/mysql-core";
import type { ForeignKey as MysqlForeignKey, MySqlTableWithColumns } from "drizzle-orm/mysql-core";
import type { PgTableWithColumns, ForeignKey as PostgresqlForeignKey } from "drizzle-orm/pg-core";
import { SQLiteTable, getTableConfig as getSqliteTableConfig } from "drizzle-orm/sqlite-core";
import type { ForeignKey as SqliteForeignKey, SQLiteTableWithColumns } from "drizzle-orm/sqlite-core";
import { expect, expectTypeOf, test } from "vitest";
@@ -54,6 +57,21 @@ type FixedSqliteConfig = {
};
};
/** Identifies a foreign key by what it links rather than by its generated name. */
const referenceSignature = (foreignKey: {
reference: () => { columns: { name: string }[]; foreignTable: object; foreignColumns: { name: string }[] };
}) => {
const reference = foreignKey.reference();
const target = Object.keys(reference.foreignTable).sort().join("+");
return `${reference.columns
.map((column) => column.name)
.sort()
.join(",")}->${target}(${reference.foreignColumns
.map((column) => column.name)
.sort()
.join(",")})`;
};
test("schemas should match", () => {
expectTypeOf<SqliteTables>().toEqualTypeOf<MysqlTables>();
expectTypeOf<MysqlTables>().toEqualTypeOf<SqliteTables>();
@@ -88,56 +106,64 @@ test("schemas should match", () => {
});
const mysqlTable = mysqlTables[tableName];
const sqliteForeignKeys = sqliteTable[Symbol.for("drizzle:SQLiteInlineForeignKeys") as keyof typeof sqliteTable] as
| SqliteForeignKey[]
| undefined;
const mysqlForeignKeys = mysqlTable[Symbol.for("drizzle:MySqlInlineForeignKeys") as keyof typeof mysqlTable] as
| MysqlForeignKey[]
| undefined;
// Read through getTableConfig so keys declared at table level count too, not just inline
// `.references()` ones. MySQL needs a table-level declaration wherever drizzle's derived name
// would exceed its 64-character identifier limit.
if (!is(sqliteTable, SQLiteTable) || !is(mysqlTable, MySqlTable)) return;
if (!sqliteForeignKeys && !mysqlForeignKeys) return;
const sqliteForeignKeys = getSqliteTableConfig(sqliteTable).foreignKeys as SqliteForeignKey[];
const mysqlForeignKeys = getMysqlTableConfig(mysqlTable).foreignKeys as MysqlForeignKey[];
expect(mysqlForeignKeys, `mysql foreign key for ${tableName} to be defined`).toBeDefined();
expect(sqliteForeignKeys, `sqlite foreign key for ${tableName} to be defined`).toBeDefined();
if (sqliteForeignKeys.length === 0 && mysqlForeignKeys.length === 0) return;
expect(
sqliteForeignKeys!.length,
sqliteForeignKeys.length,
`expect number of foreign keys in table ${tableName} to be the same for both schemas`,
).toEqual(mysqlForeignKeys!.length);
).toEqual(mysqlForeignKeys.length);
sqliteForeignKeys?.forEach((sqliteForeignKey) => {
sqliteForeignKey.getName();
const mysqlForeignKey = mysqlForeignKeys!.find((key) => key.getName() === sqliteForeignKey.getName());
expect(
mysqlForeignKey,
`expect foreign key ${sqliteForeignKey.getName()} to be defined in mysql schema`,
).toBeDefined();
sqliteForeignKeys.forEach((sqliteForeignKey) => {
const describeKey = `${tableName}.${referenceSignature(sqliteForeignKey)}`;
// Matched on what the key actually does rather than on its generated name, because the name
// is allowed to differ per dialect.
const mysqlForeignKey = mysqlForeignKeys.find(
(key) => referenceSignature(key) === referenceSignature(sqliteForeignKey),
);
expect(mysqlForeignKey, `expect foreign key ${describeKey} to be defined in mysql schema`).toBeDefined();
expect(
sqliteForeignKey.onDelete,
`expect foreign key (${sqliteForeignKey.getName()}) onDelete to be the same for both schemas`,
`expect foreign key (${describeKey}) onDelete to be the same for both schemas`,
).toEqual(mysqlForeignKey!.onDelete);
expect(
sqliteForeignKey.onUpdate,
`expect foreign key (${sqliteForeignKey.getName()}) onUpdate to be the same for both schemas`,
`expect foreign key (${describeKey}) onUpdate to be the same for both schemas`,
).toEqual(mysqlForeignKey!.onUpdate);
sqliteForeignKey.reference().foreignColumns.forEach((column) => {
expect(
mysqlForeignKey!.reference().foreignColumns.map((column) => column.name),
`expect foreign key (${sqliteForeignKey.getName()}) columns to be the same for both schemas`,
).toContainEqual(column.name);
});
expect(
Object.keys(sqliteForeignKey.reference().foreignTable),
`expect foreign key (${sqliteForeignKey.getName()}) table to be the same for both schemas`,
).toEqual(Object.keys(mysqlForeignKey!.reference().foreignTable));
});
});
});
test("mysql identifiers stay within the 64 character limit", () => {
// MySQL rejects longer identifiers with error 1059, and drizzle derives foreign key names from
// the table and column names, so a long pairing silently produces a migration that cannot apply.
const tooLong: string[] = [];
objectEntries(mysqlSchema).forEach(([, table]) => {
if (!is(table, MySqlTable)) return;
const config = getMysqlTableConfig(table);
if (config.name.length > 64) tooLong.push(config.name);
for (const foreignKey of config.foreignKeys) {
if (foreignKey.getName().length > 64) tooLong.push(foreignKey.getName());
}
for (const index of config.indexes) {
if (index.config.name.length > 64) tooLong.push(index.config.name);
}
});
expect(tooLong).toEqual([]);
});
test("schemas should match for postgresql", () => {
expectTypeOf<SqliteTables>().toEqualTypeOf<PostgresqlTables>();
expectTypeOf<PostgresqlTables>().toEqualTypeOf<SqliteTables>();