fix(assistant): mysql message storage, migration metadata and per-page bundle cost (#6530)
This commit is contained in:
1 parent
0e3461c6d7
commit
20fd49f082
20 files changed
+6417
-167
No files matched your search
@@ -26,7 +26,7 @@ CREATE TABLE `assistant_message` (
|
||||
`thread_id` varchar(64) NOT NULL,
|
||||
`parent_id` varchar(128),
|
||||
`format` varchar(64) DEFAULT 'ai-sdk/v6' NOT NULL,
|
||||
`content` text DEFAULT ('{"json": {}}') NOT NULL,
|
||||
`content` mediumtext DEFAULT ('{"json": {}}') NOT NULL,
|
||||
`created_at` timestamp DEFAULT (now()) NOT NULL,
|
||||
CONSTRAINT `assistant_message_id` PRIMARY KEY(`id`)
|
||||
);
|
||||
|
||||
@@ -2627,7 +2627,7 @@
|
||||
}
|
||||
},
|
||||
"uniqueConstraints": {},
|
||||
"checkConstraints": {}
|
||||
"checkConstraint": {}
|
||||
}
|
||||
},
|
||||
"views": {},
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -6,8 +6,10 @@ import type { AnyMySqlColumn } from "drizzle-orm/mysql-core";
|
||||
import {
|
||||
boolean,
|
||||
customType,
|
||||
foreignKey,
|
||||
index,
|
||||
int,
|
||||
mediumtext,
|
||||
mysqlTable,
|
||||
primaryKey,
|
||||
smallint,
|
||||
@@ -536,7 +538,9 @@ export const assistantMessages = mysqlTable(
|
||||
.references(() => assistantThreads.id, { onDelete: "cascade" }),
|
||||
parentId: varchar({ length: 128 }),
|
||||
format: varchar({ length: 64 }).notNull().default("ai-sdk/v6"),
|
||||
content: text().default(emptySuperJSON).notNull(),
|
||||
// `text` only holds 64KB on MySQL, which a single base64 image attachment already exceeds.
|
||||
// `mediumtext` holds 16MB, comfortably above the 2MB request cap enforced by the chat route.
|
||||
content: mediumtext().default(emptySuperJSON).notNull(),
|
||||
createdAt: timestamp().notNull().defaultNow(),
|
||||
},
|
||||
(message) => ({
|
||||
@@ -594,14 +598,20 @@ export const customWidgetSecrets = mysqlTable(
|
||||
updatedAt: timestamp()
|
||||
.$onUpdateFn(() => new Date())
|
||||
.notNull(),
|
||||
definitionId: varchar({ length: 64 })
|
||||
.notNull()
|
||||
.references(() => customWidgetDefinitions.id, { onDelete: "cascade" }),
|
||||
definitionId: varchar({ length: 64 }).notNull(),
|
||||
},
|
||||
(table) => ({
|
||||
compoundKey: primaryKey({
|
||||
columns: [table.definitionId, table.kind],
|
||||
}),
|
||||
// The name drizzle derives here is 65 characters, one over MySQL's 64-character identifier
|
||||
// limit, so migration 0040 created the key under this shortened name. Declaring it explicitly
|
||||
// keeps `drizzle-kit generate` from re-proposing an ALTER that MySQL rejects with error 1059.
|
||||
definitionReference: foreignKey({
|
||||
name: "cw_secret_definition_id_cw_definition_id_fk",
|
||||
columns: [table.definitionId],
|
||||
foreignColumns: [customWidgetDefinitions.id],
|
||||
}).onDelete("cascade"),
|
||||
}),
|
||||
);
|
||||
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { describe, expect, test } from "vitest";
|
||||
|
||||
/**
|
||||
* Guards the migration folders against the failure modes that have actually shipped:
|
||||
*
|
||||
* - v1.70.0 shipped `0042_add_widget_secret.sql` for MySQL without a `_journal.json` entry, so the
|
||||
* migrator never ran it and MySQL instances upgraded from that release had no `widget_secret`
|
||||
* table.
|
||||
* - The follow-up fix hand-wrote `0042_snapshot.json` with a `checkConstraints` key instead of
|
||||
* `checkConstraint`, which made `drizzle-kit generate` reject the folder and silently exit 0 for
|
||||
* every later MySQL migration.
|
||||
*/
|
||||
const dialects = ["mysql", "postgresql", "sqlite"] as const;
|
||||
|
||||
const migrationsRoot = path.join(__dirname, "..", "migrations");
|
||||
|
||||
interface Journal {
|
||||
entries: { idx: number; tag: string }[];
|
||||
}
|
||||
|
||||
const readDialect = (dialect: (typeof dialects)[number]) => {
|
||||
const folder = path.join(migrationsRoot, dialect);
|
||||
const journal = JSON.parse(fs.readFileSync(path.join(folder, "meta", "_journal.json"), "utf8")) as Journal;
|
||||
const sqlFiles = fs
|
||||
.readdirSync(folder)
|
||||
.filter((file) => file.endsWith(".sql"))
|
||||
.map((file) => file.replace(/\.sql$/u, ""));
|
||||
const snapshots = fs
|
||||
.readdirSync(path.join(folder, "meta"))
|
||||
.filter((file) => /^\d+_snapshot\.json$/u.test(file))
|
||||
.map((file) => file.split("_")[0]);
|
||||
|
||||
return { folder, journal, sqlFiles, snapshots };
|
||||
};
|
||||
|
||||
describe.each(dialects)("%s migrations", (dialect) => {
|
||||
test("every journal entry has a matching .sql file", () => {
|
||||
const { journal, sqlFiles } = readDialect(dialect);
|
||||
const missing = journal.entries.filter((entry) => !sqlFiles.includes(entry.tag)).map((entry) => entry.tag);
|
||||
|
||||
expect(missing).toEqual([]);
|
||||
});
|
||||
|
||||
test("every .sql file is registered in the journal", () => {
|
||||
const { journal, sqlFiles } = readDialect(dialect);
|
||||
const tags = new Set(journal.entries.map((entry) => entry.tag));
|
||||
const orphans = sqlFiles.filter((file) => !tags.has(file));
|
||||
|
||||
expect(orphans).toEqual([]);
|
||||
});
|
||||
|
||||
test("every journal entry has a matching snapshot", () => {
|
||||
const { journal, snapshots } = readDialect(dialect);
|
||||
const missing = journal.entries
|
||||
.filter((entry) => !snapshots.includes(String(entry.idx).padStart(4, "0")))
|
||||
.map((entry) => entry.tag);
|
||||
|
||||
expect(missing).toEqual([]);
|
||||
});
|
||||
|
||||
test("journal indexes are unique and gap-free", () => {
|
||||
const { journal } = readDialect(dialect);
|
||||
const indexes = journal.entries.map((entry) => entry.idx);
|
||||
|
||||
expect(indexes).toEqual(indexes.map((_, position) => position));
|
||||
});
|
||||
|
||||
// The check-constraint key is spelled differently per snapshot format version (MySQL's v5 uses
|
||||
// `checkConstraint`, the newer SQLite/Postgres formats use `checkConstraints`), so rather than
|
||||
// hard-coding a spelling this asserts every snapshot in a dialect agrees with the others. A
|
||||
// hand-written snapshot that gets it wrong stands out as the odd one and fails drizzle-kit.
|
||||
test("all snapshots agree on the check-constraint key spelling", () => {
|
||||
const { folder, snapshots } = readDialect(dialect);
|
||||
|
||||
const spellingsPerSnapshot = snapshots.map((snapshot) => {
|
||||
const file = path.join(folder, "meta", `${snapshot}_snapshot.json`);
|
||||
const parsed = JSON.parse(fs.readFileSync(file, "utf8")) as { tables?: Record<string, object> };
|
||||
const spellings = new Set<string>();
|
||||
|
||||
for (const table of Object.values(parsed.tables ?? {})) {
|
||||
if ("checkConstraint" in table) spellings.add("checkConstraint");
|
||||
if ("checkConstraints" in table) spellings.add("checkConstraints");
|
||||
}
|
||||
|
||||
return { snapshot, spellings: [...spellings].toSorted().join("+") };
|
||||
});
|
||||
|
||||
const used = [...new Set(spellingsPerSnapshot.map((entry) => entry.spellings))].filter(
|
||||
(spelling) => spelling.length > 0,
|
||||
);
|
||||
|
||||
expect(used, `snapshots disagree: ${JSON.stringify(spellingsPerSnapshot.filter((e) => e.spellings))}`).toHaveLength(
|
||||
1,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,9 +1,10 @@
|
||||
import path from "path";
|
||||
import { MySqlContainer } from "@testcontainers/mysql";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { drizzle } from "drizzle-orm/mysql2";
|
||||
import { migrate } from "drizzle-orm/mysql2/migrator";
|
||||
import mysql from "mysql2";
|
||||
import { describe, test } from "vitest";
|
||||
import { describe, expect, test } from "vitest";
|
||||
|
||||
import { DB_CASING } from "@homarr/core/infrastructure/db/constants";
|
||||
|
||||
@@ -35,6 +36,22 @@ describe("Mysql Migration", () => {
|
||||
// Check if users table exists
|
||||
await database.query.users.findMany();
|
||||
|
||||
// A single base64 image attachment already exceeds the 64KB that `text` holds on MySQL, so
|
||||
// `assistant_message.content` has to be `mediumtext`. Everything below 64KB passes either way.
|
||||
const largeContent = "x".repeat(200_000);
|
||||
await database.insert(mysqlSchema.users).values({ id: "size-check-user", name: "size-check" });
|
||||
await database.insert(mysqlSchema.assistantThreads).values({ id: "size-check-thread", userId: "size-check-user" });
|
||||
await database.insert(mysqlSchema.assistantMessages).values({
|
||||
id: "size-check-message",
|
||||
threadId: "size-check-thread",
|
||||
parentId: null,
|
||||
content: largeContent,
|
||||
});
|
||||
const [storedMessage] = await database.query.assistantMessages.findMany({
|
||||
where: eq(mysqlSchema.assistantMessages.id, "size-check-message"),
|
||||
});
|
||||
expect(storedMessage?.content).toHaveLength(largeContent.length);
|
||||
|
||||
connection.end();
|
||||
await mysqlContainer.stop();
|
||||
}, 120_000);
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
/* eslint-disable @typescript-eslint/no-non-null-assertion */
|
||||
import { is } from "drizzle-orm";
|
||||
import type { Column, InferSelectModel } from "drizzle-orm";
|
||||
import { MySqlTable, getTableConfig as getMysqlTableConfig } from "drizzle-orm/mysql-core";
|
||||
import type { ForeignKey as MysqlForeignKey, MySqlTableWithColumns } from "drizzle-orm/mysql-core";
|
||||
import type { PgTableWithColumns, ForeignKey as PostgresqlForeignKey } from "drizzle-orm/pg-core";
|
||||
import { SQLiteTable, getTableConfig as getSqliteTableConfig } from "drizzle-orm/sqlite-core";
|
||||
import type { ForeignKey as SqliteForeignKey, SQLiteTableWithColumns } from "drizzle-orm/sqlite-core";
|
||||
import { expect, expectTypeOf, test } from "vitest";
|
||||
|
||||
@@ -54,6 +57,21 @@ type FixedSqliteConfig = {
|
||||
};
|
||||
};
|
||||
|
||||
/** Identifies a foreign key by what it links rather than by its generated name. */
|
||||
const referenceSignature = (foreignKey: {
|
||||
reference: () => { columns: { name: string }[]; foreignTable: object; foreignColumns: { name: string }[] };
|
||||
}) => {
|
||||
const reference = foreignKey.reference();
|
||||
const target = Object.keys(reference.foreignTable).sort().join("+");
|
||||
return `${reference.columns
|
||||
.map((column) => column.name)
|
||||
.sort()
|
||||
.join(",")}->${target}(${reference.foreignColumns
|
||||
.map((column) => column.name)
|
||||
.sort()
|
||||
.join(",")})`;
|
||||
};
|
||||
|
||||
test("schemas should match", () => {
|
||||
expectTypeOf<SqliteTables>().toEqualTypeOf<MysqlTables>();
|
||||
expectTypeOf<MysqlTables>().toEqualTypeOf<SqliteTables>();
|
||||
@@ -88,56 +106,64 @@ test("schemas should match", () => {
|
||||
});
|
||||
|
||||
const mysqlTable = mysqlTables[tableName];
|
||||
const sqliteForeignKeys = sqliteTable[Symbol.for("drizzle:SQLiteInlineForeignKeys") as keyof typeof sqliteTable] as
|
||||
| SqliteForeignKey[]
|
||||
| undefined;
|
||||
const mysqlForeignKeys = mysqlTable[Symbol.for("drizzle:MySqlInlineForeignKeys") as keyof typeof mysqlTable] as
|
||||
| MysqlForeignKey[]
|
||||
| undefined;
|
||||
// Read through getTableConfig so keys declared at table level count too, not just inline
|
||||
// `.references()` ones. MySQL needs a table-level declaration wherever drizzle's derived name
|
||||
// would exceed its 64-character identifier limit.
|
||||
if (!is(sqliteTable, SQLiteTable) || !is(mysqlTable, MySqlTable)) return;
|
||||
|
||||
if (!sqliteForeignKeys && !mysqlForeignKeys) return;
|
||||
const sqliteForeignKeys = getSqliteTableConfig(sqliteTable).foreignKeys as SqliteForeignKey[];
|
||||
const mysqlForeignKeys = getMysqlTableConfig(mysqlTable).foreignKeys as MysqlForeignKey[];
|
||||
|
||||
expect(mysqlForeignKeys, `mysql foreign key for ${tableName} to be defined`).toBeDefined();
|
||||
expect(sqliteForeignKeys, `sqlite foreign key for ${tableName} to be defined`).toBeDefined();
|
||||
if (sqliteForeignKeys.length === 0 && mysqlForeignKeys.length === 0) return;
|
||||
|
||||
expect(
|
||||
sqliteForeignKeys!.length,
|
||||
sqliteForeignKeys.length,
|
||||
`expect number of foreign keys in table ${tableName} to be the same for both schemas`,
|
||||
).toEqual(mysqlForeignKeys!.length);
|
||||
).toEqual(mysqlForeignKeys.length);
|
||||
|
||||
sqliteForeignKeys?.forEach((sqliteForeignKey) => {
|
||||
sqliteForeignKey.getName();
|
||||
const mysqlForeignKey = mysqlForeignKeys!.find((key) => key.getName() === sqliteForeignKey.getName());
|
||||
expect(
|
||||
mysqlForeignKey,
|
||||
`expect foreign key ${sqliteForeignKey.getName()} to be defined in mysql schema`,
|
||||
).toBeDefined();
|
||||
sqliteForeignKeys.forEach((sqliteForeignKey) => {
|
||||
const describeKey = `${tableName}.${referenceSignature(sqliteForeignKey)}`;
|
||||
// Matched on what the key actually does rather than on its generated name, because the name
|
||||
// is allowed to differ per dialect.
|
||||
const mysqlForeignKey = mysqlForeignKeys.find(
|
||||
(key) => referenceSignature(key) === referenceSignature(sqliteForeignKey),
|
||||
);
|
||||
expect(mysqlForeignKey, `expect foreign key ${describeKey} to be defined in mysql schema`).toBeDefined();
|
||||
|
||||
expect(
|
||||
sqliteForeignKey.onDelete,
|
||||
`expect foreign key (${sqliteForeignKey.getName()}) onDelete to be the same for both schemas`,
|
||||
`expect foreign key (${describeKey}) onDelete to be the same for both schemas`,
|
||||
).toEqual(mysqlForeignKey!.onDelete);
|
||||
|
||||
expect(
|
||||
sqliteForeignKey.onUpdate,
|
||||
`expect foreign key (${sqliteForeignKey.getName()}) onUpdate to be the same for both schemas`,
|
||||
`expect foreign key (${describeKey}) onUpdate to be the same for both schemas`,
|
||||
).toEqual(mysqlForeignKey!.onUpdate);
|
||||
|
||||
sqliteForeignKey.reference().foreignColumns.forEach((column) => {
|
||||
expect(
|
||||
mysqlForeignKey!.reference().foreignColumns.map((column) => column.name),
|
||||
`expect foreign key (${sqliteForeignKey.getName()}) columns to be the same for both schemas`,
|
||||
).toContainEqual(column.name);
|
||||
});
|
||||
|
||||
expect(
|
||||
Object.keys(sqliteForeignKey.reference().foreignTable),
|
||||
`expect foreign key (${sqliteForeignKey.getName()}) table to be the same for both schemas`,
|
||||
).toEqual(Object.keys(mysqlForeignKey!.reference().foreignTable));
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test("mysql identifiers stay within the 64 character limit", () => {
|
||||
// MySQL rejects longer identifiers with error 1059, and drizzle derives foreign key names from
|
||||
// the table and column names, so a long pairing silently produces a migration that cannot apply.
|
||||
const tooLong: string[] = [];
|
||||
|
||||
objectEntries(mysqlSchema).forEach(([, table]) => {
|
||||
if (!is(table, MySqlTable)) return;
|
||||
|
||||
const config = getMysqlTableConfig(table);
|
||||
if (config.name.length > 64) tooLong.push(config.name);
|
||||
for (const foreignKey of config.foreignKeys) {
|
||||
if (foreignKey.getName().length > 64) tooLong.push(foreignKey.getName());
|
||||
}
|
||||
for (const index of config.indexes) {
|
||||
if (index.config.name.length > 64) tooLong.push(index.config.name);
|
||||
}
|
||||
});
|
||||
|
||||
expect(tooLong).toEqual([]);
|
||||
});
|
||||
|
||||
test("schemas should match for postgresql", () => {
|
||||
expectTypeOf<SqliteTables>().toEqualTypeOf<PostgresqlTables>();
|
||||
expectTypeOf<PostgresqlTables>().toEqualTypeOf<SqliteTables>();
|
||||
|
||||
Reference in new issue
Block a user