diff --git a/scripts/etl-mariadb-to-pg.mjs b/scripts/etl-mariadb-to-pg.mjs index 1386760..ee5f178 100644 --- a/scripts/etl-mariadb-to-pg.mjs +++ b/scripts/etl-mariadb-to-pg.mjs @@ -47,21 +47,6 @@ const TABLES = [ const ROW_CAP = { ServerNodeHistory: 20000, MihomeEnvSample: 20000, MihomeEnvHour: 20000 } -const JSON_COLUMNS = { - Strategy: ['data'], - ScraperCache: ['metadata'], - MihomeEnvSample: ['raw'], - MihomeEnvHour: ['raw'], - Project: ['notes', 'taskKinds'], - Task: ['extras'], - CheckListTemplate: ['items'], -} - -const BIGINT_COLUMNS = { - ServerNode: ['memTotal', 'diskTotal', 'ramUsed', 'swapUsed', 'diskUsed', 'netIn', 'netOut', 'uptime'], - ServerNodeHistory: ['ram', 'disk', 'netIn', 'netOut'], -} - const UUID_TABLES = new Set(['Project', 'ProjectMember', 'BoardColumn', 'Task', 'TaskMember', 'TaskTag']) function parseUrl(url) { @@ -85,6 +70,12 @@ function toBigInt(v) { return BigInt(v) } +// MariaDB TINYINT(1) 返回 0/1, Prisma Boolean 需要 true/false +function toBoolean(v) { + if (v === null || v === undefined) return v + return !!v +} + function randCode(len) { return crypto.randomBytes(len * 2).toString('hex').toUpperCase().replace(/[^A-Z0-9]/g, '').padEnd(len, 'X').slice(0, len) } @@ -109,6 +100,30 @@ async function main() { return } + // 目标 PG 列清单 (类型感知转换 + 源库多余列丢弃): 现网库可能比本分支 schema 新 + const targetColumns = new Map() + for (const [table] of TABLES) { + const cols = await prisma.$queryRawUnsafe( + `SELECT column_name, data_type FROM information_schema.columns WHERE table_schema = 'public' AND table_name = $1`, + table, + ) + targetColumns.set(table, new Map(cols.map((c) => [c.column_name, c.data_type]))) + } + + function adaptRow(table, r) { + const cols = targetColumns.get(table) + const out = {} + for (const [name, dataType] of cols) { + let v = r[name] + if (v === undefined) v = null + if (dataType === 'boolean') v = toBoolean(v) + else if (dataType === 'bigint') v = toBigInt(v) + else if (dataType === 'json' || dataType === 'jsonb') v = parseJson(v) + out[name] = v + } + return out + } + // Project.code 回填: 优先按名称推导字母, 冲突加随机尾缀 const projectRows = await conn.query('SELECT * FROM `Project`') const usedCodes = new Set() @@ -152,16 +167,13 @@ async function main() { rows = await conn.query(cap ? `SELECT * FROM \`${table}\` ORDER BY id DESC LIMIT ${cap}` : `SELECT * FROM \`${table}\``) } if (rows.length) { - const jsonCols = JSON_COLUMNS[table] || [] - const bigCols = BIGINT_COLUMNS[table] || [] - for (const r of rows) { - for (const c of jsonCols) r[c] = parseJson(r[c]) - for (const c of bigCols) r[c] = toBigInt(r[c]) + const adapted = rows.map((r) => { if (UUID_TABLES.has(table)) r.uuid = crypto.randomUUID() - } + return adaptRow(table, r) + }) const chunk = 500 - for (let i = 0; i < rows.length; i += chunk) { - await prisma[model].createMany({ data: rows.slice(i, i + chunk) }) + for (let i = 0; i < adapted.length; i += chunk) { + await prisma[model].createMany({ data: adapted.slice(i, i + chunk) }) } } counts[table] = rows.length