From 1a695920ce294442b0d793569e5b9be9b25233ac Mon Sep 17 00:00:00 2001
From: "jif2.zhang" <jif2.zhang>
Date: Mon, 26 Oct 2026 00:29:08 +0800
Subject: [PATCH] =?UTF-8?q?fix(0.2.1):=20ETL=20=E6=8C=89=E7=9B=AE=E6=A0=87?=
 =?UTF-8?q?=20PG=20=E5=88=97=E7=99=BD=E5=90=8D=E5=8D=95=E5=8F=96=E6=95=B0?=
 =?UTF-8?q?=20+=20boolean/bigint/json=20=E7=B1=BB=E5=9E=8B=E6=84=9F?=
 =?UTF-8?q?=E7=9F=A5=E8=BD=AC=E6=8D=A2=20(=E7=8E=B0=E7=BD=91=E5=BA=93=20sc?=
 =?UTF-8?q?hema=20=E9=A2=86=E5=85=88=E4=BA=8E=E6=9C=AC=E5=88=86=E6=94=AF)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 scripts/etl-mariadb-to-pg.mjs | 58 +++++++++++++++++++++--------------
 1 file changed, 35 insertions(+), 23 deletions(-)

diff --git a/scripts/etl-mariadb-to-pg.mjs b/scripts/etl-mariadb-to-pg.mjs
index 1386760..ee5f178 100644
--- a/scripts/etl-mariadb-to-pg.mjs
+++ b/scripts/etl-mariadb-to-pg.mjs
@@ -47,21 +47,6 @@ const TABLES = [
 
 const ROW_CAP = { ServerNodeHistory: 20000, MihomeEnvSample: 20000, MihomeEnvHour: 20000 }
 
-const JSON_COLUMNS = {
-  Strategy: ['data'],
-  ScraperCache: ['metadata'],
-  MihomeEnvSample: ['raw'],
-  MihomeEnvHour: ['raw'],
-  Project: ['notes', 'taskKinds'],
-  Task: ['extras'],
-  CheckListTemplate: ['items'],
-}
-
-const BIGINT_COLUMNS = {
-  ServerNode: ['memTotal', 'diskTotal', 'ramUsed', 'swapUsed', 'diskUsed', 'netIn', 'netOut', 'uptime'],
-  ServerNodeHistory: ['ram', 'disk', 'netIn', 'netOut'],
-}
-
 const UUID_TABLES = new Set(['Project', 'ProjectMember', 'BoardColumn', 'Task', 'TaskMember', 'TaskTag'])
 
 function parseUrl(url) {
@@ -85,6 +70,12 @@ function toBigInt(v) {
   return BigInt(v)
 }
 
+// MariaDB TINYINT(1) 返回 0/1, Prisma Boolean 需要 true/false
+function toBoolean(v) {
+  if (v === null || v === undefined) return v
+  return !!v
+}
+
 function randCode(len) {
   return crypto.randomBytes(len * 2).toString('hex').toUpperCase().replace(/[^A-Z0-9]/g, '').padEnd(len, 'X').slice(0, len)
 }
@@ -109,6 +100,30 @@ async function main() {
     return
   }
 
+  // 目标 PG 列清单 (类型感知转换 + 源库多余列丢弃): 现网库可能比本分支 schema 新
+  const targetColumns = new Map()
+  for (const [table] of TABLES) {
+    const cols = await prisma.$queryRawUnsafe(
+      `SELECT column_name, data_type FROM information_schema.columns WHERE table_schema = 'public' AND table_name = $1`,
+      table,
+    )
+    targetColumns.set(table, new Map(cols.map((c) => [c.column_name, c.data_type])))
+  }
+
+  function adaptRow(table, r) {
+    const cols = targetColumns.get(table)
+    const out = {}
+    for (const [name, dataType] of cols) {
+      let v = r[name]
+      if (v === undefined) v = null
+      if (dataType === 'boolean') v = toBoolean(v)
+      else if (dataType === 'bigint') v = toBigInt(v)
+      else if (dataType === 'json' || dataType === 'jsonb') v = parseJson(v)
+      out[name] = v
+    }
+    return out
+  }
+
   // Project.code 回填: 优先按名称推导字母, 冲突加随机尾缀
   const projectRows = await conn.query('SELECT * FROM `Project`')
   const usedCodes = new Set()
@@ -152,16 +167,13 @@ async function main() {
       rows = await conn.query(cap ? `SELECT * FROM \`${table}\` ORDER BY id DESC LIMIT ${cap}` : `SELECT * FROM \`${table}\``)
     }
     if (rows.length) {
-      const jsonCols = JSON_COLUMNS[table] || []
-      const bigCols = BIGINT_COLUMNS[table] || []
-      for (const r of rows) {
-        for (const c of jsonCols) r[c] = parseJson(r[c])
-        for (const c of bigCols) r[c] = toBigInt(r[c])
+      const adapted = rows.map((r) => {
         if (UUID_TABLES.has(table)) r.uuid = crypto.randomUUID()
-      }
+        return adaptRow(table, r)
+      })
       const chunk = 500
-      for (let i = 0; i < rows.length; i += chunk) {
-        await prisma[model].createMany({ data: rows.slice(i, i + chunk) })
+      for (let i = 0; i < adapted.length; i += chunk) {
+        await prisma[model].createMany({ data: adapted.slice(i, i + chunk) })
       }
     }
     counts[table] = rows.length