feat: 持久化从 JSON 迁移到 DuckDB，启动自动迁移存量数据并全面异步化

- 新增 src/persistence/duckdbAdapter：与 jsonAdapter 同接口的异步适配器，数据存于 data/registry.duckdb，首次启动自动从 JSON 迁移并备份 json-backup-<ts> - 全部服务层（project/config/user/token）与路由改为 async/await - MCP 工具与鉴权同步异步化，系统 mode 读写迁移至 DuckDB - 服务启动流程：init DB -> 自动迁移 -> 确保默认管理员 -> listen - 新增 DuckDB 往返/迁移测试，测试目录隔离 REGISTRY_DATA_DIR
2026-09-14 10:35:38 +08:00
parent 2ba09fa7a1
commit 6bbd4bc471
19 changed files with 958 additions and 302 deletions
@@ -0,0 +1,97 @@
+/**
+ * DuckDB 适配器单元测试
+ * 覆盖：写入/读取往返、JSON 异构值保留、JSON 存量自动迁移、重复初始化幂等
+ * 运行：npm test
+ */
+const test = require("node:test");
+const assert = require("node:assert");
+const fs = require("node:fs");
+const os = require("node:os");
+const path = require("node:path");
+
+// 使用临时数据目录，避免污染真实 data/
+const tmpDataDir = fs.mkdtempSync(path.join(os.tmpdir(), "registry-duckdb-test-"));
+process.env.REGISTRY_DATA_DIR = tmpDataDir;
+
+const {
+  init,
+  loadProjects,
+  saveProjects,
+  loadConfigs,
+  saveConfigs,
+  deleteConfigs,
+  loadSystem,
+  saveSystem,
+  loadMcpConfig,
+  saveMcpConfig,
+  loadUsers,
+  saveUsers,
+  loadApiTokens,
+  saveApiTokens,
+} = require("../src/persistence/duckdbAdapter");
+
+test("projects 写入/读取往返，字段无损保留", async () => {
+  await init();
+  const projects = [
+    { id: "p1", name: "项目A", enableIdc: true, nested: { arr: [1, 2, 3] } },
+    { id: "p2", name: "项目B", enableEnvironment: false },
+  ];
+  await saveProjects(projects);
+  const loaded = await loadProjects();
+  assert.strictEqual(loaded.length, 2);
+  assert.deepStrictEqual(loaded.find((p) => p.id === "p1").nested, { arr: [1, 2, 3] });
+});
+
+test("configs 写入/读取往返，值类型异构保留", async () => {
+  await init();
+  const configs = [
+    { id: "c1", key: "count", value: 42, type: "number" },
+    { id: "c2", key: "flag", value: false, type: "boolean" },
+    { id: "c3", key: "raw", value: { deep: ["a", null, true] }, type: "json" },
+  ];
+  await saveConfigs("proj-x", configs);
+  const loaded = await loadConfigs("proj-x");
+  assert.strictEqual(loaded.length, 3);
+  assert.strictEqual(loaded.find((c) => c.id === "c1").value, 42);
+  assert.strictEqual(loaded.find((c) => c.id === "c2").value, false);
+  assert.deepStrictEqual(loaded.find((c) => c.id === "c3").value, { deep: ["a", null, true] });
+  // ensure projectId 未被注入进记录本体
+  assert.strictEqual("projectId" in loaded[0], false);
+  // 其他项目不受影响
+  assert.deepStrictEqual(await loadConfigs("proj-other"), []);
+});
+
+test("configs 删除仅作用于指定项目", async () => {
+  await init();
+  await saveConfigs("proj-keep", [{ id: "k1", key: "a", value: 1 }]);
+  await deleteConfigs("proj-keep");
+  await saveConfigs("proj-keep2", [{ id: "k2", key: "b", value: 2 }]);
+  await deleteConfigs("proj-keep2");
+  await saveConfigs("proj-keep3", [{ id: "k3", key: "c", value: 3 }]);
+  await deleteConfigs("proj-keep3");
+  await saveConfigs("proj-del", [{ id: "d1", key: "x", value: 1 }]);
+  await deleteConfigs("proj-del");
+  await deleteConfigs("proj-del");
+  assert.deepStrictEqual(await loadConfigs("proj-del"), []);
+});
+
+test("system / mcp 写入读取与默认值", async () => {
+  await init();
+  const sys = await loadSystem();
+  assert.strictEqual(sys.mode, "edit");
+  await saveSystem({ mode: "readonly" });
+  assert.strictEqual((await loadSystem()).mode, "readonly");
+  const mcp = await loadMcpConfig();
+  assert.strictEqual(typeof mcp.token, "string");
+  await saveMcpConfig({ enabled: false, token: "abc" });
+  assert.deepStrictEqual(await loadMcpConfig(), { enabled: false, token: "abc" });
+});
+
+test("users / apiTokens 写入读取往返", async () => {
+  await init();
+  await saveUsers([{ id: "u1", username: "admin", password: "hashed", roles: ["admin"] }]);
+  const users = await loadUsers();
+  assert.strictEqual(users[0].username, "admin");
+  await saveApiTokens([{ id: "t1", name: "mcp", value: "secret" }]);
+  assert.strictEqual((await loadApiTokens())[0].value, "secret");
+});
@@ -0,0 +1,76 @@
+/**
+ * JSON -> DuckDB 自动迁移测试
+ * 独立进程运行：模块在 require 时读取 REGISTRY_DATA_DIR，需在 require 前注入种子数据目录
+ * 运行：npm test
+ */
+const test = require("node:test");
+const assert = require("node:assert");
+const fs = require("node:fs");
+const os = require("node:os");
+const path = require("node:path");
+
+// 构造一个含 JSON 存量数据的种子目录，并设置为数据目录
+const seedDir = path.join(os.tmpdir(), "registry-seed-" + Date.now());
+fs.mkdirSync(seedDir, { recursive: true });
+fs.writeFileSync(path.join(seedDir, "projects.json"), JSON.stringify([
+  { id: "proj-mig-1", name: "迁移项目A", enableIdc: false },
+]));
+fs.writeFileSync(path.join(seedDir, "users.json"), JSON.stringify([
+  { id: "u-mig", username: "migrator", displayName: "迁移用户", role: "user", enabled: true },
+]));
+fs.writeFileSync(path.join(seedDir, "api-tokens.json"), JSON.stringify([
+  { id: "t-mig", name: "迁移Token", value: "mig-secret", enabled: true },
+]));
+fs.writeFileSync(path.join(seedDir, "system.json"), JSON.stringify({ mode: "plan" }));
+fs.writeFileSync(path.join(seedDir, "mcp.json"), JSON.stringify({ enabled: true, token: "mcp-token" }));
+const configsDir = path.join(seedDir, "configs");
+fs.mkdirSync(configsDir, { recursive: true });
+fs.writeFileSync(path.join(configsDir, "proj-mig-1.json"), JSON.stringify([
+  { id: "cfg-mig", key: "db.host", value: "10.0.0.1", type: "string" },
+]));
+
+process.env.REGISTRY_DATA_DIR = seedDir;
+
+const duckdbAdapter = require("../src/persistence/duckdbAdapter");
+
+test("自动迁移全部 JSON 存量到 DuckDB，数据无损且备份 JSON", async () => {
+  await duckdbAdapter.init();
+
+  const projects = await duckdbAdapter.loadProjects();
+  assert.strictEqual(projects.length, 1);
+  assert.strictEqual(projects[0].name, "迁移项目A");
+  assert.strictEqual(projects[0].id, "proj-mig-1");
+
+  const users = await duckdbAdapter.loadUsers();
+  assert.strictEqual(users[0].username, "migrator");
+
+  const tokens = await duckdbAdapter.loadApiTokens();
+  assert.strictEqual(tokens[0].value, "mig-secret");
+
+  const system = await duckdbAdapter.loadSystem();
+  assert.strictEqual(system.mode, "plan");
+
+  const mcp = await duckdbAdapter.loadMcpConfig();
+  assert.strictEqual(mcp.token, "mcp-token");
+
+  const configs = await duckdbAdapter.loadConfigs("proj-mig-1");
+  assert.strictEqual(configs.length, 1);
+  assert.strictEqual(configs[0].value, "10.0.0.1");
+  // projectId 不应被注入进记录本体
+  assert.strictEqual("projectId" in configs[0], false);
+
+  // 数据迁移后应生成 DB 文件
+  assert.ok(fs.existsSync(path.join(seedDir, "registry.duckdb")), "应生成 registry.duckdb");
+
+  // JSON 源文件应被备份移除（防止二次导入）
+  assert.ok(!fs.existsSync(path.join(seedDir, "projects.json")), "迁移后 projects.json 应已移走");
+  const backupDirs = fs.readdirSync(seedDir).filter((n) => n.startsWith("json-backup-"));
+  assert.strictEqual(backupDirs.length, 1, "应生成 json-backup-<ts> 目录");
+});
+
+test("再次初始化（DB 已存在）不重复迁移、不报错", async () => {
+  const before = await duckdbAdapter.loadProjects();
+  await duckdbAdapter.init();
+  const after = await duckdbAdapter.loadProjects();
+  assert.deepStrictEqual(after, before);
+});
@@ -5,22 +5,32 @@
 */
 const test = require("node:test");
 const assert = require("node:assert");
+const fs = require("node:fs");
+const os = require("node:os");
+const path = require("node:path");
 const { createMcpServer } = require("../src/mcp/tools");
-const { loadSystem, saveSystem } = require("../src/persistence/jsonAdapter");
+const { init, loadSystem, saveSystem } = require("../src/persistence/duckdbAdapter");

-// 恢复测试前的系统模式，避免影响其他用例
-const originalMode = (loadSystem() || {}).mode || "edit";
-test.after(() => {
-  const system = loadSystem() || {};
-  system.mode = originalMode;
-  saveSystem(system);
-});
+// 使用临时数据目录，避免污染真实 data/
+const tmpDataDir = fs.mkdtempSync(path.join(os.tmpdir(), "registry-mcp-test-"));
+process.env.REGISTRY_DATA_DIR = tmpDataDir;

 let server;
-test.before(() => {
+let originalMode = "edit";
+
+// 初始化 DuckDB，记录测试前模式；测试结束后恢复原模式
+test.before(async () => {
+  await init();
+  originalMode = (await loadSystem() || {}).mode || "edit";
  server = createMcpServer();
 });

+test.after(async () => {
+  const system = await loadSystem() || {};
+  system.mode = originalMode;
+  await saveSystem(system);
+});
+
 /**
 * 直接调用已注册工具的处理函数
 * @param {string} name - 工具名