From 6bbd4bc471d0ce37e919bb3573ad42102bf41871 Mon Sep 17 00:00:00 2001 From: "jif2.zhang" Date: Mon, 14 Sep 2026 10:35:38 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=8C=81=E4=B9=85=E5=8C=96=E4=BB=8E=20?= =?UTF-8?q?JSON=20=E8=BF=81=E7=A7=BB=E5=88=B0=20DuckDB=EF=BC=8C=E5=90=AF?= =?UTF-8?q?=E5=8A=A8=E8=87=AA=E5=8A=A8=E8=BF=81=E7=A7=BB=E5=AD=98=E9=87=8F?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=B9=B6=E5=85=A8=E9=9D=A2=E5=BC=82=E6=AD=A5?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 src/persistence/duckdbAdapter：与 jsonAdapter 同接口的异步适配器， 数据存于 data/registry.duckdb，首次启动自动从 JSON 迁移并备份 json-backup- - 全部服务层（project/config/user/token）与路由改为 async/await - MCP 工具与鉴权同步异步化，系统 mode 读写迁移至 DuckDB - 服务启动流程：init DB -> 自动迁移 -> 确保默认管理员 -> listen - 新增 DuckDB 往返/迁移测试，测试目录隔离 REGISTRY_DATA_DIR --- .gitignore | 4 + package-lock.json | 154 ++++++++++++++++ package.json | 1 + src/mcp/index.js | 12 +- src/mcp/tools.js | 70 +++---- src/persistence/duckdbAdapter.js | 305 +++++++++++++++++++++++++++++++ src/routes/auth.js | 4 +- src/routes/configs.js | 52 +++--- src/routes/projects.js | 24 +-- src/routes/tokens.js | 20 +- src/routes/users.js | 20 +- src/server.js | 33 ++-- src/services/configService.js | 134 +++++++------- src/services/projectService.js | 110 +++++------ src/services/tokenService.js | 50 ++--- src/services/userService.js | 66 +++---- test/duckdbAdapter.test.js | 97 ++++++++++ test/duckdbMigration.test.js | 76 ++++++++ test/mcp.test.js | 28 ++- 19 files changed, 958 insertions(+), 302 deletions(-) create mode 100644 src/persistence/duckdbAdapter.js create mode 100644 test/duckdbAdapter.test.js create mode 100644 test/duckdbMigration.test.js diff --git a/.gitignore b/.gitignore index 913f013..3965958 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,9 @@ data/configs/*.json data/mcp.json data/users.json data/api-tokens.json +data/registry.duckdb +data/registry.duckdb.wal +data/registry.duckdb.tmp +data/json-backup-*/ *.log .reasonix/* \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 8cfc2b1..c3253f0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,10 +9,155 @@ "version": "1.0.0", "license": "MIT", "dependencies": { + "@duckdb/node-api": "^1.5.5-r.5", "@modelcontextprotocol/sdk": "^1.30.0", "express": "^4.18.2" } }, + "node_modules/@duckdb/node-api": { + "version": "1.5.5-r.5", + "resolved": "https://registry.npmjs.org/@duckdb/node-api/-/node-api-1.5.5-r.5.tgz", + "integrity": "sha512-ZMX+sel3DjSKbGuQ44Y5Tp5LwYjlJkhB5faKr+9sSMcxixDMSW+8vaTOD1uRDuN9o0udUoCY/XX9PI1haEZuZg==", + "license": "MIT", + "dependencies": { + "@duckdb/node-bindings": "1.5.5-r.5" + } + }, + "node_modules/@duckdb/node-bindings": { + "version": "1.5.5-r.5", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings/-/node-bindings-1.5.5-r.5.tgz", + "integrity": "sha512-sR7x91TfBunLqLbdvrcdiPCU+VGEQFPicxh1STZM402fuHW2ituPTAo2bY3IlntsvJShltrKF4R6B+JYPlAPZg==", + "license": "MIT", + "dependencies": { + "detect-libc": "^2.1.2" + }, + "optionalDependencies": { + "@duckdb/node-bindings-darwin-arm64": "1.5.5-r.5", + "@duckdb/node-bindings-darwin-x64": "1.5.5-r.5", + "@duckdb/node-bindings-linux-arm64": "1.5.5-r.5", + "@duckdb/node-bindings-linux-arm64-musl": "1.5.5-r.5", + "@duckdb/node-bindings-linux-x64": "1.5.5-r.5", + "@duckdb/node-bindings-linux-x64-musl": "1.5.5-r.5", + "@duckdb/node-bindings-win32-arm64": "1.5.5-r.5", + "@duckdb/node-bindings-win32-x64": "1.5.5-r.5" + } + }, + "node_modules/@duckdb/node-bindings-darwin-arm64": { + "version": "1.5.5-r.5", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-darwin-arm64/-/node-bindings-darwin-arm64-1.5.5-r.5.tgz", + "integrity": "sha512-WJdeLTFy4pRg60rkbp9smtlP1iIQ8ErtIJkxS+2LQtM1ZZMSByyW+3WPpNoKe7IwPtVl2fLlPw7tDmQW4Z974g==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@duckdb/node-bindings-darwin-x64": { + "version": "1.5.5-r.5", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-darwin-x64/-/node-bindings-darwin-x64-1.5.5-r.5.tgz", + "integrity": "sha512-rN0KjX13MwVQ5V5ufykolty/tf4SaafOpGWcUA1Cegf98jPeRdsJsEE1IMXn8n3u6WvMSZEVJmHJ3ma4Hr7Vzg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@duckdb/node-bindings-linux-arm64": { + "version": "1.5.5-r.5", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-linux-arm64/-/node-bindings-linux-arm64-1.5.5-r.5.tgz", + "integrity": "sha512-CafH2AIvXtpwDynG3KB/4uEOHRGlFoB7auCRMIwH3BKNIobsANV5LSpDJ3tfyRCFIsK6e8AT1eseWVnu2Dv3cw==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@duckdb/node-bindings-linux-arm64-musl": { + "version": "1.5.5-r.5", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-linux-arm64-musl/-/node-bindings-linux-arm64-musl-1.5.5-r.5.tgz", + "integrity": "sha512-00z3mYHUBAucp3L73EoftKDdaGOaV8ENELejiT+PK7dFjTzdXxqhJwZ1LaQV1/zEhIecz+x23f6WvFhWw+7XXQ==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@duckdb/node-bindings-linux-x64": { + "version": "1.5.5-r.5", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-linux-x64/-/node-bindings-linux-x64-1.5.5-r.5.tgz", + "integrity": "sha512-kCyq6BXhn6QXfwF8yVdVKriSSHP+rfVWcJ56cG8qZzlOZpxyvT5WGm6jMshxg4ditLlOsHsIN/D47RYrcjXuHQ==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@duckdb/node-bindings-linux-x64-musl": { + "version": "1.5.5-r.5", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-linux-x64-musl/-/node-bindings-linux-x64-musl-1.5.5-r.5.tgz", + "integrity": "sha512-S5qEc7F+GjXt85T2xArjjvN5oc/IjtH6zsLJsuo8N3fQolheBCefemIMdZoK2Jik8a0sBxidiS6Ld4qprMPPDw==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@duckdb/node-bindings-win32-arm64": { + "version": "1.5.5-r.5", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-win32-arm64/-/node-bindings-win32-arm64-1.5.5-r.5.tgz", + "integrity": "sha512-L3Mroc23xhqOowWVElni8LgEmsm5e9DdG1y2pLka/1ug3Pl2vpQMl1Phxwz4UEMF8DOBbMgN5BBNrYwG2BFo+g==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@duckdb/node-bindings-win32-x64": { + "version": "1.5.5-r.5", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-win32-x64/-/node-bindings-win32-x64-1.5.5-r.5.tgz", + "integrity": "sha512-mBGFn8netT/AAhK4TOG6Fw47+Db68CxCVtQ9qfxZc1kZAjds+aYrh17Fd/u5CbTDdxILsrcyuF8q9cUcm54kQA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@hono/node-server": { "version": "2.1.1", "resolved": "https://registry.npmmirror.com/@hono/node-server/-/node-server-2.1.1.tgz", @@ -628,6 +773,15 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmmirror.com/dunder-proto/-/dunder-proto-1.0.1.tgz", diff --git a/package.json b/package.json index 9f436c2..4d55e12 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ ], "license": "MIT", "dependencies": { + "@duckdb/node-api": "^1.5.5-r.5", "@modelcontextprotocol/sdk": "^1.30.0", "express": "^4.18.2" } diff --git a/src/mcp/index.js b/src/mcp/index.js index 943810c..cfb2b5b 100644 --- a/src/mcp/index.js +++ b/src/mcp/index.js @@ -11,7 +11,7 @@ const { } = require("@modelcontextprotocol/sdk/server/streamableHttp.js"); const { isInitializeRequest } = require("@modelcontextprotocol/sdk/types.js"); const { createMcpServer } = require("./tools"); -const { loadMcpConfig } = require("../persistence/jsonAdapter"); +const { loadMcpConfig } = require("../persistence/duckdbAdapter"); const router = express.Router(); @@ -21,10 +21,10 @@ const transports = new Map(); /** * 校验请求携带的 Bearer token 是否与配置中的 token 匹配 * @param {string} [header] - Authorization 请求头 - * @returns {boolean} 是否鉴权通过 + * @returns {Promise} 是否鉴权通过 */ -function isAuthorized(header) { - const mcp = loadMcpConfig(); +async function isAuthorized(header) { + const mcp = await loadMcpConfig(); // 未配置 token 时视为未启用鉴权 if (!mcp.token) { return true; @@ -43,8 +43,8 @@ function isAuthorized(header) { /** * 鉴权中间件：token 校验失败返回 401 */ -router.use((req, res, next) => { - if (!isAuthorized(req.headers.authorization)) { +router.use(async (req, res, next) => { + if (!(await isAuthorized(req.headers.authorization))) { return res .status(401) .json({ jsonrpc: "2.0", error: { code: -32001, message: "Unauthorized: invalid or missing token" }, id: null }); diff --git a/src/mcp/tools.js b/src/mcp/tools.js index 3c0616b..11a94b8 100644 --- a/src/mcp/tools.js +++ b/src/mcp/tools.js @@ -11,7 +11,7 @@ const z = require("zod/v4"); const { McpServer } = require("@modelcontextprotocol/sdk/server/mcp.js"); const projectService = require("../services/projectService"); const configService = require("../services/configService"); -const { loadSystem, loadMcpConfig } = require("../persistence/jsonAdapter"); +const { loadSystem, loadMcpConfig } = require("../persistence/duckdbAdapter"); // 待确认删除的登记表：token -> { type, projectId, configId, expiresAt } const pendingDeletions = new Map(); @@ -20,11 +20,11 @@ const PENDING_TTL = 5 * 60 * 1000; /** * 获取当前系统模式 - * @returns {string} plan/edit/readonly + * @returns {Promise} plan/edit/readonly */ -function currentMode() { +async function currentMode() { try { - return (loadSystem() || {}).mode || "edit"; + return (await loadSystem() || {}).mode || "edit"; } catch { return "edit"; } @@ -34,8 +34,8 @@ function currentMode() { * 只读模式下拒绝写操作 * @throws {Error} 只读模式 */ -function assertWritable() { - if (currentMode() === "readonly") { +async function assertWritable() { + if ((await await currentMode()) === "readonly") { throw new Error("系统当前为只读模式，不允许写操作"); } } @@ -145,8 +145,8 @@ function createMcpServer() { inputSchema: {}, }, async () => ({ - content: [{ type: "text", text: `当前模式：${currentMode()}` }], - structuredContent: { mode: currentMode() }, + content: [{ type: "text", text: `当前模式：${await currentMode()}` }], + structuredContent: { mode: await currentMode() }, }) ); @@ -159,10 +159,10 @@ function createMcpServer() { }, }, async ({ mode }) => { - const { loadSystem, saveSystem } = require("../persistence/jsonAdapter"); - const system = loadSystem() || {}; + const { loadSystem, saveSystem } = require("../persistence/duckdbAdapter"); + const system = await loadSystem() || {}; system.mode = mode; - saveSystem(system); + await saveSystem(system); return { content: [{ type: "text", text: `模式已切换为：${mode}` }], structuredContent: { mode }, @@ -180,9 +180,9 @@ function createMcpServer() { }, async () => ({ content: [ - { type: "text", text: JSON.stringify(projectService.getAllProjects(), null, 2) }, + { type: "text", text: JSON.stringify(await projectService.getAllProjects(), null, 2) }, ], - structuredContent: { projects: projectService.getAllProjects() }, + structuredContent: { projects: await projectService.getAllProjects() }, }) ); @@ -195,7 +195,7 @@ function createMcpServer() { }, }, async ({ projectId }) => { - const project = projectService.getProjectById(projectId); + const project = await projectService.getProjectById(projectId); if (!project) { throw new Error(`项目不存在：${projectId}`); } @@ -223,8 +223,8 @@ function createMcpServer() { }, }, async (args) => { - assertWritable(); - const project = projectService.createProject(args); + await assertWritable(); + const project = await projectService.createProject(args); return { content: [{ type: "text", text: JSON.stringify(project, null, 2) }], structuredContent: { project }, @@ -244,8 +244,8 @@ function createMcpServer() { }, }, async ({ projectId, idcMapping, environmentMapping, groupMapping }) => { - assertWritable(); - const project = projectService.updateMappings(projectId, { + await assertWritable(); + const project = await projectService.updateMappings(projectId, { idcMapping, environmentMapping, groupMapping, @@ -267,8 +267,8 @@ function createMcpServer() { }, }, async ({ projectId }) => { - assertWritable(); - const project = projectService.getProjectById(projectId); + await assertWritable(); + const project = await projectService.getProjectById(projectId); if (!project) { throw new Error(`项目不存在：${projectId}`); } @@ -291,7 +291,7 @@ function createMcpServer() { }, }, async ({ confirmation_token }) => { - assertWritable(); + await assertWritable(); const entry = takePendingDeletion(confirmation_token); if (!entry) { return { @@ -309,7 +309,7 @@ function createMcpServer() { isError: true, }; } - const ok = projectService.deleteProject(entry.projectId); + const ok = await projectService.deleteProject(entry.projectId); if (!ok) { return { content: [{ type: "text", text: `项目不存在或已删除：${entry.projectId}` }], @@ -335,7 +335,7 @@ function createMcpServer() { }, }, async ({ projectId }) => { - const configs = configService.getConfigsByProject(projectId); + const configs = await configService.getConfigsByProject(projectId); return { content: [{ type: "text", text: JSON.stringify(configs, null, 2) }], structuredContent: { configs }, @@ -353,7 +353,7 @@ function createMcpServer() { }, }, async ({ projectId, configId }) => { - const config = configService.getConfigById(projectId, configId); + const config = await configService.getConfigById(projectId, configId); if (!config) { throw new Error(`配置项不存在：${configId}`); } @@ -386,10 +386,10 @@ function createMcpServer() { }, }, async (args) => { - assertWritable(); + await assertWritable(); const { projectId, ...data } = args; - const mode = currentMode(); - const config = configService.createConfig(projectId, data, mode); + const mode = await currentMode(); + const config = await configService.createConfig(projectId, data, mode); return { content: [{ type: "text", text: JSON.stringify(config, null, 2) }], structuredContent: { config }, @@ -420,7 +420,7 @@ function createMcpServer() { }, }, async (args) => { - assertWritable(); + await assertWritable(); const { projectId, configId, ...data } = args; const picks = {}; for (const field of [ @@ -431,8 +431,8 @@ function createMcpServer() { picks[field] = data[field]; } } - const mode = currentMode(); - const config = configService.updateConfig(projectId, configId, picks, mode); + const mode = await currentMode(); + const config = await configService.updateConfig(projectId, configId, picks, mode); return { content: [{ type: "text", text: JSON.stringify(config, null, 2) }], structuredContent: { config }, @@ -451,8 +451,8 @@ function createMcpServer() { }, }, async ({ projectId, configId }) => { - assertWritable(); - const config = configService.getConfigById(projectId, configId); + await assertWritable(); + const config = await configService.getConfigById(projectId, configId); if (!config) { throw new Error(`配置项不存在：${configId}`); } @@ -475,7 +475,7 @@ function createMcpServer() { }, }, async ({ confirmation_token }) => { - assertWritable(); + await assertWritable(); const entry = takePendingDeletion(confirmation_token); if (!entry) { return { @@ -493,8 +493,8 @@ function createMcpServer() { isError: true, }; } - const mode = currentMode(); - const ok = configService.deleteConfig(entry.projectId, entry.configId, mode); + const mode = await currentMode(); + const ok = await configService.deleteConfig(entry.projectId, entry.configId, mode); if (!ok) { return { content: [{ type: "text", text: `配置项不存在或已删除：${entry.configId}` }], diff --git a/src/persistence/duckdbAdapter.js b/src/persistence/duckdbAdapter.js new file mode 100644 index 0000000..635ba12 --- /dev/null +++ b/src/persistence/duckdbAdapter.js @@ -0,0 +1,305 @@ +/** + * DuckDB 持久化适配器 + * 与 jsonAdapter 暴露相同的异步接口，数据存储在 data/registry.duckdb + * 首次启动（DB 文件不存在）时自动从既有 JSON 文件迁移存量数据，并备份 JSON 为 *.bak + */ +const fs = require("fs"); +const path = require("path"); +const { DuckDBInstance } = require("@duckdb/node-api"); + +// 数据存储根目录（支持 REGISTRY_DATA_DIR 覆盖，测试隔离用） +const DATA_DIR = process.env.REGISTRY_DATA_DIR || path.join(__dirname, "..", "..", "data"); +const DB_FILE = path.join(DATA_DIR, "registry.duckdb"); + +const JSON_FILES = { + projects: "projects.json", + system: "system.json", + mcp: "mcp.json", + users: "users.json", + apiTokens: "api-tokens.json", + configs: null, // 配置项存于 configs/.json +}; + +let connection = null; + +/** + * 确保数据目录存在 + */ +function ensureDataDir() { + if (!fs.existsSync(DATA_DIR)) { + fs.mkdirSync(DATA_DIR, { recursive: true }); + } +} + +/** + * 创建表结构（幂等） + * 使用 JSON 列存储完整记录，value 异构类型无损保留 + */ +async function createTables() { + await connection.run(` + CREATE TABLE IF NOT EXISTS projects ( + id VARCHAR PRIMARY KEY, + data JSON NOT NULL + ); + CREATE TABLE IF NOT EXISTS configs ( + project_id VARCHAR, + id VARCHAR, + data JSON NOT NULL, + PRIMARY KEY (project_id, id) + ); + CREATE TABLE IF NOT EXISTS users ( + id VARCHAR PRIMARY KEY, + data JSON NOT NULL + ); + CREATE TABLE IF NOT EXISTS api_tokens ( + id VARCHAR PRIMARY KEY, + data JSON NOT NULL + ); + CREATE TABLE IF NOT EXISTS system_kv ( + key VARCHAR PRIMARY KEY, + data JSON NOT NULL + ); + CREATE TABLE IF NOT EXISTS mcp_kv ( + key VARCHAR PRIMARY KEY, + data JSON NOT NULL + ); + `); +} + +/** + * 读取 JSON 文件，返回解析后对象或 null + */ +function readJsonFile(filePath) { + if (!fs.existsSync(filePath)) return null; + const content = fs.readFileSync(filePath, "utf-8"); + return JSON.parse(content); +} + +/** + * 读取 JSON 外层数组/对象并包裹为标准 JSON 字符串（供入库） + */ +function jsonStringifyList(list) { + return JSON.stringify(list); +} + +async function insertRows(table, rows) { + if (!rows || rows.length === 0) return; + for (const r of rows) { + await connection.run(`INSERT INTO ${table} (id, data) VALUES (?, ?)`, [r.id, JSON.stringify(r)]); + } +} + +/** + * 解析查询结果中的 JSON 列（DuckDB 可能以字符串或对象形式返回） + */ +function parseData(data) { + return typeof data === "string" ? JSON.parse(data) : data; +} + +/** + * 从 JSON 文件迁移指定数据到 DuckDB + */ +async function migrateCollection(name, filePath, table, listToRows) { + if (!fs.existsSync(filePath)) return 0; + const raw = readJsonFile(filePath); + const rows = listToRows ? listToRows(raw) : raw; + await insertRows(table, rows); + return Array.isArray(rows) ? rows.length : 1; +} + +/** + * 一键迁移所有 JSON 存量数据 + * 迁移成功后把 JSON 改为 *.bak 备份，避免二次导入 + */ +async function migrateAllFromJson() { + console.log("[duckdb] 首次启动，从 JSON 迁移存量数据..."); + const results = {}; + + results.projects = await migrateCollection("projects", path.join(DATA_DIR, "projects.json"), "projects", (list) => list || []); + results.users = await migrateCollection("users", path.join(DATA_DIR, "users.json"), "users", (list) => list || []); + results.apiTokens = await migrateCollection("apiTokens", path.join(DATA_DIR, "api-tokens.json"), "api_tokens", (list) => list || []); + + // 系统配置：system.json -> system_kv (key='system') + const system = readJsonFile(path.join(DATA_DIR, "system.json")); + if (system) { + await connection.run(`INSERT INTO system_kv (key, data) VALUES ('system', ?)`, [JSON.stringify(system)]); + results.system = 1; + } + + // MCP 配置：mcp.json -> mcp_kv (key='mcp') + const mcp = readJsonFile(path.join(DATA_DIR, "mcp.json")); + if (mcp) { + await connection.run(`INSERT INTO mcp_kv (key, data) VALUES ('mcp', ?)`, [JSON.stringify(mcp)]); + results.mcp = 1; + } + + // 配置项：configs/.json -> configs + const configsDir = path.join(DATA_DIR, "configs"); + let configCount = 0; + if (fs.existsSync(configsDir)) { + for (const file of fs.readdirSync(configsDir)) { + if (!file.endsWith(".json")) continue; + const projectId = file.replace(/\.json$/, ""); + const configs = readJsonFile(path.join(configsDir, file)) || []; + for (const cfg of configs) { + await connection.run(`INSERT INTO configs (project_id, id, data) VALUES (?, ?, ?)`, [projectId, cfg.id, JSON.stringify(cfg)]); + configCount++; + } + } + } + results.configs = configCount; + + console.log("[duckdb] 迁移完成：", JSON.stringify(results)); + + // 迁移成功后备份 JSON（保留合理回滚余地）：统一改为 data/json-backup-/ + if (fs.existsSync(path.join(DATA_DIR, "projects.json")) || + fs.existsSync(path.join(DATA_DIR, "users.json")) || + fs.existsSync(path.join(DATA_DIR, "system.json"))) { + const backupDir = path.join(DATA_DIR, "json-backup-" + Date.now()); + fs.mkdirSync(backupDir, { recursive: true }); + for (const name of ["projects.json", "system.json", "mcp.json", "users.json", "api-tokens.json"]) { + const src = path.join(DATA_DIR, name); + if (fs.existsSync(src)) fs.renameSync(src, path.join(backupDir, name)); + } + const cfgSrc = path.join(DATA_DIR, "configs"); + if (fs.existsSync(cfgSrc)) { + const files = fs.readdirSync(cfgSrc); + if (files.length) { + fs.mkdirSync(path.join(backupDir, "configs"), { recursive: true }); + for (const f of files) fs.renameSync(path.join(cfgSrc, f), path.join(backupDir, "configs", f)); + } + } + console.log(`[duckdb] JSON 已备份至 ${backupDir}`); + } +} + +/** + * 初始化 DuckDB：建库建表，必要时自动迁移 + */ +async function init() { + if (connection) return; + ensureDataDir(); + const dbExists = fs.existsSync(DB_FILE); + const instance = await DuckDBInstance.create(DB_FILE); + connection = await instance.connect(); + await createTables(); + if (!dbExists) { + await migrateAllFromJson(); + } + return connection; +} + +// ==================== projects ==================== + +async function loadProjects() { + await init(); + const reader = await connection.runAndReadAll("SELECT data FROM projects"); + return reader.getRowObjects().map((r) => parseData(r.data)); +} + +async function saveProjects(projects) { + await init(); + await connection.run("DELETE FROM projects"); + await insertRows("projects", projects); +} + +// ==================== configs ==================== + +async function loadConfigs(projectId) { + await init(); + const reader = await connection.runAndReadAll("SELECT data FROM configs WHERE project_id = ?", [projectId]); + return reader.getRowObjects().map((r) => parseData(r.data)); +} + +async function saveConfigs(projectId, configs) { + await init(); + await connection.run("DELETE FROM configs WHERE project_id = ?", [projectId]); + for (const c of configs || []) { + await connection.run( + `INSERT INTO configs (project_id, id, data) VALUES (?, ?, ?)`, + [projectId, c.id, JSON.stringify(c)] + ); + } +} + +async function deleteConfigs(projectId) { + await init(); + await connection.run("DELETE FROM configs WHERE project_id = ?", [projectId]); +} + +// ==================== system / mcp ==================== + +async function loadSystem() { + await init(); + const reader = await connection.runAndReadAll("SELECT data FROM system_kv WHERE key = 'system'"); + const row = reader.getRowObjects()[0]; + return row ? parseData(row.data) : { mode: "edit" }; +} + +async function saveSystem(system) { + await init(); + await connection.run( + `INSERT INTO system_kv (key, data) VALUES ('system', ?) + ON CONFLICT (key) DO UPDATE SET data = excluded.data`, + [JSON.stringify(system)] + ); +} + +async function loadMcpConfig() { + await init(); + const reader = await connection.runAndReadAll("SELECT data FROM mcp_kv WHERE key = 'mcp'"); + const row = reader.getRowObjects()[0]; + return row ? parseData(row.data) : { enabled: true, token: "" }; +} + +async function saveMcpConfig(mcp) { + await init(); + await connection.run( + `INSERT INTO mcp_kv (key, data) VALUES ('mcp', ?) + ON CONFLICT (key) DO UPDATE SET data = excluded.data`, + [JSON.stringify(mcp)] + ); +} + +// ==================== users / api tokens ==================== + +async function loadUsers() { + await init(); + const reader = await connection.runAndReadAll("SELECT data FROM users"); + return reader.getRowObjects().map((r) => parseData(r.data)); +} + +async function saveUsers(users) { + await init(); + await connection.run("DELETE FROM users"); + await insertRows("users", users); +} + +async function loadApiTokens() { + await init(); + const reader = await connection.runAndReadAll("SELECT data FROM api_tokens"); + return reader.getRowObjects().map((r) => parseData(r.data)); +} + +async function saveApiTokens(tokens) { + await init(); + await connection.run("DELETE FROM api_tokens"); + await insertRows("api_tokens", tokens); +} + +module.exports = { + init, + loadProjects, + saveProjects, + loadConfigs, + saveConfigs, + deleteConfigs, + loadSystem, + saveSystem, + loadMcpConfig, + saveMcpConfig, + loadUsers, + saveUsers, + loadApiTokens, + saveApiTokens, +}; \ No newline at end of file diff --git a/src/routes/auth.js b/src/routes/auth.js index 399dd2d..f82d0d8 100644 --- a/src/routes/auth.js +++ b/src/routes/auth.js @@ -63,13 +63,13 @@ function destroySession(token) { * POST /api/auth/login * Body: { username, password } */ -router.post("/login", (req, res) => { +router.post("/login", async (req, res) => { try { const { username, password } = req.body; if (!username || !password) { return res.status(400).json({ success: false, message: "用户名和密码不能为空" }); } - const user = userService.authenticate(username, password); + const user = await userService.authenticate(username, password); if (!user) { return res.status(401).json({ success: false, message: "用户名或密码错误" }); } diff --git a/src/routes/configs.js b/src/routes/configs.js index b10335e..213c506 100644 --- a/src/routes/configs.js +++ b/src/routes/configs.js @@ -5,28 +5,28 @@ const express = require("express"); const router = express.Router({ mergeParams: true }); const configService = require("../services/configService"); -const { loadSystem, saveSystem } = require("../persistence/jsonAdapter"); - -/** - * 获取当前模式 - * @returns {string} 当前模式 - */ -function getCurrentMode() { - try { - const system = loadSystem(); - return system.mode || "edit"; - } catch { - return "edit"; - } +const { loadSystem, saveSystem } = require("../persistence/duckdbAdapter"); + +/** + * 获取当前模式 + * @returns {Promise} 当前模式 + */ +async function getCurrentMode() { + try { + const system = await loadSystem(); + return system.mode || "edit"; + } catch { + return "edit"; + } } /** * GET /api/projects/:projectId/configs * 获取项目下所有配置项 */ -router.get("/", (req, res) => { +router.get("/", async (req, res) => { try { - const configs = configService.getConfigsByProject(req.params.projectId); + const configs = await configService.getConfigsByProject(req.params.projectId); res.json({ success: true, data: configs }); } catch (err) { res.status(400).json({ success: false, message: err.message }); @@ -37,9 +37,9 @@ router.get("/", (req, res) => { * GET /api/projects/:projectId/configs/:configId * 获取单个配置项 */ -router.get("/:configId", (req, res) => { +router.get("/:configId", async (req, res) => { try { - const config = configService.getConfigById(req.params.projectId, req.params.configId); + const config = await configService.getConfigById(req.params.projectId, req.params.configId); if (!config) { return res.status(404).json({ success: false, message: "配置项不存在" }); } @@ -53,10 +53,10 @@ router.get("/:configId", (req, res) => { * POST /api/projects/:projectId/configs * 新增配置项 */ -router.post("/", (req, res) => { +router.post("/", async (req, res) => { try { - const mode = getCurrentMode(); - const config = configService.createConfig(req.params.projectId, req.body, mode); + const mode = await getCurrentMode(); + const config = await configService.createConfig(req.params.projectId, req.body, mode); res.status(201).json({ success: true, data: config }); } catch (err) { res.status(400).json({ success: false, message: err.message }); @@ -67,10 +67,10 @@ router.post("/", (req, res) => { * PUT /api/projects/:projectId/configs/:configId * 修改配置项 */ -router.put("/:configId", (req, res) => { +router.put("/:configId", async (req, res) => { try { - const mode = getCurrentMode(); - const config = configService.updateConfig(req.params.projectId, req.params.configId, req.body, mode); + const mode = await getCurrentMode(); + const config = await configService.updateConfig(req.params.projectId, req.params.configId, req.body, mode); res.json({ success: true, data: config }); } catch (err) { res.status(400).json({ success: false, message: err.message }); @@ -81,10 +81,10 @@ router.put("/:configId", (req, res) => { * DELETE /api/projects/:projectId/configs/:configId * 删除配置项 */ -router.delete("/:configId", (req, res) => { +router.delete("/:configId", async (req, res) => { try { - const mode = getCurrentMode(); - const result = configService.deleteConfig(req.params.projectId, req.params.configId, mode); + const mode = await getCurrentMode(); + const result = await configService.deleteConfig(req.params.projectId, req.params.configId, mode); if (!result) { return res.status(404).json({ success: false, message: "配置项不存在" }); } diff --git a/src/routes/projects.js b/src/routes/projects.js index 075f7c0..7e08ac0 100644 --- a/src/routes/projects.js +++ b/src/routes/projects.js @@ -10,9 +10,9 @@ const projectService = require("../services/projectService"); * GET /api/projects * 获取所有项目列表 */ -router.get("/", (req, res) => { +router.get("/", async (req, res) => { try { - const projects = projectService.getAllProjects(); + const projects = await projectService.getAllProjects(); res.json({ success: true, data: projects }); } catch (err) { res.status(500).json({ success: false, message: err.message }); @@ -23,9 +23,9 @@ router.get("/", (req, res) => { * GET /api/projects/:id * 获取单个项目详情 */ -router.get("/:id", (req, res) => { +router.get("/:id", async (req, res) => { try { - const project = projectService.getProjectById(req.params.id); + const project = await projectService.getProjectById(req.params.id); if (!project) { return res.status(404).json({ success: false, message: "项目不存在" }); } @@ -40,9 +40,9 @@ router.get("/:id", (req, res) => { * 创建新项目 * Body: { name, description, enableIdc, enableEnvironment, enableGroup, idcMapping, environmentMapping, groupMapping } */ -router.post("/", (req, res) => { +router.post("/", async (req, res) => { try { - const project = projectService.createProject(req.body); + const project = await projectService.createProject(req.body); res.status(201).json({ success: true, data: project }); } catch (err) { res.status(400).json({ success: false, message: err.message }); @@ -54,9 +54,9 @@ router.post("/", (req, res) => { * 更新项目基本信息（重命名/修改描述） * Body: { name, description } */ -router.put("/:id", (req, res) => { +router.put("/:id", async (req, res) => { try { - const project = projectService.updateProject(req.params.id, req.body); + const project = await projectService.updateProject(req.params.id, req.body); res.json({ success: true, data: project }); } catch (err) { res.status(400).json({ success: false, message: err.message }); @@ -68,9 +68,9 @@ router.put("/:id", (req, res) => { * 更新项目的维度映射方式（path/label） * Body: { idcMapping, environmentMapping, groupMapping } */ -router.put("/:id/mappings", (req, res) => { +router.put("/:id/mappings", async (req, res) => { try { - const project = projectService.updateMappings(req.params.id, req.body); + const project = await projectService.updateMappings(req.params.id, req.body); res.json({ success: true, data: project }); } catch (err) { res.status(400).json({ success: false, message: err.message }); @@ -81,9 +81,9 @@ router.put("/:id/mappings", (req, res) => { * DELETE /api/projects/:id * 删除项目及其所有配置 */ -router.delete("/:id", (req, res) => { +router.delete("/:id", async (req, res) => { try { - const result = projectService.deleteProject(req.params.id); + const result = await projectService.deleteProject(req.params.id); if (!result) { return res.status(404).json({ success: false, message: "项目不存在" }); } diff --git a/src/routes/tokens.js b/src/routes/tokens.js index f0126df..234d4ad 100644 --- a/src/routes/tokens.js +++ b/src/routes/tokens.js @@ -25,9 +25,9 @@ router.use(requireAdmin); /** * GET /api/tokens */ -router.get("/", (req, res) => { +router.get("/", async (req, res) => { try { - res.json({ success: true, data: tokenService.getAllTokens() }); + res.json({ success: true, data: await tokenService.getAllTokens() }); } catch (err) { res.status(500).json({ success: false, message: err.message }); } @@ -37,9 +37,9 @@ router.get("/", (req, res) => { * POST /api/tokens * Body: { name, projectId, permission, expiresInDays } */ -router.post("/", (req, res) => { +router.post("/", async (req, res) => { try { - const token = tokenService.createToken(req.body); + const token = await tokenService.createToken(req.body); res.status(201).json({ success: true, data: token }); } catch (err) { res.status(400).json({ success: false, message: err.message }); @@ -50,9 +50,9 @@ router.post("/", (req, res) => { * PUT /api/tokens/:id * Body: { name, projectId, permission, enabled } */ -router.put("/:id", (req, res) => { +router.put("/:id", async (req, res) => { try { - const token = tokenService.updateToken(req.params.id, req.body); + const token = await tokenService.updateToken(req.params.id, req.body); res.json({ success: true, data: token }); } catch (err) { res.status(400).json({ success: false, message: err.message }); @@ -62,9 +62,9 @@ router.put("/:id", (req, res) => { /** * PUT /api/tokens/:id/revoke */ -router.put("/:id/revoke", (req, res) => { +router.put("/:id/revoke", async (req, res) => { try { - const ok = tokenService.revokeToken(req.params.id); + const ok = await tokenService.revokeToken(req.params.id); if (!ok) return res.status(404).json({ success: false, message: "Token 不存在" }); res.json({ success: true, message: "Token 已吊销" }); } catch (err) { @@ -75,9 +75,9 @@ router.put("/:id/revoke", (req, res) => { /** * DELETE /api/tokens/:id */ -router.delete("/:id", (req, res) => { +router.delete("/:id", async (req, res) => { try { - const ok = tokenService.deleteToken(req.params.id); + const ok = await tokenService.deleteToken(req.params.id); if (!ok) return res.status(404).json({ success: false, message: "Token 不存在" }); res.json({ success: true, message: "Token 已删除" }); } catch (err) { diff --git a/src/routes/users.js b/src/routes/users.js index 8149917..a777358 100644 --- a/src/routes/users.js +++ b/src/routes/users.js @@ -26,9 +26,9 @@ router.use(requireAdmin); /** * GET /api/users */ -router.get("/", (req, res) => { +router.get("/", async (req, res) => { try { - res.json({ success: true, data: userService.getAllUsers() }); + res.json({ success: true, data: await userService.getAllUsers() }); } catch (err) { res.status(500).json({ success: false, message: err.message }); } @@ -38,9 +38,9 @@ router.get("/", (req, res) => { * POST /api/users * Body: { username, displayName, password, role, enabled } */ -router.post("/", (req, res) => { +router.post("/", async (req, res) => { try { - const user = userService.createUser(req.body); + const user = await userService.createUser(req.body); res.status(201).json({ success: true, data: user }); } catch (err) { res.status(400).json({ success: false, message: err.message }); @@ -51,9 +51,9 @@ router.post("/", (req, res) => { * PUT /api/users/:id * Body: { displayName, role, enabled } */ -router.put("/:id", (req, res) => { +router.put("/:id", async (req, res) => { try { - const user = userService.updateUser(req.params.id, req.body); + const user = await userService.updateUser(req.params.id, req.body); res.json({ success: true, data: user }); } catch (err) { res.status(400).json({ success: false, message: err.message }); @@ -64,9 +64,9 @@ router.put("/:id", (req, res) => { * PUT /api/users/:id/password * Body: { password } */ -router.put("/:id/password", (req, res) => { +router.put("/:id/password", async (req, res) => { try { - userService.resetPassword(req.params.id, req.body.password); + await userService.resetPassword(req.params.id, req.body.password); res.json({ success: true, message: "密码已重置" }); } catch (err) { res.status(400).json({ success: false, message: err.message }); @@ -76,9 +76,9 @@ router.put("/:id/password", (req, res) => { /** * DELETE /api/users/:id */ -router.delete("/:id", (req, res) => { +router.delete("/:id", async (req, res) => { try { - const ok = userService.deleteUser(req.params.id); + const ok = await userService.deleteUser(req.params.id); if (!ok) return res.status(404).json({ success: false, message: "用户不存在" }); res.json({ success: true, message: "用户已删除" }); } catch (err) { diff --git a/src/server.js b/src/server.js index 3d88710..73399c8 100644 --- a/src/server.js +++ b/src/server.js @@ -9,7 +9,8 @@ const configRoutes = require("./routes/configs"); const { router: authRoutes, getSession, extractToken } = require("./routes/auth"); const userRoutes = require("./routes/users"); const tokenRoutes = require("./routes/tokens"); -const { loadSystem, saveSystem } = require("./persistence/jsonAdapter"); +const { loadSystem, saveSystem } = require("./persistence/duckdbAdapter"); +const duckdbAdapter = require("./persistence/duckdbAdapter"); const userService = require("./services/userService"); const versionInfo = require("./version"); @@ -22,9 +23,6 @@ const PORT = process.env.PORT || 3000; // 解析 JSON 请求体 app.use(express.json()); -// 初始化默认管理员账号（首次启动时自动创建） -userService.ensureDefaultAdmin(); - // 全局认证中间件：解析 Bearer token，将用户信息挂载到 req.user（非强制） app.use((req, res, next) => { const token = extractToken(req); @@ -66,9 +64,9 @@ app.get("/api/version", (req, res) => { * GET /api/mode * 获取当前系统模式 */ -app.get("/api/mode", (req, res) => { +app.get("/api/mode", async (req, res) => { try { - const system = loadSystem(); + const system = await loadSystem(); res.json({ success: true, data: { mode: system.mode || "edit" } }); } catch (err) { res.status(500).json({ success: false, message: err.message }); @@ -80,23 +78,34 @@ app.get("/api/mode", (req, res) => { * 切换系统模式 * Body: { mode: "plan" | "edit" | "readonly" } */ -app.put("/api/mode", (req, res) => { +app.put("/api/mode", async (req, res) => { try { const { mode } = req.body; const validModes = ["plan", "edit", "readonly"]; if (!validModes.includes(mode)) { return res.status(400).json({ success: false, message: `无效的模式: ${mode}，支持: ${validModes.join(", ")}` }); } - const system = loadSystem(); + const system = await loadSystem(); system.mode = mode; - saveSystem(system); + await saveSystem(system); res.json({ success: true, data: { mode } }); } catch (err) { res.status(500).json({ success: false, message: err.message }); } }); -// 启动服务 -app.listen(PORT, () => { - console.log(`Registry 服务已启动: http://localhost:${PORT}`); +/** + * 启动服务：先初始化 DuckDB（自动迁移 JSON 存量数据），再创建默认管理员账号 + */ +async function start() { + await duckdbAdapter.init(); + await userService.ensureDefaultAdmin(); + app.listen(PORT, () => { + console.log(`Registry 服务已启动: http://localhost:${PORT}`); + }); +} + +start().catch((err) => { + console.error("服务启动失败:", err); + process.exit(1); }); diff --git a/src/services/configService.js b/src/services/configService.js index 4f66fa7..779ee04 100644 --- a/src/services/configService.js +++ b/src/services/configService.js @@ -4,7 +4,7 @@ * 路径规则：仅映射方式为 path 的已启用维度作为路径前缀，label 维度仅作筛选标签不进入路径； * 配置键 key 自身可用 / 嵌套，json 类型在前端按 . 分割展示 */ -const { loadConfigs, saveConfigs } = require("../persistence/jsonAdapter"); +const { loadConfigs, saveConfigs } = require("../persistence/duckdbAdapter"); const { getProjectById } = require("./projectService"); const { validateByType, validateRange, VALID_TYPES } = require("../utils/validator"); @@ -57,32 +57,32 @@ function buildConfigPath(project, config) { /** * 获取项目下所有配置项，自动填充路径 * @param {string} projectId - 项目 ID - * @returns {Array} 配置项列表（含 path 字段） - */ -function getConfigsByProject(projectId) { - const project = getProjectById(projectId); - if (!project) { - throw new Error("项目不存在"); - } - const configs = loadConfigs(projectId); - return configs.map((c) => ({ - ...c, - path: buildConfigPath(project, c), - })); +* @returns {Promise} 配置项列表（含 path 字段） + */ +async function getConfigsByProject(projectId) { + const project = await getProjectById(projectId); + if (!project) { + throw new Error("项目不存在"); + } + const configs = await loadConfigs(projectId); + return configs.map((c) => ({ + ...c, + path: buildConfigPath(project, c), + })); } /** * 根据 ID 获取单个配置项 * @param {string} projectId - 项目 ID * @param {string} configId - 配置项 ID - * @returns {Object|null} 配置项对象，不存在时返回 null - */ -function getConfigById(projectId, configId) { - const project = getProjectById(projectId); - if (!project) { - throw new Error("项目不存在"); - } - const configs = loadConfigs(projectId); +* @returns {Promise} 配置项对象，不存在时返回 null + */ +async function getConfigById(projectId, configId) { + const project = await getProjectById(projectId); + if (!project) { + throw new Error("项目不存在"); + } + const configs = await loadConfigs(projectId); const config = configs.find((c) => c.id === configId); if (!config) { return null; @@ -112,12 +112,12 @@ function getConfigById(projectId, configId) { * @param {string} mode - 当前模式（plan/edit/readonly） * @returns {Object} 创建的配置项 */ -function createConfig(projectId, data, mode) { - if (mode === "readonly") { - throw new Error("只读模式下不允许新增配置"); - } - - const project = getProjectById(projectId); +async function createConfig(projectId, data, mode) { + if (mode === "readonly") { + throw new Error("只读模式下不允许新增配置"); + } + + const project = await getProjectById(projectId); if (!project) { throw new Error("项目不存在"); } @@ -152,14 +152,14 @@ function createConfig(projectId, data, mode) { validateEditValue(config); } - const configs = loadConfigs(projectId); - configs.push(config); - saveConfigs(projectId, configs); - - return { - ...config, - path: buildConfigPath(project, config), - }; +const configs = await loadConfigs(projectId); + configs.push(config); + await saveConfigs(projectId, configs); + + return { + ...config, + path: buildConfigPath(project, config), + }; } /** @@ -247,17 +247,17 @@ function validateStructure(config, project) { * @param {string} mode - 当前模式 * @returns {Object} 更新后的配置项 */ -function updateConfig(projectId, configId, data, mode) { - if (mode === "readonly") { - throw new Error("只读模式下不允许修改配置"); - } - - const project = getProjectById(projectId); - if (!project) { - throw new Error("项目不存在"); - } - - const configs = loadConfigs(projectId); +async function updateConfig(projectId, configId, data, mode) { + if (mode === "readonly") { + throw new Error("只读模式下不允许修改配置"); + } + + const project = await getProjectById(projectId); + if (!project) { + throw new Error("项目不存在"); + } + + const configs = await loadConfigs(projectId); const index = configs.findIndex((c) => c.id === configId); if (index === -1) { throw new Error("配置项不存在"); @@ -291,14 +291,14 @@ function updateConfig(projectId, configId, data, mode) { validateStructure(config, project); } - config.updatedAt = new Date().toISOString(); - configs[index] = config; - saveConfigs(projectId, configs); - - return { - ...config, - path: buildConfigPath(project, config), - }; +config.updatedAt = new Date().toISOString(); + configs[index] = config; + await saveConfigs(projectId, configs); + + return { + ...config, + path: buildConfigPath(project, config), + }; } /** @@ -308,19 +308,19 @@ function updateConfig(projectId, configId, data, mode) { * @param {string} mode - 当前模式 * @returns {boolean} 是否删除成功 */ -function deleteConfig(projectId, configId, mode) { - if (mode === "readonly") { - throw new Error("只读模式下不允许删除配置"); - } - - const configs = loadConfigs(projectId); - const index = configs.findIndex((c) => c.id === configId); - if (index === -1) { - return false; - } - configs.splice(index, 1); - saveConfigs(projectId, configs); - return true; +async function deleteConfig(projectId, configId, mode) { + if (mode === "readonly") { + throw new Error("只读模式下不允许删除配置"); + } + + const configs = await loadConfigs(projectId); + const index = configs.findIndex((c) => c.id === configId); + if (index === -1) { + return false; + } + configs.splice(index, 1); + await saveConfigs(projectId, configs); + return true; } module.exports = { diff --git a/src/services/projectService.js b/src/services/projectService.js index 0fa0e5a..24bcaea 100644 --- a/src/services/projectService.js +++ b/src/services/projectService.js @@ -3,7 +3,7 @@ * 处理项目的业务逻辑，包括 CRUD 操作、维度校验与维度映射配置 * 注意：项目创建后维度开关不可修改，只能通过项目转换机制处理；维度映射方式可后续调整 */ -const { loadProjects, saveProjects, deleteConfigs } = require("../persistence/jsonAdapter"); +const { loadProjects, saveProjects, deleteConfigs } = require("../persistence/duckdbAdapter"); // 维度映射方式白名单：path=作为存储路径前缀；label=仅作为筛选标签不进入路径 const VALID_MAPPINGS = ["path", "label"]; @@ -25,22 +25,22 @@ function normalizeMapping(mapping) { return VALID_MAPPINGS.includes(mapping) ? mapping : "path"; } -/** - * 获取所有项目列表 - * @returns {Array} 项目列表 - */ -function getAllProjects() { - return loadProjects(); -} - -/** - * 根据 ID 获取单个项目 - * @param {string} id - 项目 ID - * @returns {Object|null} 项目对象，不存在时返回 null - */ -function getProjectById(id) { - const projects = loadProjects(); - return projects.find((p) => p.id === id) || null; +/** + * 获取所有项目列表 + * @returns {Promise} 项目列表 + */ +async function getAllProjects() { + return loadProjects(); +} + +/** + * 根据 ID 获取单个项目 + * @param {string} id - 项目 ID + * @returns {Promise} 项目对象，不存在时返回 null + */ +async function getProjectById(id) { + const projects = await loadProjects(); + return projects.find((p) => p.id === id) || null; } /** @@ -56,14 +56,14 @@ function getProjectById(id) { * @param {string} data.idcMapping - 机房映射方式（path/label） * @param {string} data.environmentMapping - 环境映射方式（path/label） * @param {string} data.groupMapping - 分组映射方式（path/label） - * @returns {Object} 创建的项目对象 - */ -function createProject(data) { - if (!data.name || data.name.trim() === "") { - throw new Error("项目名称不能为空"); - } - - const projects = loadProjects(); +* @returns {Promise} 创建的项目对象 + */ +async function createProject(data) { + if (!data.name || data.name.trim() === "") { + throw new Error("项目名称不能为空"); + } + + const projects = await loadProjects(); const now = new Date().toISOString(); const project = { id: generateId(), @@ -79,9 +79,9 @@ function createProject(data) { updatedAt: now, }; - projects.push(project); - saveProjects(projects); - return project; +projects.push(project); + await saveProjects(projects); + return project; } /** @@ -91,10 +91,10 @@ function createProject(data) { * @param {Object} data - 待更新字段 * @param {string} [data.name] - 新项目名称 * @param {string} [data.description] - 新项目描述 - * @returns {Object} 更新后的项目对象 + * @returns {Promise} 更新后的项目对象 */ -function updateProject(id, data) { - const projects = loadProjects(); +async function updateProject(id, data) { + const projects = await loadProjects(); const index = projects.findIndex((p) => p.id === id); if (index === -1) { throw new Error("项目不存在"); @@ -108,7 +108,7 @@ function updateProject(id, data) { projects[index].name = name.trim(); projects[index].description = (data.description === undefined ? projects[index].description : data.description || "").trim(); projects[index].updatedAt = new Date().toISOString(); - saveProjects(projects); + await saveProjects(projects); return projects[index]; } @@ -122,8 +122,8 @@ function updateProject(id, data) { * @param {string} mappings.groupMapping - 分组映射方式（path/label） * @returns {Object} 更新后的项目对象 */ -function updateMappings(id, mappings) { - const projects = loadProjects(); +async function updateMappings(id, mappings) { + const projects = await loadProjects(); const index = projects.findIndex((p) => p.id === id); if (index === -1) { throw new Error("项目不存在"); @@ -138,27 +138,27 @@ function updateMappings(id, mappings) { if (project.enableGroup && mappings.groupMapping !== undefined) { project.groupMapping = normalizeMapping(mappings.groupMapping); } - project.updatedAt = new Date().toISOString(); - projects[index] = project; - saveProjects(projects); - return project; -} - -/** - * 删除项目及其所有关联配置 - * @param {string} id - 项目 ID - * @returns {boolean} 是否删除成功 - */ -function deleteProject(id) { - const projects = loadProjects(); - const index = projects.findIndex((p) => p.id === id); - if (index === -1) { - return false; - } - projects.splice(index, 1); - saveProjects(projects); - deleteConfigs(id); - return true; +project.updatedAt = new Date().toISOString(); + projects[index] = project; + await saveProjects(projects); + return project; +} + +/** + * 删除项目及其所有关联配置 + * @param {string} id - 项目 ID + * @returns {Promise} 是否删除成功 + */ +async function deleteProject(id) { + const projects = await loadProjects(); + const index = projects.findIndex((p) => p.id === id); + if (index === -1) { + return false; + } + projects.splice(index, 1); + await saveProjects(projects); + await deleteConfigs(id); + return true; } module.exports = { diff --git a/src/services/tokenService.js b/src/services/tokenService.js index 6c69421..e0bb610 100644 --- a/src/services/tokenService.js +++ b/src/services/tokenService.js @@ -3,33 +3,33 @@ * 负责 API 访问令牌的增删改查与吊销，持久化到 data/api-tokens.json * （与 MCP token 独立，MCP token 由 data/mcp.json 管理） */ -const { loadApiTokens, saveApiTokens } = require("../persistence/jsonAdapter"); +const { loadApiTokens, saveApiTokens } = require("../persistence/duckdbAdapter"); const { generateToken } = require("./userService"); /** * 获取所有 Token（完整值，供列表展示） - * @returns {Array} + * @returns {Promise} */ -function getAllTokens() { +async function getAllTokens() { return loadApiTokens(); } /** * 按 ID 查找 Token * @param {string} id - * @returns {Object|null} + * @returns {Promise} */ -function findById(id) { - return loadApiTokens().find((t) => t.id === id) || null; +async function findById(id) { + return (await loadApiTokens()).find((t) => t.id === id) || null; } /** * 创建 Token * @param {Object} data - { name, projectId, permission, expiresInDays } - * @returns {Object} 创建的 Token（含明文 value） + * @returns {Promise} 创建的 Token（含明文 value） */ -function createToken(data) { - const tokens = loadApiTokens(); +async function createToken(data) { + const tokens = await loadApiTokens(); if (!data.name || !data.name.trim()) { throw new Error("Token 名称不能为空"); } @@ -50,7 +50,7 @@ function createToken(data) { expiresAt, }; tokens.push(token); - saveApiTokens(tokens); + await saveApiTokens(tokens); return token; } @@ -58,56 +58,56 @@ function createToken(data) { * 更新 Token 元信息 * @param {string} id * @param {Object} data - { name, projectId, permission, enabled } - * @returns {Object} + * @returns {Promise} */ -function updateToken(id, data) { - const tokens = loadApiTokens(); +async function updateToken(id, data) { + const tokens = await loadApiTokens(); const idx = tokens.findIndex((t) => t.id === id); if (idx === -1) throw new Error("Token 不存在"); if (data.name !== undefined) tokens[idx].name = data.name.trim(); if (data.projectId !== undefined) tokens[idx].projectId = data.projectId; if (data.permission !== undefined) tokens[idx].permission = data.permission; if (data.enabled !== undefined) tokens[idx].enabled = !!data.enabled; - saveApiTokens(tokens); + await saveApiTokens(tokens); return tokens[idx]; } /** * 吊销 Token（设置 enabled = false） * @param {string} id - * @returns {boolean} + * @returns {Promise} */ -function revokeToken(id) { - const tokens = loadApiTokens(); +async function revokeToken(id) { + const tokens = await loadApiTokens(); const idx = tokens.findIndex((t) => t.id === id); if (idx === -1) return false; tokens[idx].enabled = false; - saveApiTokens(tokens); + await saveApiTokens(tokens); return true; } /** * 删除 Token * @param {string} id - * @returns {boolean} + * @returns {Promise} */ -function deleteToken(id) { - const tokens = loadApiTokens(); +async function deleteToken(id) { + const tokens = await loadApiTokens(); const idx = tokens.findIndex((t) => t.id === id); if (idx === -1) return false; tokens.splice(idx, 1); - saveApiTokens(tokens); + await saveApiTokens(tokens); return true; } /** * 根据 token value 校验是否有效（用于 API 鉴权） * @param {string} value - * @returns {Object|null} token 对象或 null + * @returns {Promise} token 对象或 null */ -function validateToken(value) { +async function validateToken(value) { if (!value) return null; - const token = loadApiTokens().find((t) => t.value === value); + const token = (await loadApiTokens()).find((t) => t.value === value); if (!token || !token.enabled) return null; if (token.expiresAt && new Date(token.expiresAt) < new Date()) return null; return token; diff --git a/src/services/userService.js b/src/services/userService.js index 4fc7801..37c8c5b 100644 --- a/src/services/userService.js +++ b/src/services/userService.js @@ -3,7 +3,7 @@ * 负责用户的增删改查、密码校验与哈希，数据持久化到 data/users.json */ const crypto = require("crypto"); -const { loadUsers, saveUsers } = require("../persistence/jsonAdapter"); +const { loadUsers, saveUsers } = require("../persistence/duckdbAdapter"); /** * 密码哈希：使用 scrypt 算法 + 随机盐值 @@ -40,8 +40,8 @@ function generateToken(length = 32) { /** * 确保默认管理员账号存在（首次启动时初始化） */ -function ensureDefaultAdmin() { - const users = loadUsers(); +async function ensureDefaultAdmin() { + const users = await loadUsers(); if (users.length === 0) { users.push({ id: generateToken(8), @@ -54,7 +54,7 @@ function ensureDefaultAdmin() { updatedAt: new Date().toISOString(), lastLoginAt: null, }); - saveUsers(users); + await saveUsers(users); console.log("已创建默认管理员账号: admin"); } } @@ -62,37 +62,37 @@ function ensureDefaultAdmin() { /** * 按用户名查找用户 * @param {string} username - * @returns {Object|null} + * @returns {Promise} */ -function findByUsername(username) { - return loadUsers().find((u) => u.username === username) || null; +async function findByUsername(username) { + return (await loadUsers()).find((u) => u.username === username) || null; } /** * 按 ID 查找用户 * @param {string} id - * @returns {Object|null} + * @returns {Promise} */ -function findById(id) { - return loadUsers().find((u) => u.id === id) || null; +async function findById(id) { + return (await loadUsers()).find((u) => u.id === id) || null; } /** * 获取所有用户（不含密码字段） - * @returns {Array} + * @returns {Promise} */ -function getAllUsers() { - return loadUsers().map(({ password, ...rest }) => rest); +async function getAllUsers() { + return (await loadUsers()).map(({ password, ...rest }) => rest); } /** * 创建用户 * @param {Object} data - { username, displayName, password, role, enabled } - * @returns {Object} 创建的用户（不含密码） + * @returns {Promise} 创建的用户（不含密码） * @throws {Error} */ -function createUser(data) { - const users = loadUsers(); +async function createUser(data) { + const users = await loadUsers(); if (!data.username || !data.username.trim()) { throw new Error("用户名不能为空"); } @@ -114,7 +114,7 @@ function createUser(data) { lastLoginAt: null, }; users.push(user); - saveUsers(users); + await saveUsers(users); const { password, ...rest } = user; return rest; } @@ -123,18 +123,18 @@ function createUser(data) { * 更新用户 * @param {string} id * @param {Object} data - { displayName, role, enabled } - * @returns {Object} + * @returns {Promise} * @throws {Error} */ -function updateUser(id, data) { - const users = loadUsers(); +async function updateUser(id, data) { + const users = await loadUsers(); const idx = users.findIndex((u) => u.id === id); if (idx === -1) throw new Error("用户不存在"); if (data.displayName !== undefined) users[idx].displayName = data.displayName.trim(); if (data.role !== undefined) users[idx].role = data.role === "admin" ? "admin" : "user"; if (data.enabled !== undefined) users[idx].enabled = !!data.enabled; users[idx].updatedAt = new Date().toISOString(); - saveUsers(users); + await saveUsers(users); const { password, ...rest } = users[idx]; return rest; } @@ -145,27 +145,27 @@ function updateUser(id, data) { * @param {string} newPassword * @throws {Error} */ -function resetPassword(id, newPassword) { - const users = loadUsers(); +async function resetPassword(id, newPassword) { + const users = await loadUsers(); const idx = users.findIndex((u) => u.id === id); if (idx === -1) throw new Error("用户不存在"); if (!newPassword) throw new Error("新密码不能为空"); users[idx].password = hashPassword(newPassword); users[idx].updatedAt = new Date().toISOString(); - saveUsers(users); + await saveUsers(users); } /** * 删除用户 * @param {string} id - * @returns {boolean} + * @returns {Promise} */ -function deleteUser(id) { - const users = loadUsers(); +async function deleteUser(id) { + const users = await loadUsers(); const idx = users.findIndex((u) => u.id === id); if (idx === -1) return false; users.splice(idx, 1); - saveUsers(users); + await saveUsers(users); return true; } @@ -173,18 +173,18 @@ function deleteUser(id) { * 登录校验，成功后更新 lastLoginAt * @param {string} username * @param {string} password - * @returns {Object|null} 用户对象（不含密码）或 null + * @returns {Promise} 用户对象（不含密码）或 null */ -function authenticate(username, password) { - const user = findByUsername(username); +async function authenticate(username, password) { + const user = await findByUsername(username); if (!user || !user.enabled) return null; if (!verifyPassword(password, user.password)) return null; // 更新最后登录时间 - const users = loadUsers(); + const users = await loadUsers(); const idx = users.findIndex((u) => u.id === user.id); if (idx !== -1) { users[idx].lastLoginAt = new Date().toISOString(); - saveUsers(users); + await saveUsers(users); } const { password: _, ...rest } = user; return { ...rest, lastLoginAt: users[idx].lastLoginAt }; diff --git a/test/duckdbAdapter.test.js b/test/duckdbAdapter.test.js new file mode 100644 index 0000000..87b9316 --- /dev/null +++ b/test/duckdbAdapter.test.js @@ -0,0 +1,97 @@ +/** + * DuckDB 适配器单元测试 + * 覆盖：写入/读取往返、JSON 异构值保留、JSON 存量自动迁移、重复初始化幂等 + * 运行：npm test + */ +const test = require("node:test"); +const assert = require("node:assert"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); + +// 使用临时数据目录，避免污染真实 data/ +const tmpDataDir = fs.mkdtempSync(path.join(os.tmpdir(), "registry-duckdb-test-")); +process.env.REGISTRY_DATA_DIR = tmpDataDir; + +const { + init, + loadProjects, + saveProjects, + loadConfigs, + saveConfigs, + deleteConfigs, + loadSystem, + saveSystem, + loadMcpConfig, + saveMcpConfig, + loadUsers, + saveUsers, + loadApiTokens, + saveApiTokens, +} = require("../src/persistence/duckdbAdapter"); + +test("projects 写入/读取往返，字段无损保留", async () => { + await init(); + const projects = [ + { id: "p1", name: "项目A", enableIdc: true, nested: { arr: [1, 2, 3] } }, + { id: "p2", name: "项目B", enableEnvironment: false }, + ]; + await saveProjects(projects); + const loaded = await loadProjects(); + assert.strictEqual(loaded.length, 2); + assert.deepStrictEqual(loaded.find((p) => p.id === "p1").nested, { arr: [1, 2, 3] }); +}); + +test("configs 写入/读取往返，值类型异构保留", async () => { + await init(); + const configs = [ + { id: "c1", key: "count", value: 42, type: "number" }, + { id: "c2", key: "flag", value: false, type: "boolean" }, + { id: "c3", key: "raw", value: { deep: ["a", null, true] }, type: "json" }, + ]; + await saveConfigs("proj-x", configs); + const loaded = await loadConfigs("proj-x"); + assert.strictEqual(loaded.length, 3); + assert.strictEqual(loaded.find((c) => c.id === "c1").value, 42); + assert.strictEqual(loaded.find((c) => c.id === "c2").value, false); + assert.deepStrictEqual(loaded.find((c) => c.id === "c3").value, { deep: ["a", null, true] }); + // ensure projectId 未被注入进记录本体 + assert.strictEqual("projectId" in loaded[0], false); + // 其他项目不受影响 + assert.deepStrictEqual(await loadConfigs("proj-other"), []); +}); + +test("configs 删除仅作用于指定项目", async () => { + await init(); + await saveConfigs("proj-keep", [{ id: "k1", key: "a", value: 1 }]); + await deleteConfigs("proj-keep"); + await saveConfigs("proj-keep2", [{ id: "k2", key: "b", value: 2 }]); + await deleteConfigs("proj-keep2"); + await saveConfigs("proj-keep3", [{ id: "k3", key: "c", value: 3 }]); + await deleteConfigs("proj-keep3"); + await saveConfigs("proj-del", [{ id: "d1", key: "x", value: 1 }]); + await deleteConfigs("proj-del"); + await deleteConfigs("proj-del"); + assert.deepStrictEqual(await loadConfigs("proj-del"), []); +}); + +test("system / mcp 写入读取与默认值", async () => { + await init(); + const sys = await loadSystem(); + assert.strictEqual(sys.mode, "edit"); + await saveSystem({ mode: "readonly" }); + assert.strictEqual((await loadSystem()).mode, "readonly"); + const mcp = await loadMcpConfig(); + assert.strictEqual(typeof mcp.token, "string"); + await saveMcpConfig({ enabled: false, token: "abc" }); + assert.deepStrictEqual(await loadMcpConfig(), { enabled: false, token: "abc" }); +}); + +test("users / apiTokens 写入读取往返", async () => { + await init(); + await saveUsers([{ id: "u1", username: "admin", password: "hashed", roles: ["admin"] }]); + const users = await loadUsers(); + assert.strictEqual(users[0].username, "admin"); + await saveApiTokens([{ id: "t1", name: "mcp", value: "secret" }]); + assert.strictEqual((await loadApiTokens())[0].value, "secret"); +}); \ No newline at end of file diff --git a/test/duckdbMigration.test.js b/test/duckdbMigration.test.js new file mode 100644 index 0000000..b84734a --- /dev/null +++ b/test/duckdbMigration.test.js @@ -0,0 +1,76 @@ +/** + * JSON -> DuckDB 自动迁移测试 + * 独立进程运行：模块在 require 时读取 REGISTRY_DATA_DIR，需在 require 前注入种子数据目录 + * 运行：npm test + */ +const test = require("node:test"); +const assert = require("node:assert"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); + +// 构造一个含 JSON 存量数据的种子目录，并设置为数据目录 +const seedDir = path.join(os.tmpdir(), "registry-seed-" + Date.now()); +fs.mkdirSync(seedDir, { recursive: true }); +fs.writeFileSync(path.join(seedDir, "projects.json"), JSON.stringify([ + { id: "proj-mig-1", name: "迁移项目A", enableIdc: false }, +])); +fs.writeFileSync(path.join(seedDir, "users.json"), JSON.stringify([ + { id: "u-mig", username: "migrator", displayName: "迁移用户", role: "user", enabled: true }, +])); +fs.writeFileSync(path.join(seedDir, "api-tokens.json"), JSON.stringify([ + { id: "t-mig", name: "迁移Token", value: "mig-secret", enabled: true }, +])); +fs.writeFileSync(path.join(seedDir, "system.json"), JSON.stringify({ mode: "plan" })); +fs.writeFileSync(path.join(seedDir, "mcp.json"), JSON.stringify({ enabled: true, token: "mcp-token" })); +const configsDir = path.join(seedDir, "configs"); +fs.mkdirSync(configsDir, { recursive: true }); +fs.writeFileSync(path.join(configsDir, "proj-mig-1.json"), JSON.stringify([ + { id: "cfg-mig", key: "db.host", value: "10.0.0.1", type: "string" }, +])); + +process.env.REGISTRY_DATA_DIR = seedDir; + +const duckdbAdapter = require("../src/persistence/duckdbAdapter"); + +test("自动迁移全部 JSON 存量到 DuckDB，数据无损且备份 JSON", async () => { + await duckdbAdapter.init(); + + const projects = await duckdbAdapter.loadProjects(); + assert.strictEqual(projects.length, 1); + assert.strictEqual(projects[0].name, "迁移项目A"); + assert.strictEqual(projects[0].id, "proj-mig-1"); + + const users = await duckdbAdapter.loadUsers(); + assert.strictEqual(users[0].username, "migrator"); + + const tokens = await duckdbAdapter.loadApiTokens(); + assert.strictEqual(tokens[0].value, "mig-secret"); + + const system = await duckdbAdapter.loadSystem(); + assert.strictEqual(system.mode, "plan"); + + const mcp = await duckdbAdapter.loadMcpConfig(); + assert.strictEqual(mcp.token, "mcp-token"); + + const configs = await duckdbAdapter.loadConfigs("proj-mig-1"); + assert.strictEqual(configs.length, 1); + assert.strictEqual(configs[0].value, "10.0.0.1"); + // projectId 不应被注入进记录本体 + assert.strictEqual("projectId" in configs[0], false); + + // 数据迁移后应生成 DB 文件 + assert.ok(fs.existsSync(path.join(seedDir, "registry.duckdb")), "应生成 registry.duckdb"); + + // JSON 源文件应被备份移除（防止二次导入） + assert.ok(!fs.existsSync(path.join(seedDir, "projects.json")), "迁移后 projects.json 应已移走"); + const backupDirs = fs.readdirSync(seedDir).filter((n) => n.startsWith("json-backup-")); + assert.strictEqual(backupDirs.length, 1, "应生成 json-backup- 目录"); +}); + +test("再次初始化（DB 已存在）不重复迁移、不报错", async () => { + const before = await duckdbAdapter.loadProjects(); + await duckdbAdapter.init(); + const after = await duckdbAdapter.loadProjects(); + assert.deepStrictEqual(after, before); +}); \ No newline at end of file diff --git a/test/mcp.test.js b/test/mcp.test.js index fbaf85a..93a3fff 100644 --- a/test/mcp.test.js +++ b/test/mcp.test.js @@ -5,22 +5,32 @@ */ const test = require("node:test"); const assert = require("node:assert"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); const { createMcpServer } = require("../src/mcp/tools"); -const { loadSystem, saveSystem } = require("../src/persistence/jsonAdapter"); +const { init, loadSystem, saveSystem } = require("../src/persistence/duckdbAdapter"); -// 恢复测试前的系统模式，避免影响其他用例 -const originalMode = (loadSystem() || {}).mode || "edit"; -test.after(() => { - const system = loadSystem() || {}; - system.mode = originalMode; - saveSystem(system); -}); +// 使用临时数据目录，避免污染真实 data/ +const tmpDataDir = fs.mkdtempSync(path.join(os.tmpdir(), "registry-mcp-test-")); +process.env.REGISTRY_DATA_DIR = tmpDataDir; let server; -test.before(() => { +let originalMode = "edit"; + +// 初始化 DuckDB，记录测试前模式；测试结束后恢复原模式 +test.before(async () => { + await init(); + originalMode = (await loadSystem() || {}).mode || "edit"; server = createMcpServer(); }); +test.after(async () => { + const system = await loadSystem() || {}; + system.mode = originalMode; + await saveSystem(system); +}); + /** * 直接调用已注册工具的处理函数 * @param {string} name - 工具名