98 lines
3.3 KiB
JavaScript
98 lines
3.3 KiB
JavaScript
/**
|
|
* 拉所有启用的数据源, upsert 到 Indicator 表
|
|
* 用法: npm run data:refresh
|
|
* 幂等: 同 name 覆盖 data, 跨源独立, 错误隔离
|
|
*
|
|
* DB 优先策略:
|
|
* 1. 先从 DB 查已存在的 name 集合
|
|
* 2. 把集合传给每个 fetcher, fetcher 跳过已存在的
|
|
* 3. 只对 DB 没有的指标调远程拉取, 节省 token (尤其妙想 MCP 这种 LLM 接口)
|
|
*
|
|
* 加新数据源: 写 scripts/fetchers/<name>.fetcher.js 导出 META + fetchAll(opts),
|
|
* 然后在 scripts/fetchers/index.js 的 FETCHERS 注册
|
|
*/
|
|
require('dotenv').config()
|
|
const mysql = require('mysql2/promise')
|
|
|
|
function parseUrl(url) {
|
|
const u = new URL(url)
|
|
return {
|
|
host: u.hostname,
|
|
port: Number(u.port) || 3306,
|
|
user: decodeURIComponent(u.username),
|
|
password: decodeURIComponent(u.password),
|
|
database: u.pathname.replace(/^\//, ''),
|
|
}
|
|
}
|
|
|
|
async function upsertIndicator(conn, payload) {
|
|
const sql =
|
|
'INSERT INTO `Indicator` (`name`, `groupName`, `data`, `createdAt`, `updatedAt`) ' +
|
|
'VALUES (?, ?, ?, NOW(3), NOW(3)) ' +
|
|
'ON DUPLICATE KEY UPDATE `groupName` = VALUES(`groupName`), `data` = VALUES(`data`), `updatedAt` = NOW(3)'
|
|
const [r] = await conn.execute(sql, [payload.name, payload.groupName, JSON.stringify(payload.data)])
|
|
return r.affectedRows
|
|
}
|
|
|
|
async function fetchExistingNames(conn) {
|
|
const [rows] = await conn.execute('SELECT `name` FROM `Indicator`')
|
|
return new Set(rows.map((r) => r.name))
|
|
}
|
|
|
|
async function main() {
|
|
if (!process.env.DATABASE_URL) {
|
|
console.error('缺少 DATABASE_URL 环境变量')
|
|
process.exit(1)
|
|
}
|
|
const conn = await mysql.createConnection(parseUrl(process.env.DATABASE_URL))
|
|
try {
|
|
// 1) 查 DB 已存在的指标名
|
|
const existing = await fetchExistingNames(conn)
|
|
console.log(`[Refresh] DB 已有 ${existing.size} 个指标: ${[...existing].join(', ') || '(空)'}`)
|
|
|
|
// 2) 跑所有 fetcher, 传入 existing (DB 优先)
|
|
const { runFetchers } = await import('./fetchers/index.js')
|
|
const fetcherResults = await runFetchers({ skipNames: existing })
|
|
|
|
// 3) 汇总
|
|
const allItems = []
|
|
let skipped = 0, failed = 0
|
|
for (const r of fetcherResults) {
|
|
if (r.skipped) { skipped++; continue }
|
|
if (!r.ok) { failed++; continue }
|
|
// fetcher 内部已经按 skipNames 过滤过, 这里再做一次防御性去重
|
|
for (const it of r.items) {
|
|
if (existing.has(it.name)) continue
|
|
allItems.push(it)
|
|
}
|
|
}
|
|
console.log(`\n[Refresh] 汇总: ${allItems.length} 个新指标 (跳过 ${skipped} 个数据源, 失败 ${failed} 个, DB 已存在 ${existing.size} 个)`)
|
|
|
|
if (failed > 0) {
|
|
console.error('[Refresh] 警告: 有数据源失败, 仅写入成功的指标')
|
|
}
|
|
|
|
if (allItems.length === 0) {
|
|
console.log('[Refresh] 无新指标, 不写库 (DB 优先策略生效)')
|
|
return
|
|
}
|
|
|
|
// 4) 写库
|
|
console.log('[Refresh] 写入数据库...')
|
|
let ins = 0, upd = 0
|
|
for (const it of allItems) {
|
|
const n = await upsertIndicator(conn, { name: it.name, groupName: it.groupName, data: it.data })
|
|
if (n === 1) ins++
|
|
else if (n === 2) upd++
|
|
}
|
|
console.log(`[Refresh] 写入完成: ${ins} 新增, ${upd} 更新\n`)
|
|
} finally {
|
|
await conn.end()
|
|
}
|
|
}
|
|
|
|
main().catch((e) => {
|
|
console.error('[Refresh] 异常:', e)
|
|
process.exit(1)
|
|
})
|