feat(宏观): ifinance 分类动态挂载为宏观 tab, 报警迁入监控, 下线油价/本地经济数据/策略研究
Docker Build and Push / build-image (push) Successful in 2m37s
Docker Build and Push / build-image (push) Successful in 2m37s
- 宏观只保留本地 新闻/经济文章, 外接 ifinance categories[] 动态成为 tab (纯前端直连, CORS/_headers, https) - 报警从宏观迁到监控第 4 个 tab (保留 Alert 模块, 供未来新增报警源) - 彻底删除油价/经济数据/策略研究: 前端组件+api、后端 apizero/data 路由、cron、fetchers、种子数据 - 新增迁移 DROP Indicator/IndicatorPoint/Strategy/OilPriceHistory; 重新生成 Prisma Client - CSP 放开 *.honor3.com (含端口通配), 避免新增子域反复重装外壳
This commit is contained in:
jif2.zhang
@@ -1,72 +0,0 @@
|
||||
const fs = require('fs')
|
||||
const content = `import cron from 'node-cron'
|
||||
import { prisma } from './db.js'
|
||||
import mysql from 'mysql2/promise'
|
||||
import { fetchNewsList, persistNews } from '../scripts/fetchers/news.fetcher.js'
|
||||
import { checkAfterRefresh } from './alert-engine.js'
|
||||
|
||||
function parseUrl(url) {
|
||||
const u = new URL(url)
|
||||
return { host: u.hostname, port: Number(u.port) || 3306, user: decodeURIComponent(u.username), password: decodeURIComponent(u.password), database: u.pathname.replace(/^\\//, '') }
|
||||
}
|
||||
|
||||
async function openConn() {
|
||||
return mysql.createConnection(parseUrl(process.env.DATABASE_URL))
|
||||
}
|
||||
|
||||
// 每天凌晨 3:00 (本地时间) 拉取新闻 + 跑跨源对比
|
||||
function scheduleNews() {
|
||||
cron.schedule('0 3 * * *', async () => {
|
||||
console.log('[cron] 3:00 触发')
|
||||
try {
|
||||
const list = await fetchNewsList()
|
||||
const r = await persistNews(list)
|
||||
console.log('[cron] 新闻入库: 新增 ' + r.inserted + ' / 跳过 ' + r.skipped)
|
||||
const conn = await openConn()
|
||||
try {
|
||||
const a = await checkAfterRefresh(conn)
|
||||
console.log('[cron] 报警检测: 新增 ' + a.inserted + ' 条 (扫描 ' + a.scanned + ')')
|
||||
} finally { await conn.end() }
|
||||
} catch (e) {
|
||||
console.error('[cron] 任务失败:', e.message)
|
||||
}
|
||||
})
|
||||
console.log('[cron] 已注册: 每天 03:00 拉取新闻 + 跨源报警检测')
|
||||
}
|
||||
|
||||
// 每 2 分钟: 拉取 komari 节点 + 服务自检
|
||||
function scheduleMonitors() {
|
||||
cron.schedule('*/2 * * * *', async () => {
|
||||
try {
|
||||
const k = require('./komari-monitor.cjs')
|
||||
const r = await k.runOnce()
|
||||
console.log('[cron] komari: nodes=' + r.nodes + ' snapshots=' + r.snapshots + ' history=' + r.history + ' failed=' + r.failed)
|
||||
} catch (e) { console.error('[cron] komari 任务失败:', e.message) }
|
||||
try {
|
||||
const h = require('./health-check.cjs')
|
||||
const arr = await h.runOnce()
|
||||
const down = arr.filter(x => x.status !== 'up').map(x => x.name)
|
||||
console.log('[cron] health: ' + arr.length + ' 项, down=' + (down.length ? down.join(',') : '无'))
|
||||
} catch (e) { console.error('[cron] health 任务失败:', e.message) }
|
||||
})
|
||||
console.log('[cron] 已注册: 每 2 分钟 komari + 服务自检')
|
||||
}
|
||||
|
||||
// 每小时: 清理 24h 前的 komari 历史
|
||||
function scheduleMonitorCleanup() {
|
||||
cron.schedule('0 * * * *', async () => {
|
||||
try {
|
||||
const k = require('./komari-monitor.cjs')
|
||||
const n = await k.cleanupOld()
|
||||
console.log('[cron] 监控历史清理: 删除 ' + n + ' 条')
|
||||
} catch (e) { console.error('[cron] 监控清理失败:', e.message) }
|
||||
})
|
||||
console.log('[cron] 已注册: 每小时清理 24h 前监控历史')
|
||||
}
|
||||
|
||||
scheduleNews()
|
||||
scheduleMonitors()
|
||||
scheduleMonitorCleanup()
|
||||
`
|
||||
fs.writeFileSync('server/cron.js', content, 'utf8')
|
||||
console.log('OK len=' + content.length)
|
||||
@@ -1,17 +0,0 @@
|
||||
require('dotenv').config();
|
||||
const mysql = require('mysql2/promise');
|
||||
(async () => {
|
||||
const u = new URL(process.env.DATABASE_URL);
|
||||
const c = await mysql.createConnection({
|
||||
host: u.hostname, port: Number(u.port),
|
||||
user: decodeURIComponent(u.username),
|
||||
password: decodeURIComponent(u.password),
|
||||
database: u.pathname.replace(/^\//, ''),
|
||||
});
|
||||
// 保留 id 最小的, 删掉其余重名行
|
||||
for (const t of ['Strategy', 'Indicator']) {
|
||||
const [r] = await c.query('DELETE FROM ' + t + ' WHERE id NOT IN (SELECT min_id FROM (SELECT MIN(id) AS min_id FROM ' + t + ' GROUP BY name) AS keep)');
|
||||
console.log('[' + t + '] 删除 ' + r.affectedRows + ' 条重复行');
|
||||
}
|
||||
await c.end();
|
||||
})().catch(e=>{console.error(e);process.exit(1)});
|
||||
@@ -1,32 +0,0 @@
|
||||
/**
|
||||
* 列出东方财富妙想 MCP 暴露的所有工具
|
||||
* 用法: npm run data:discover
|
||||
* 输出: tool 名称 + 描述 + 输入 schema
|
||||
*/
|
||||
require('dotenv').config()
|
||||
|
||||
async function main() {
|
||||
if (!process.env.EM_API_KEY) {
|
||||
console.error('缺少 EM_API_KEY (在 .env 配)')
|
||||
process.exit(1)
|
||||
}
|
||||
const { listAvailableTools } = await import('./fetchers/eastmoney-mcp.fetcher.js')
|
||||
console.log('[Discover] 调妙想 MCP tools/list...')
|
||||
const tools = await listAvailableTools()
|
||||
console.log(`[Discover] 找到 ${tools.length} 个工具\n`)
|
||||
for (const t of tools) {
|
||||
console.log(`── ${t.name} ${'─'.repeat(Math.max(0, 60 - t.name.length))}`)
|
||||
if (t.description) console.log(` ${t.description}`)
|
||||
if (t.inputSchema) {
|
||||
console.log(' inputSchema:')
|
||||
const s = JSON.stringify(t.inputSchema, null, 2)
|
||||
s.split('\n').forEach((line) => console.log(' ' + line))
|
||||
}
|
||||
console.log('')
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
console.error('[Discover] 失败:', e.message)
|
||||
process.exit(1)
|
||||
})
|
||||
@@ -27,9 +27,6 @@ const TABLES = [
|
||||
['TaskMember', 'taskMember'],
|
||||
['TaskTag', 'taskTag'],
|
||||
['Article', 'article'],
|
||||
['Strategy', 'strategy'],
|
||||
['Indicator', 'indicator'],
|
||||
['IndicatorPoint', 'indicatorPoint'],
|
||||
['News', 'news'],
|
||||
['Alert', 'alert'],
|
||||
['ServerNode', 'serverNode'],
|
||||
@@ -37,7 +34,6 @@ const TABLES = [
|
||||
['ServiceCheck', 'serviceCheck'],
|
||||
['Econ_SowInventory', 'econ_SowInventory'],
|
||||
['ScraperCache', 'scraperCache'],
|
||||
['OilPriceHistory', 'oilPriceHistory'],
|
||||
['MihomeAccount', 'mihomeAccount'],
|
||||
['MihomeHome', 'mihomeHome'],
|
||||
['MihomeDevice', 'mihomeDevice'],
|
||||
|
||||
@@ -1,79 +0,0 @@
|
||||
|
||||
import { spawn } from 'child_process';
|
||||
|
||||
async function runScraper(scraperPath, startDate, endDate) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const args = [scraperPath];
|
||||
if (startDate) {
|
||||
args.push('--start', startDate);
|
||||
}
|
||||
if (endDate) {
|
||||
args.push('--end', endDate);
|
||||
}
|
||||
|
||||
const proc = spawn('bun', args, { stdio: 'inherit' });
|
||||
proc.on('close', (code) => {
|
||||
if (code === 0) {
|
||||
resolve();
|
||||
} else {
|
||||
reject(new Error(`Process exited with code ${code}`));
|
||||
}
|
||||
});
|
||||
proc.on('error', (err) => {
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchOneYearData() {
|
||||
console.log('='.repeat(70));
|
||||
console.log('📊 获取近一年数据');
|
||||
console.log('='.repeat(70));
|
||||
|
||||
const currentDate = new Date();
|
||||
const startDate = new Date(currentDate);
|
||||
startDate.setFullYear(startDate.getFullYear() - 1);
|
||||
|
||||
const startDateStr = startDate.toISOString().split('T')[0];
|
||||
const endDateStr = currentDate.toISOString().split('T')[0];
|
||||
|
||||
console.log(`时间范围: ${startDateStr} 至 ${endDateStr}`);
|
||||
console.log('');
|
||||
|
||||
console.log('='.repeat(70));
|
||||
console.log('💱 开始获取 USD-CNY 汇率数据...');
|
||||
console.log('='.repeat(70));
|
||||
|
||||
let usdSuccess = false;
|
||||
try {
|
||||
await runScraper('./scraper/usd-cny-rate.scraper.js', startDateStr, endDateStr);
|
||||
console.log(`✅ USD-CNY 汇率数据更新完成`);
|
||||
usdSuccess = true;
|
||||
} catch (error) {
|
||||
console.error(`❌ USD-CNY 汇率数据更新失败:`, error.message);
|
||||
}
|
||||
|
||||
console.log('');
|
||||
console.log('='.repeat(70));
|
||||
console.log('📈 开始获取沪深300指数数据...');
|
||||
console.log('='.repeat(70));
|
||||
|
||||
let sh300Success = false;
|
||||
try {
|
||||
await runScraper('./scraper/sh300.scraper.js', startDateStr, endDateStr);
|
||||
console.log(`✅ 沪深300指数数据更新完成`);
|
||||
sh300Success = true;
|
||||
} catch (error) {
|
||||
console.error(`❌ 沪深300指数数据更新失败:`, error.message);
|
||||
}
|
||||
|
||||
console.log('');
|
||||
console.log('='.repeat(70));
|
||||
console.log('📊 数据获取完成');
|
||||
console.log('='.repeat(70));
|
||||
console.log(`USD-CNY 汇率: ${usdSuccess ? '✅ 成功' : '❌ 失败'}`);
|
||||
console.log(`沪深300指数: ${sh300Success ? '✅ 成功' : '❌ 失败'}`);
|
||||
console.log('='.repeat(70));
|
||||
}
|
||||
|
||||
fetchOneYearData();
|
||||
@@ -1,303 +0,0 @@
|
||||
// 东方财富妙想 MCP 数据源
|
||||
// 协议: StreamableHttp (MCP 2025-03-26 spec), HTTP POST + JSON-RPC 2.0
|
||||
// - 认证: header `em_api_key: <EM_API_KEY>`
|
||||
// - 妙想 MCP 是"自然语言驱动": 传 query 字符串, 内部 LLM 解析 + 调底层数据接口
|
||||
// - 重要: 每次调用 5-20s + LLM token 成本, 必须配合"DB 优先" (refresh 时已存在的指标跳过)
|
||||
//
|
||||
// 需要的 .env:
|
||||
// EM_API_KEY=xxx 必填
|
||||
// EASTMONEY_MCP_BASE=... 可选, 默认 https://mxapi.eastmoney.com/mxds/mcp
|
||||
//
|
||||
// INDICATORS 数组: 配置要拉的指标
|
||||
// 每项:
|
||||
// name → Indicator.name (主键)
|
||||
// groupName → Indicator.groupName
|
||||
// title → data.title
|
||||
// unit → data.unit
|
||||
// tool → MCP tool 名
|
||||
// query → 自然语言查询
|
||||
// pickSheet(sheets) → 选一张 sheet (默认取第一张)
|
||||
// parseSheet(sheet) → [{period, value}, ...] 默认按宽表转长表 (列=日期, 行=值)
|
||||
|
||||
import axios from 'axios'
|
||||
|
||||
const DEFAULT_BASE = 'https://mxapi.eastmoney.com/mxds/mcp'
|
||||
const TIMEOUT = 60000
|
||||
|
||||
// ====== INDICATORS 配置 ======
|
||||
// 跑 npm run data:discover 看可用 tool, 再加新指标
|
||||
const INDICATORS = [
|
||||
{
|
||||
name: 'china-cpi-yoy',
|
||||
groupName: 'china-macro',
|
||||
title: '中国 CPI 同比',
|
||||
unit: '%',
|
||||
tool: 'mx_macro_data',
|
||||
query: '查询中国 2015 年至今 CPI 月度同比数据，每一个月的数值',
|
||||
pickSheet: (sheets) =>
|
||||
sheets.find((s) => s.items?.[0]?.[0]?.includes('CPI') && s.items[0][0].includes('同比'))
|
||||
|| sheets.find((s) => s.sheetName?.includes('宏观数据'))
|
||||
|| sheets[0],
|
||||
},
|
||||
{
|
||||
name: 'china-gdp-yoy',
|
||||
groupName: 'china-macro',
|
||||
title: '中国 GDP 同比',
|
||||
unit: '%',
|
||||
tool: 'mx_macro_data',
|
||||
query: '查询中国 2015-2025 年 GDP 不变价同比增速，季度数据，每一季度的数值',
|
||||
pickSheet: (sheets) =>
|
||||
sheets.find((s) => s.items?.[0]?.[0]?.includes('GDP') && s.items[0][0].includes('同比'))
|
||||
|| sheets[0],
|
||||
},
|
||||
{
|
||||
// 国债收益率: MCP 返回日频, 用月末降采样成月频点
|
||||
name: 'china-bond-10y',
|
||||
groupName: 'china-macro',
|
||||
title: '中国国债 10 年期收益率',
|
||||
unit: '%',
|
||||
frequency: 'month',
|
||||
tool: 'mx_macro_data',
|
||||
query: '查询中国10年期国债到期收益率 2024年至今 每日数据',
|
||||
pickSheet: (sheets) =>
|
||||
sheets.find((s) => s.items?.some?.((row) => String(row[0] || '').includes('国债到期收益率:10年')))
|
||||
|| sheets[0],
|
||||
parseSheet: (sheet) => monthEndSample(sheet, '国债到期收益率:10年'),
|
||||
},
|
||||
{
|
||||
name: 'china-bond-1y',
|
||||
groupName: 'china-macro',
|
||||
title: '中国国债 1 年期收益率',
|
||||
unit: '%',
|
||||
frequency: 'month',
|
||||
tool: 'mx_macro_data',
|
||||
query: '查询中国1年期国债到期收益率 2024年至今 每日数据',
|
||||
pickSheet: (sheets) =>
|
||||
sheets.find((s) => s.items?.some?.((row) => String(row[0] || '').includes('国债到期收益率:1年')))
|
||||
|| sheets[0],
|
||||
parseSheet: (sheet) => monthEndSample(sheet, '国债到期收益率:1年'),
|
||||
},
|
||||
{
|
||||
name: 'china-lpr-1y',
|
||||
groupName: 'china-macro',
|
||||
title: '中国 LPR 1 年期',
|
||||
unit: '%',
|
||||
frequency: 'month',
|
||||
tool: 'mx_macro_data',
|
||||
query: '查询中国LPR 1年期贷款市场报价利率 2024年至今 月度数据，每一个月的数值',
|
||||
pickSheet: (sheets) =>
|
||||
sheets.find((s) => s.sheetName?.includes('宏观数据（月）'))
|
||||
|| sheets[0],
|
||||
parseSheet: (sheet) => pickRowByKeyword(sheet, '(LPR):1年'),
|
||||
},
|
||||
{
|
||||
name: 'china-lpr-5y',
|
||||
groupName: 'china-macro',
|
||||
title: '中国 LPR 5 年期',
|
||||
unit: '%',
|
||||
frequency: 'month',
|
||||
tool: 'mx_macro_data',
|
||||
query: '查询中国LPR 5年期以上贷款市场报价利率 2024年至今 月度数据，每一个月的数值',
|
||||
pickSheet: (sheets) =>
|
||||
sheets.find((s) => s.sheetName?.includes('宏观数据（月）'))
|
||||
|| sheets[0],
|
||||
parseSheet: (sheet) => pickRowByKeyword(sheet, '(LPR):5年'),
|
||||
},
|
||||
]
|
||||
|
||||
export const META = {
|
||||
name: 'eastmoney-mcp',
|
||||
description: '东方财富妙想 MCP (StreamableHttp, NL 查询)',
|
||||
docs: 'https://mxapi.eastmoney.com/mxds/doc/mcp-install.md',
|
||||
}
|
||||
|
||||
// ====== StreamableHttp 客户端 ======
|
||||
|
||||
class McpClient {
|
||||
constructor(baseUrl, apiKey) {
|
||||
this.baseUrl = baseUrl
|
||||
this.apiKey = apiKey
|
||||
this.id = 0
|
||||
}
|
||||
async _request(method, params) {
|
||||
const payload = { jsonrpc: '2.0', id: ++this.id, method, ...(params ? { params } : {}) }
|
||||
const resp = await axios.post(this.baseUrl, payload, {
|
||||
timeout: TIMEOUT,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Accept': 'application/json, text/event-stream',
|
||||
'em_api_key': this.apiKey,
|
||||
},
|
||||
responseType: 'text',
|
||||
transformResponse: [(d) => d],
|
||||
validateStatus: () => true,
|
||||
})
|
||||
if (resp.status >= 400) {
|
||||
throw new Error(`HTTP ${resp.status}: ${String(resp.data).slice(0, 300)}`)
|
||||
}
|
||||
const parsed = parseMcpResponse(String(resp.data || ''))
|
||||
// JSON-RPC 信封 {jsonrpc,id,result|error}, 统一解包 result
|
||||
if (parsed.error) {
|
||||
throw new Error(`MCP error ${parsed.error.code}: ${parsed.error.message}`)
|
||||
}
|
||||
return parsed.result || parsed
|
||||
}
|
||||
async listTools() {
|
||||
const r = await this._request('tools/list')
|
||||
return r.tools || []
|
||||
}
|
||||
async callTool(name, args = {}) {
|
||||
const r = await this._request('tools/call', { name, arguments: args })
|
||||
if (r.isError) {
|
||||
const msg = (r.content || []).map((c) => c.text || '').join('; ')
|
||||
throw new Error(`tool ${name} error: ${msg || JSON.stringify(r).slice(0, 200)}`)
|
||||
}
|
||||
return r
|
||||
}
|
||||
}
|
||||
|
||||
// 解析响应: JSON 或 SSE
|
||||
function parseMcpResponse(text) {
|
||||
const t = text.trim()
|
||||
if (t.startsWith('{')) return JSON.parse(t)
|
||||
if (t.startsWith('data:')) {
|
||||
const lines = t.split('\n').filter((l) => l.startsWith('data:'))
|
||||
let last = null
|
||||
for (const line of lines) {
|
||||
const json = line.slice(5).trim()
|
||||
if (!json || json === '[DONE]') continue
|
||||
try { last = JSON.parse(json) } catch { /* skip */ }
|
||||
}
|
||||
if (!last) throw new Error('SSE 无有效 data 块: ' + text.slice(0, 200))
|
||||
return last
|
||||
}
|
||||
throw new Error('未知响应格式: ' + t.slice(0, 200))
|
||||
}
|
||||
|
||||
// 默认 sheet 解析: 宽表 (列=日期, 行=值) → 长表
|
||||
// 例: { columns:["指标","来源","2024-12","2024-11"], items:[["CPI","统计局","0.1","0.2"]] }
|
||||
// → [{period:"2024-12", value:0.1}, {period:"2024-11", value:0.2}]
|
||||
// 返回时反转成从老到新
|
||||
function defaultParseSheet(sheet) {
|
||||
if (!sheet || !sheet.columns || !sheet.items || !sheet.items[0]) return []
|
||||
const cols = sheet.columns
|
||||
const row = sheet.items[0]
|
||||
const dates = cols.slice(2) // 跳过 [指标名, 来源]
|
||||
const values = row.slice(2)
|
||||
const out = []
|
||||
for (let i = 0; i < dates.length; i++) {
|
||||
const v = parseFloat(values[i])
|
||||
if (!isNaN(v)) out.push({ period: dates[i], value: v })
|
||||
}
|
||||
return out.reverse() // 从老到新
|
||||
}
|
||||
|
||||
// 宽表 → 长表, 指定 item 行 (按指标全名前缀匹配), 日频列降采样为月末值
|
||||
// 例: columns=["指标","来源","2026-08-05","2026-08-04",...] → [{period:"2024-01",value},...]
|
||||
function monthEndSample(sheet, keyword) {
|
||||
if (!sheet || !sheet.columns || !sheet.items) return []
|
||||
const row = sheet.items.find((r) => String(r[0] || '').includes(keyword))
|
||||
if (!row) return []
|
||||
const cols = sheet.columns
|
||||
const dates = cols.slice(2) // 新 → 老
|
||||
const values = row.slice(2)
|
||||
const out = []
|
||||
const seen = new Set()
|
||||
for (let i = 0; i < dates.length; i++) {
|
||||
const v = parseFloat(values[i])
|
||||
const m = String(dates[i] || '').slice(0, 7) // "2024-01-02" → "2024-01"
|
||||
if (!/^\d{4}-\d{2}$/.test(m) || Number.isNaN(v)) continue
|
||||
if (seen.has(m)) continue // 列是降序, 第一次遇到的即当月最后一个交易日
|
||||
seen.add(m)
|
||||
out.push({ period: m, value: v })
|
||||
}
|
||||
return out.reverse() // 从老到新
|
||||
}
|
||||
|
||||
// 宽表 → 长表, 指定 item 行 (按指标全名前缀匹配), 保留原频率 (月/季/年)
|
||||
function pickRowByKeyword(sheet, keyword) {
|
||||
if (!sheet || !sheet.columns || !sheet.items) return []
|
||||
const row = sheet.items.find((r) => String(r[0] || '').includes(keyword))
|
||||
if (!row) return []
|
||||
const cols = sheet.columns
|
||||
const dates = cols.slice(2)
|
||||
const values = row.slice(2)
|
||||
const out = []
|
||||
for (let i = 0; i < dates.length; i++) {
|
||||
const v = parseFloat(values[i])
|
||||
if (!Number.isNaN(v)) out.push({ period: String(dates[i]), value: v })
|
||||
}
|
||||
return out.reverse() // 从老到新
|
||||
}
|
||||
|
||||
// ====== fetcher 入口 ======
|
||||
|
||||
export async function listAvailableTools() {
|
||||
const key = process.env.EM_API_KEY
|
||||
if (!key) throw new Error('缺少 EM_API_KEY')
|
||||
const base = process.env.EASTMONEY_MCP_BASE || DEFAULT_BASE
|
||||
const client = new McpClient(base, key)
|
||||
return client.listTools()
|
||||
}
|
||||
|
||||
// opts.skipNames: Set<string> — DB 已存在的指标名, 跳过不调 MCP (省 token)
|
||||
// opts.onlyNames: Set<string> — 白名单, 只跑这些 (用于补跑单个)
|
||||
export async function fetchAll(opts = {}) {
|
||||
if (INDICATORS.length === 0) {
|
||||
throw new Error('eastmoney-mcp: INDICATORS 数组为空, 加新指标见文件顶部注释')
|
||||
}
|
||||
const skip = opts.skipNames || new Set()
|
||||
const force = opts.forceNames || new Set()
|
||||
const only = opts.onlyNames || null
|
||||
|
||||
const key = process.env.EM_API_KEY
|
||||
if (!key) throw new Error('缺少 EM_API_KEY')
|
||||
const base = process.env.EASTMONEY_MCP_BASE || DEFAULT_BASE
|
||||
const client = new McpClient(base, key)
|
||||
|
||||
const results = []
|
||||
for (const cfg of INDICATORS) {
|
||||
if (skip.has(cfg.name) && !force.has(cfg.name)) {
|
||||
console.log(`[MCP] ${cfg.name} 跳过 (DB 已存在)`)
|
||||
continue
|
||||
}
|
||||
if (only && !only.has(cfg.name)) continue
|
||||
try {
|
||||
console.log(`[MCP] ${cfg.name} 调用 ${cfg.tool}: ${cfg.query}`)
|
||||
const r = await client.callTool(cfg.tool, { query: cfg.query })
|
||||
const text = (r.content || []).map((c) => c.text || '').join('')
|
||||
let payload
|
||||
try { payload = JSON.parse(text) } catch { payload = { raw: text } }
|
||||
const sheets = payload.data || (Array.isArray(payload) ? payload : [payload])
|
||||
const sheet = cfg.pickSheet ? cfg.pickSheet(sheets) : sheets[0]
|
||||
const parsed = cfg.parseSheet ? cfg.parseSheet(sheet) : defaultParseSheet(sheet)
|
||||
if (parsed.length === 0) {
|
||||
console.log(`[MCP] ${cfg.name} 返回空 rows, 跳过`)
|
||||
continue
|
||||
}
|
||||
results.push({
|
||||
name: cfg.name,
|
||||
groupName: cfg.groupName,
|
||||
data: {
|
||||
unit: cfg.unit,
|
||||
title: cfg.title,
|
||||
source: 'eastmoney-mcp',
|
||||
rows: parsed,
|
||||
},
|
||||
})
|
||||
console.log(`[MCP] ${cfg.name} OK, ${parsed.length} rows (${parsed[0]?.period}..${parsed.at(-1)?.period})`)
|
||||
} catch (e) {
|
||||
console.error(`[MCP] ${cfg.name} FAIL: ${e.message}`)
|
||||
}
|
||||
}
|
||||
// 全部被 DB 优先跳过时, 返回显式标记而不是空数组 (避免被误判为失败)
|
||||
if (results.length === 0 && INDICATORS.every((cfg) => skip.has(cfg.name) && !force.has(cfg.name))) {
|
||||
return { skipped: true, reason: 'DB 已存在全部指标' }
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -1,83 +0,0 @@
|
||||
// Fetcher 注册表: 列出所有启用的数据源
|
||||
// 加新数据源只要:
|
||||
// 1. 写一个 scripts/fetchers/<source>.fetcher.js, 导出 META + fetchAll
|
||||
// 2. 在本文件 FETCHERS 数组里加一行
|
||||
// 3. 完事, refresh 脚本会自动跑
|
||||
|
||||
// 每项的 schema:
|
||||
// name: 数据源标识 (用于日志)
|
||||
// module: fetcher 模块路径 (相对 scripts/fetchers/)
|
||||
// enabled: true/false, 可由 env 关闭 (如 ENABLE_EASTMONEY_MCP=0)
|
||||
// env: 需要的环境变量名列表, 缺一即跳过
|
||||
export const FETCHERS = [
|
||||
{
|
||||
name: 'wb-indicators',
|
||||
module: './wb-indicators.fetcher.js',
|
||||
enabled: true,
|
||||
env: [],
|
||||
},
|
||||
{
|
||||
name: 'eastmoney-mcp',
|
||||
module: './eastmoney-mcp.fetcher.js',
|
||||
// 默认开启, 有 APIKey 才真正能跑 (fetcher 内部会自检)
|
||||
enabled: process.env.ENABLE_EASTMONEY_MCP !== '0',
|
||||
env: ['EM_API_KEY'],
|
||||
},
|
||||
// 后续在这里追加:
|
||||
// { name: 'fred', module: './fred.fetcher.js', enabled: true, env: ['FRED_API_KEY'] },
|
||||
// { name: 'tushare', module: './tushare.fetcher.js', enabled: true, env: ['TUSHARE_TOKEN'] },
|
||||
]
|
||||
|
||||
// 校验环境变量, 返回 { ok, missing[] }
|
||||
export function checkEnv(required) {
|
||||
const missing = required.filter((k) => !process.env[k] || !process.env[k].trim())
|
||||
return { ok: missing.length === 0, missing }
|
||||
}
|
||||
|
||||
// 加载并运行所有启用的 fetcher, 返回 [{ name, ok, items, error, skipped }]
|
||||
// 错误隔离: 一个 fetcher 挂了不影响其他
|
||||
// opts.skipNames: Set<string> — DB 优先, 已存在的指标名, 透传给 fetcher
|
||||
// opts.forceNames: Set<string> — 强制刷新名单, 即使 skipNames 里有也照拉
|
||||
export async function runFetchers(opts = {}) {
|
||||
const results = []
|
||||
for (const meta of FETCHERS) {
|
||||
if (!meta.enabled) {
|
||||
results.push({ name: meta.name, skipped: true, reason: 'disabled' })
|
||||
console.log(`[Fetcher] ${meta.name} 跳过 (disabled)`)
|
||||
continue
|
||||
}
|
||||
const envCheck = checkEnv(meta.env)
|
||||
if (!envCheck.ok) {
|
||||
results.push({ name: meta.name, skipped: true, reason: `缺少 env: ${envCheck.missing.join(', ')}` })
|
||||
console.log(`[Fetcher] ${meta.name} 跳过 (${envCheck.missing.join(', ')})`)
|
||||
continue
|
||||
}
|
||||
try {
|
||||
const mod = await import(meta.module)
|
||||
if (!mod.META || !mod.fetchAll) {
|
||||
throw new Error('fetcher 必须导出 META 和 fetchAll')
|
||||
}
|
||||
console.log(`[Fetcher] ${meta.name} 开始...`)
|
||||
const out = await mod.fetchAll({ skipNames: opts.skipNames, onlyNames: opts.onlyNames, forceNames: opts.forceNames })
|
||||
// fetcher 可返回 { skipped: true, reason } 表示"全部被 DB 优先跳过", 不算失败
|
||||
if (out && out.skipped) {
|
||||
results.push({ name: meta.name, skipped: true, reason: out.reason })
|
||||
console.log(`[Fetcher] ${meta.name} 跳过 (${out.reason})`)
|
||||
continue
|
||||
}
|
||||
const items = Array.isArray(out) ? out : []
|
||||
// 校验返回结构
|
||||
const valid = items.filter((it) => it && it.name && it.data && Array.isArray(it.data.rows))
|
||||
if (valid.length === 0) {
|
||||
throw new Error('fetcher 未返回任何有效指标 (需包含 name/data.rows)')
|
||||
}
|
||||
console.log(`[Fetcher] ${meta.name} 拉取 ${valid.length} 个指标`)
|
||||
results.push({ name: meta.name, ok: true, items: valid })
|
||||
} catch (e) {
|
||||
results.push({ name: meta.name, ok: false, error: e.message })
|
||||
console.error(`[Fetcher] ${meta.name} 失败: ${e.message}`)
|
||||
}
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
@@ -1,168 +0,0 @@
|
||||
import axios from 'axios';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
async function fetch(startDate = null, endDate = null, retryCount = 3) {
|
||||
try {
|
||||
const currentDate = new Date();
|
||||
|
||||
if (!startDate) {
|
||||
startDate = new Date(currentDate);
|
||||
startDate.setDate(startDate.getDate() - 30);
|
||||
startDate = startDate.toISOString().split('T')[0];
|
||||
}
|
||||
|
||||
if (!endDate) {
|
||||
endDate = currentDate.toISOString().split('T')[0];
|
||||
}
|
||||
|
||||
console.log('正在获取沪深300指数数据...');
|
||||
console.log(`时间范围: ${startDate} 至 ${endDate}`);
|
||||
console.log(`数据源: 腾讯财经API`);
|
||||
|
||||
// 沪深300指数代码
|
||||
const symbol = '000300';
|
||||
|
||||
// 使用腾讯财经的API端点获取历史数据
|
||||
const response = await axios.get('https://web.ifzq.gtimg.cn/appstock/app/kline/kline?param=sh' + symbol + ',day,' + startDate + ',' + endDate + ',640', {
|
||||
headers: {
|
||||
'Accept': 'application/json',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
|
||||
},
|
||||
timeout: 10000 // 设置10秒超时
|
||||
});
|
||||
|
||||
console.log(`\nAPI状态: ${response.status}`);
|
||||
|
||||
const data = response.data;
|
||||
console.log(`响应数据:`, JSON.stringify(data, null, 2));
|
||||
|
||||
const allData = [];
|
||||
|
||||
if (data && data.data && data.data['sh' + symbol] && data.data['sh' + symbol].day) {
|
||||
const klineData = data.data['sh' + symbol].day;
|
||||
console.log(`数据条数: ${klineData.length}`);
|
||||
|
||||
for (const item of klineData) {
|
||||
allData.push({
|
||||
date: item[0],
|
||||
open: parseFloat(item[1]),
|
||||
close: parseFloat(item[2]),
|
||||
high: parseFloat(item[3]),
|
||||
low: parseFloat(item[4]),
|
||||
volume: parseFloat(item[5]),
|
||||
amount: 0, // 腾讯财经API没有返回成交额数据，设置为0
|
||||
source: '腾讯财经API',
|
||||
fetchTime: new Date().toISOString()
|
||||
});
|
||||
}
|
||||
} else {
|
||||
console.log('数据条数: 0');
|
||||
}
|
||||
|
||||
allData.sort((a, b) => a.date.localeCompare(b.date));
|
||||
|
||||
console.log(`\n共获取到 ${allData.length} 条沪深300指数数据`);
|
||||
|
||||
return {
|
||||
symbol: 'sh000300',
|
||||
name: '沪深300指数',
|
||||
startDate,
|
||||
endDate,
|
||||
count: allData.length,
|
||||
data: allData,
|
||||
fetchTime: new Date().toISOString(),
|
||||
note: '数据来源: 腾讯财经API'
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
console.error('获取沪深300指数数据失败:', error.message);
|
||||
if (error.response) {
|
||||
console.error(`HTTP状态码: ${error.response.status}`);
|
||||
console.error(`响应数据:`, error.response.data);
|
||||
}
|
||||
|
||||
// 重试机制
|
||||
if (retryCount > 0) {
|
||||
console.log(`正在重试... (剩余重试次数: ${retryCount - 1})`);
|
||||
// 等待1秒后重试
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
return fetch(startDate, endDate, retryCount - 1);
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export {
|
||||
fetch
|
||||
};
|
||||
|
||||
const isMainModule = () => {
|
||||
const currentFile = fileURLToPath(import.meta.url);
|
||||
const mainFile = process.argv[1];
|
||||
return currentFile === mainFile;
|
||||
};
|
||||
|
||||
function parseCommandLineArgs() {
|
||||
const args = process.argv.slice(2);
|
||||
let startDate = null;
|
||||
let endDate = null;
|
||||
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
if (args[i] === '--start' && i + 1 < args.length) {
|
||||
startDate = args[i + 1];
|
||||
i++;
|
||||
} else if (args[i] === '--end' && i + 1 < args.length) {
|
||||
endDate = args[i + 1];
|
||||
i++;
|
||||
} else if (args[i] === '--help') {
|
||||
console.log('使用方法:');
|
||||
console.log(' bun sh300.fetcher.js [选项]');
|
||||
console.log('');
|
||||
console.log('选项:');
|
||||
console.log(' --start YYYY-MM-DD 指定开始日期（默认：30天前）');
|
||||
console.log(' --end YYYY-MM-DD 指定结束日期（默认：今天）');
|
||||
console.log(' --help 显示帮助信息');
|
||||
console.log('');
|
||||
console.log('示例:');
|
||||
console.log(' bun sh300.fetcher.js');
|
||||
console.log(' bun sh300.fetcher.js --start 2024-01-01 --end 2024-01-31');
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
return { startDate, endDate };
|
||||
}
|
||||
|
||||
if (isMainModule()) {
|
||||
const { startDate, endDate } = parseCommandLineArgs();
|
||||
|
||||
console.log('='.repeat(50));
|
||||
console.log('沪深300指数数据获取脚本');
|
||||
console.log('='.repeat(50));
|
||||
|
||||
fetch(startDate, endDate)
|
||||
.then(result => {
|
||||
console.log('\n' + '='.repeat(50));
|
||||
console.log('获取结果:');
|
||||
console.log(` 指数代码: ${result.symbol}`);
|
||||
console.log(` 指数名称: ${result.name}`);
|
||||
console.log(` 开始日期: ${result.startDate}`);
|
||||
console.log(` 结束日期: ${result.endDate}`);
|
||||
console.log(` 数据条数: ${result.count}`);
|
||||
console.log(` 获取时间: ${result.fetchTime}`);
|
||||
console.log(` 提示: ${result.note}`);
|
||||
if (result.count > 0) {
|
||||
console.log('-'.repeat(50));
|
||||
console.log('指数数据:');
|
||||
result.data.forEach(item => {
|
||||
console.log(` ${item.date} 开盘: ${item.open} 最高: ${item.high} 最低: ${item.low} 收盘: ${item.close}`);
|
||||
});
|
||||
}
|
||||
console.log('='.repeat(50));
|
||||
})
|
||||
.catch(error => {
|
||||
console.error('执行失败:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -1,150 +0,0 @@
|
||||
import axios from 'axios';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const FRANKFURTER_API = 'https://api.frankfurter.app';
|
||||
|
||||
async function fetch(startDate = null, endDate = null) {
|
||||
try {
|
||||
const currentDate = new Date();
|
||||
|
||||
if (!startDate) {
|
||||
startDate = new Date(currentDate);
|
||||
startDate.setDate(startDate.getDate() - 30);
|
||||
startDate = startDate.toISOString().split('T')[0];
|
||||
}
|
||||
|
||||
if (!endDate) {
|
||||
endDate = currentDate.toISOString().split('T')[0];
|
||||
}
|
||||
|
||||
console.log('正在获取人民币美元汇率数据...');
|
||||
console.log(`时间范围: ${startDate} 至 ${endDate}`);
|
||||
console.log(`数据源: Frankfurter API (欧洲央行汇率数据)`);
|
||||
|
||||
const response = await axios.get(`${FRANKFURTER_API}/${startDate}..${endDate}`, {
|
||||
params: {
|
||||
from: 'USD',
|
||||
to: 'CNY'
|
||||
},
|
||||
headers: {
|
||||
'Accept': 'application/json'
|
||||
}
|
||||
});
|
||||
|
||||
const data = response.data;
|
||||
console.log(`\nAPI状态: ${response.status}`);
|
||||
console.log(`数据条数: ${Object.keys(data.rates || {}).length}`);
|
||||
|
||||
const allRates = [];
|
||||
|
||||
if (data.rates) {
|
||||
for (const [date, rateObj] of Object.entries(data.rates)) {
|
||||
const rateValue = rateObj.CNY;
|
||||
allRates.push({
|
||||
date: date,
|
||||
currency: 'USD/CNY',
|
||||
centerPrice: parseFloat(rateValue.toFixed(4)),
|
||||
sellingRate: parseFloat(rateValue.toFixed(4)),
|
||||
buyingRate: parseFloat(rateValue.toFixed(4)),
|
||||
source: '欧洲央行 (Frankfurter API)',
|
||||
fetchTime: new Date().toISOString()
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
allRates.sort((a, b) => a.date.localeCompare(b.date));
|
||||
|
||||
console.log(`\n共获取到 ${allRates.length} 条汇率数据`);
|
||||
|
||||
return {
|
||||
currency: 'USD/CNY',
|
||||
startDate,
|
||||
endDate,
|
||||
count: allRates.length,
|
||||
data: allRates,
|
||||
fetchTime: new Date().toISOString(),
|
||||
note: '数据来源: 欧洲央行, 通过 Frankfurter API 提供'
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
console.error('获取汇率数据失败:', error.message);
|
||||
if (error.response) {
|
||||
console.error(`HTTP状态码: ${error.response.status}`);
|
||||
console.error(`响应数据:`, error.response.data);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export {
|
||||
fetch
|
||||
};
|
||||
|
||||
const isMainModule = () => {
|
||||
const currentFile = fileURLToPath(import.meta.url);
|
||||
const mainFile = process.argv[1];
|
||||
return currentFile === mainFile;
|
||||
};
|
||||
|
||||
function parseCommandLineArgs() {
|
||||
const args = process.argv.slice(2);
|
||||
let startDate = null;
|
||||
let endDate = null;
|
||||
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
if (args[i] === '--start' && i + 1 < args.length) {
|
||||
startDate = args[i + 1];
|
||||
i++;
|
||||
} else if (args[i] === '--end' && i + 1 < args.length) {
|
||||
endDate = args[i + 1];
|
||||
i++;
|
||||
} else if (args[i] === '--help') {
|
||||
console.log('使用方法:');
|
||||
console.log(' bun usd-cny-rate.fetcher.js [选项]');
|
||||
console.log('');
|
||||
console.log('选项:');
|
||||
console.log(' --start YYYY-MM-DD 指定开始日期（默认：30天前）');
|
||||
console.log(' --end YYYY-MM-DD 指定结束日期（默认：今天）');
|
||||
console.log(' --help 显示帮助信息');
|
||||
console.log('');
|
||||
console.log('示例:');
|
||||
console.log(' bun usd-cny-rate.fetcher.js');
|
||||
console.log(' bun usd-cny-rate.fetcher.js --start 2024-01-01 --end 2024-01-31');
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
return { startDate, endDate };
|
||||
}
|
||||
|
||||
if (isMainModule()) {
|
||||
const { startDate, endDate } = parseCommandLineArgs();
|
||||
|
||||
console.log('='.repeat(50));
|
||||
console.log('人民币美元汇率获取脚本');
|
||||
console.log('='.repeat(50));
|
||||
|
||||
fetch(startDate, endDate)
|
||||
.then(result => {
|
||||
console.log('\n' + '='.repeat(50));
|
||||
console.log('获取结果:');
|
||||
console.log(` 货币对: ${result.currency}`);
|
||||
console.log(` 开始日期: ${result.startDate}`);
|
||||
console.log(` 结束日期: ${result.endDate}`);
|
||||
console.log(` 数据条数: ${result.count}`);
|
||||
console.log(` 获取时间: ${result.fetchTime}`);
|
||||
console.log(` 提示: ${result.note}`);
|
||||
if (result.count > 0) {
|
||||
console.log('-'.repeat(50));
|
||||
console.log('汇率数据:');
|
||||
result.data.forEach(item => {
|
||||
console.log(` ${item.date} 中间价: ${item.centerPrice}`);
|
||||
});
|
||||
}
|
||||
console.log('='.repeat(50));
|
||||
})
|
||||
.catch(error => {
|
||||
console.error('执行失败:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -1,113 +0,0 @@
|
||||
import axios from 'axios'
|
||||
|
||||
// World Bank API 数据源
|
||||
// 文档: https://datahelpdesk.worldbank.org/knowledgebase/articles/898581
|
||||
// 优点: 官方、JSON、免 key、稳定, 中国数据滞后约 1 年 (这是全球宏观数据物理上限)
|
||||
// 接口: GET /v2/country/{code}/indicator/{id}?format=json&date={startYear}:{endYear}&per_page=100
|
||||
|
||||
const WB_BASE = 'https://api.worldbank.org/v2'
|
||||
const TIMEOUT = 15000
|
||||
|
||||
// 6 个核心宏观指标
|
||||
// - 中国 GDP (现价美元) - 国民经济总规模
|
||||
// - 中国 GDP 增速 - 增长动能
|
||||
// - 中国 CPI 同比 - 通胀
|
||||
// - 中国失业率 - 就业
|
||||
// - 美国 GDP (现价美元) - 对标
|
||||
// - 美国 CPI 同比 - 对标
|
||||
export const WB_INDICATORS = [
|
||||
{ id: 'NY.GDP.MKTP.CD', name: 'china-gdp-usd', groupName: 'china-macro', title: '中国 GDP', unit: '万亿美元', scale: 1e-8, digits: 2 },
|
||||
{ id: 'NY.GDP.MKTP.KD.ZG',name: 'china-gdp-growth', groupName: 'china-macro', title: '中国 GDP 增速', unit: '%', scale: 1, digits: 2 },
|
||||
{ id: 'FP.CPI.TOTL.ZG', name: 'china-cpi', groupName: 'china-macro', title: '中国 CPI', unit: '%', scale: 1, digits: 2 },
|
||||
{ id: 'SL.UEM.TOTL.ZS', name: 'china-unemploy', groupName: 'china-macro', title: '中国失业率', unit: '%', scale: 1, digits: 2 },
|
||||
{ id: 'NY.GDP.MKTP.CD', name: 'usa-gdp-usd', groupName: 'usa-macro', title: '美国 GDP', unit: '万亿美元', scale: 1e-8, digits: 2 },
|
||||
{ id: 'FP.CPI.TOTL.ZG', name: 'usa-cpi', groupName: 'usa-macro', title: '美国 CPI', unit: '%', scale: 1, digits: 2 },
|
||||
]
|
||||
|
||||
// World Bank 中国是 CHN, 美国是 USA
|
||||
function countryCodeOf(name) {
|
||||
if (name.startsWith('china-')) return 'CHN'
|
||||
if (name.startsWith('usa-')) return 'USA'
|
||||
throw new Error('Unknown country for ' + name)
|
||||
}
|
||||
|
||||
// 拉单个指标从 startYear 到当前年
|
||||
// 返回 { name, groupName, title, unit, rows: [{ year, value }] }
|
||||
// 注意: WB 对连续请求会限流返回 400 Invalid value, 加 3 次退避重试
|
||||
export async function fetchIndicator(meta, startYear = 2015) {
|
||||
const country = countryCodeOf(meta.name)
|
||||
const currentYear = new Date().getFullYear()
|
||||
const url = `${WB_BASE}/country/${country}/indicator/${meta.id}?format=json&date=${startYear}:${currentYear}&per_page=200`
|
||||
|
||||
let resp
|
||||
const delays = [0, 1500, 4000]
|
||||
for (let attempt = 0; attempt < delays.length; attempt++) {
|
||||
if (delays[attempt]) await new Promise((r) => setTimeout(r, delays[attempt]))
|
||||
try {
|
||||
resp = await axios.get(url, { timeout: TIMEOUT, headers: { 'User-Agent': 'iboard-refresh/1.0' } })
|
||||
break
|
||||
} catch (e) {
|
||||
if (attempt === delays.length - 1) throw e
|
||||
console.log(`[WB] ${meta.name} 请求失败, ${delays[attempt + 1] / 1000}s 后重试 (${e.message})`)
|
||||
}
|
||||
}
|
||||
// WB 返回: [{page,pages,per_page,total,...}, [{indicator, country, value, date}, ...]]
|
||||
const data = resp.data
|
||||
if (!Array.isArray(data) || data.length < 2 || !Array.isArray(data[1])) {
|
||||
throw new Error('World Bank 返回结构异常: ' + JSON.stringify(data).slice(0, 200))
|
||||
}
|
||||
const points = data[1]
|
||||
.filter((p) => p.value !== null && p.value !== undefined)
|
||||
.map((p) => ({
|
||||
year: String(p.date),
|
||||
value: Number((Number(p.value) * meta.scale).toFixed(meta.digits)),
|
||||
}))
|
||||
.sort((a, b) => Number(a.year) - Number(b.year))
|
||||
|
||||
return {
|
||||
name: meta.name,
|
||||
groupName: meta.groupName,
|
||||
data: {
|
||||
unit: meta.unit,
|
||||
title: meta.title,
|
||||
source: 'World Bank API',
|
||||
rows: points,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export const META = {
|
||||
name: 'wb-indicators',
|
||||
description: 'World Bank 官方 API 宏观数据',
|
||||
docs: 'https://datahelpdesk.worldbank.org/knowledgebase/articles/898581',
|
||||
}
|
||||
|
||||
// 拉全部 (返回符合 fetcher 契约的 [{name, groupName, data}, ...])
|
||||
// 错误隔离: 单个指标失败 -> 跳过 + 记日志, 不影响其他
|
||||
// opts.skipNames: Set<string> — DB 已存在的指标名, 跳过不拉 (WB 免费但也要省请求)
|
||||
export async function fetchAll(opts = {}) {
|
||||
const skip = opts.skipNames || new Set()
|
||||
const startYear = 2015
|
||||
const results = []
|
||||
for (const m of WB_INDICATORS) {
|
||||
if (skip.has(m.name)) {
|
||||
console.log(`[WB] ${m.name} 跳过 (DB 已存在)`)
|
||||
continue
|
||||
}
|
||||
try {
|
||||
const r = await fetchIndicator(m, startYear)
|
||||
console.log(`[WB] ${r.name} OK, ${r.data.rows.length} rows (${r.data.rows[0]?.year}..${r.data.rows.at(-1)?.year})`)
|
||||
results.push(r)
|
||||
} catch (e) {
|
||||
console.error(`[WB] ${m.name} FAIL: ${e.message}`)
|
||||
}
|
||||
}
|
||||
// 全部被 DB 优先跳过时, 返回显式标记而不是空数组 (避免被误判为失败)
|
||||
if (results.length === 0 && WB_INDICATORS.every((m) => skip.has(m.name))) {
|
||||
return { skipped: true, reason: 'DB 已存在全部指标' }
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1,119 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
// 一次性导入浙江省油价历史 (2025-2026)
|
||||
// 来源: 手工整理 (省发改委公告 / 潮新闻 / 钱江晚报 / 杭州日报 / 杭州网 等)
|
||||
// 用法: node --experimental-strip-types --no-warnings scripts/import-oil-history.mjs [--dry]
|
||||
|
||||
import path from 'node:path'
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url'
|
||||
import 'dotenv/config'
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url)
|
||||
const __dirname = path.dirname(__filename)
|
||||
const ROOT = path.resolve(__dirname, '..')
|
||||
|
||||
// 接入 prisma (用 file:// URL 避免 Windows 绝对路径 import 报错)
|
||||
const adapterMod = await import('@prisma/adapter-mariadb')
|
||||
const clientPath = pathToFileURL(path.join(ROOT, 'generated/prisma/client.ts')).href
|
||||
const { PrismaClient } = await import(clientPath)
|
||||
|
||||
const adapter = new adapterMod.PrismaMariaDb((() => {
|
||||
const u = new URL(process.env.DATABASE_URL)
|
||||
return {
|
||||
host: u.hostname,
|
||||
port: Number(u.port) || 3306,
|
||||
user: decodeURIComponent(u.username),
|
||||
password: decodeURIComponent(u.password),
|
||||
database: u.pathname.replace(/^\//, ''),
|
||||
}
|
||||
})())
|
||||
const prisma = new PrismaClient({ adapter })
|
||||
|
||||
// 数据: [调价日, 92, 95, 0, source]
|
||||
const RAW = [
|
||||
['2026-07-31', 7.94, 8.44, 7.62, 'manual'],
|
||||
['2026-07-17', 7.39, 7.86, 7.06, 'manual'],
|
||||
['2026-06-18', 7.91, 8.41, 7.59, 'manual'],
|
||||
['2026-06-04', 8.32, 8.85, 8.02, 'manual'],
|
||||
['2026-05-21', 8.74, 9.30, 8.45, 'manual'],
|
||||
['2026-02-03', 6.91, 7.35, 6.56, 'manual'],
|
||||
['2025-12-22', 6.68, 7.11, 6.32, 'manual'],
|
||||
['2025-12-08', 6.82, 7.25, 6.46, 'manual'],
|
||||
['2025-11-24', 6.86, null, null, 'manual'],
|
||||
['2025-11-10', 6.92, 7.36, 6.57, 'manual'],
|
||||
['2025-10-13', 7.03, 7.48, 6.68, 'manual'],
|
||||
['2025-06-17', 7.15, 7.60, 6.81, 'manual'],
|
||||
['2025-06-03', 6.94, 7.38, 6.59, 'manual'],
|
||||
]
|
||||
|
||||
const PROVINCE = '浙江'
|
||||
const FUEL_COLS = [
|
||||
{ fuel: '92', idx: 1 },
|
||||
{ fuel: '95', idx: 2 },
|
||||
{ fuel: '0', idx: 3 },
|
||||
]
|
||||
|
||||
const DRY = process.argv.includes('--dry')
|
||||
|
||||
async function main() {
|
||||
console.log('=== 浙江省油价历史导入 ===')
|
||||
console.log('dry run:', DRY)
|
||||
console.log('')
|
||||
|
||||
let inserted = 0
|
||||
let updated = 0
|
||||
let skipped = 0
|
||||
|
||||
for (const row of RAW) {
|
||||
const dateStr = row[0]
|
||||
const source = row[4]
|
||||
for (const { fuel, idx } of FUEL_COLS) {
|
||||
const price = row[idx]
|
||||
if (price == null || Number.isNaN(price)) {
|
||||
console.log('[SKIP] ' + dateStr + ' ' + fuel + ': 价格为空')
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
const date = new Date(dateStr + 'T00:00:00')
|
||||
if (DRY) {
|
||||
console.log('[DRY] ' + dateStr + ' ' + fuel + ' = ' + price + ' (source=' + source + ')')
|
||||
inserted++
|
||||
continue
|
||||
}
|
||||
try {
|
||||
await prisma.oilPriceHistory.create({
|
||||
data: { date, province: PROVINCE, fuel, price, source },
|
||||
})
|
||||
console.log('[OK] ' + dateStr + ' ' + fuel + ' = ' + price)
|
||||
inserted++
|
||||
} catch (e) {
|
||||
if (e?.code === 'P2002') {
|
||||
try {
|
||||
await prisma.oilPriceHistory.update({
|
||||
where: { date_province_fuel: { date, province: PROVINCE, fuel } },
|
||||
data: { price, source },
|
||||
})
|
||||
console.log('[UPD] ' + dateStr + ' ' + fuel + ' = ' + price)
|
||||
updated++
|
||||
} catch (e2) {
|
||||
console.log('[ERR] ' + dateStr + ' ' + fuel + ': ' + e2.message)
|
||||
}
|
||||
} else {
|
||||
console.log('[ERR] ' + dateStr + ' ' + fuel + ': ' + e.message)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log('')
|
||||
console.log('=== 汇总 ===')
|
||||
console.log('插入: ' + inserted)
|
||||
console.log('更新: ' + updated)
|
||||
console.log('跳过: ' + skipped)
|
||||
|
||||
await prisma.$disconnect()
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
console.error(e)
|
||||
process.exit(1)
|
||||
})
|
||||
@@ -1,84 +0,0 @@
|
||||
/**
|
||||
* 一次性数据迁移: 把 Indicator.data (JSON) 拆分到 IndicatorPoint 明细表
|
||||
* 用法: 在迁移 A (建表+加列) 之后、迁移 B (删 data 列) 之前运行
|
||||
* 幂等: 同 (indicatorId, period) 重复插入会更新 value, 可安全重跑
|
||||
*/
|
||||
require('dotenv').config()
|
||||
const mysql = require('mysql2/promise')
|
||||
|
||||
function parseUrl(url) {
|
||||
const u = new URL(url)
|
||||
return {
|
||||
host: u.hostname,
|
||||
port: Number(u.port) || 3306,
|
||||
user: decodeURIComponent(u.username),
|
||||
password: decodeURIComponent(u.password),
|
||||
database: u.pathname.replace(/^\//, ''),
|
||||
}
|
||||
}
|
||||
|
||||
// 根据 row 的时间字段推断频率: year | quarter | month | day
|
||||
function inferFrequency(row) {
|
||||
if (row && row.month !== undefined) return 'month'
|
||||
if (row && row.year !== undefined) return 'year'
|
||||
const p = String((row && row.period) || '')
|
||||
if (/^\d{4}$/.test(p)) return 'year'
|
||||
if (/^\d{4}-Q[1-4]$/.test(p)) return 'quarter'
|
||||
if (/^\d{4}-\d{2}$/.test(p)) return 'month'
|
||||
if (/^\d{4}-\d{2}-\d{2}$/.test(p)) return 'day'
|
||||
return 'month'
|
||||
}
|
||||
|
||||
// 统一 row 的时间字段为 period
|
||||
function periodOf(row) {
|
||||
if (!row) return ''
|
||||
return String(row.period !== undefined ? row.period : (row.month !== undefined ? row.month : row.year))
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const conn = await mysql.createConnection(parseUrl(process.env.DATABASE_URL))
|
||||
try {
|
||||
const [rows] = await conn.execute('SELECT id, name, groupName, data FROM Indicator')
|
||||
console.log('[Migrate] 读取', rows.length, '个 Indicator')
|
||||
|
||||
let points = 0, updated = 0
|
||||
for (const ind of rows) {
|
||||
// mysql2 对 JSON 列已自动解析为对象, 兼容字符串兜底
|
||||
let data = ind.data
|
||||
if (typeof data === 'string') {
|
||||
try { data = JSON.parse(data) } catch { data = null }
|
||||
}
|
||||
const title = (data && data.title) || ind.name
|
||||
const unit = (data && data.unit) || ''
|
||||
const source = (data && data.source) || ''
|
||||
const rowsArr = (data && Array.isArray(data.rows)) ? data.rows : []
|
||||
const freq = rowsArr.length ? inferFrequency(rowsArr[0]) : 'month'
|
||||
|
||||
// 更新 Indicator 元数据列
|
||||
await conn.execute(
|
||||
'UPDATE Indicator SET title=?, unit=?, source=?, frequency=?, updatedAt=NOW(3) WHERE id=?',
|
||||
[title, unit, source, freq, ind.id],
|
||||
)
|
||||
updated++
|
||||
|
||||
// 写入明细
|
||||
for (const r of rowsArr) {
|
||||
const period = periodOf(r)
|
||||
const value = Number(r.value)
|
||||
if (!period || Number.isNaN(value)) continue
|
||||
const [res] = await conn.execute(
|
||||
'INSERT INTO IndicatorPoint (indicatorId, period, value, createdAt, updatedAt) VALUES (?, ?, ?, NOW(3), NOW(3)) ' +
|
||||
'ON DUPLICATE KEY UPDATE value=VALUES(value), updatedAt=NOW(3)',
|
||||
[ind.id, period, value],
|
||||
)
|
||||
points += (res.affectedRows > 0 ? 1 : 0)
|
||||
}
|
||||
console.log('[Migrate]', ind.name, '->', rowsArr.length, '个点, 频率', freq)
|
||||
}
|
||||
console.log('[Migrate] 完成: 更新', updated, '个 Indicator, 写入/更新', points, '个明细点')
|
||||
} finally {
|
||||
await conn.end()
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((e) => { console.error('[Migrate] 失败:', e); process.exit(1) })
|
||||
@@ -1,185 +0,0 @@
|
||||
/**
|
||||
* 拉所有启用的数据源, upsert 到 Indicator + IndicatorPoint 明细表
|
||||
* 用法:
|
||||
* npm run data:refresh # 只拉 DB 里没有的指标
|
||||
* npm run data:refresh -- --force a,b,c # 强制刷新指定指标 (忽略 DB 已有)
|
||||
*
|
||||
* 数据结构 (2026-08 重构):
|
||||
* Indicator 元数据: name/groupName/title/unit/source/frequency/hidden
|
||||
* IndicatorPoint 明细: (indicatorId, period, value), period 兼容年/季/月/日
|
||||
*
|
||||
* DB 优先策略:
|
||||
* 1. 先从 DB 查已存在的 name 集合
|
||||
* 2. 把集合传给每个 fetcher, fetcher 跳过已存在的
|
||||
* 3. 只对 DB 没有的指标调远程拉取, 节省 token (尤其妙想 MCP 这种 LLM 接口)
|
||||
*
|
||||
* 加新数据源: 写 scripts/fetchers/<name>.fetcher.js 导出 META + fetchAll(opts),
|
||||
* 然后在 scripts/fetchers/index.js 的 FETCHERS 注册
|
||||
*/
|
||||
require('dotenv').config()
|
||||
const mysql = require('mysql2/promise')
|
||||
|
||||
// 隐藏指标: 数据与获取脚本保留, 但不参与前端展示 (API 过滤)
|
||||
const HIDDEN_INDICATORS = new Set([
|
||||
'china-gdp-usd', // 中国 GDP (现价美元)
|
||||
'china-gdp-growth', // 中国 GDP 同比 (World Bank)
|
||||
'china-gdp-yoy', // 中国 GDP 同比 (妙想季度)
|
||||
'china-unemploy', // 中国失业率
|
||||
'usa-gdp-usd', // 美国 GDP
|
||||
])
|
||||
|
||||
function parseUrl(url) {
|
||||
const u = new URL(url)
|
||||
return {
|
||||
host: u.hostname,
|
||||
port: Number(u.port) || 3306,
|
||||
user: decodeURIComponent(u.username),
|
||||
password: decodeURIComponent(u.password),
|
||||
database: u.pathname.replace(/^\//, ''),
|
||||
}
|
||||
}
|
||||
|
||||
function parseForceArg(argv) {
|
||||
const i = argv.indexOf('--force')
|
||||
if (i === -1) return new Set()
|
||||
return new Set(String(argv[i + 1] || '').split(',').map((s) => s.trim()).filter(Boolean))
|
||||
}
|
||||
|
||||
// 推断频率: year | quarter | month | day (优先用 fetcher 显式声明)
|
||||
function inferFrequency(data) {
|
||||
if (data && data.frequency) return data.frequency
|
||||
const p = String(((data && data.rows) || [])[0]?.period || '')
|
||||
if (/^\d{4}$/.test(p)) return 'year'
|
||||
if (/^\d{4}-Q[1-4]$/.test(p)) return 'quarter'
|
||||
if (/^\d{4}-\d{2}$/.test(p)) return 'month'
|
||||
if (/^\d{4}-\d{2}-\d{2}$/.test(p)) return 'day'
|
||||
return 'month'
|
||||
}
|
||||
|
||||
// 写 Indicator 元数据 + 全量替换明细点 (每指标一个事务)
|
||||
async function upsertIndicatorWithPoints(conn, payload) {
|
||||
const { name, groupName, data, hidden } = payload
|
||||
const title = data.title || name
|
||||
const unit = data.unit || ''
|
||||
const source = data.source || ''
|
||||
const frequency = inferFrequency(data)
|
||||
const rows = Array.isArray(data.rows) ? data.rows : []
|
||||
|
||||
await conn.beginTransaction()
|
||||
try {
|
||||
const [res] = await conn.execute(
|
||||
'INSERT INTO `Indicator` (`name`, `groupName`, `title`, `unit`, `source`, `frequency`, `hidden`, `createdAt`, `updatedAt`) ' +
|
||||
'VALUES (?, ?, ?, ?, ?, ?, ?, NOW(3), NOW(3)) ' +
|
||||
'ON DUPLICATE KEY UPDATE `groupName` = VALUES(`groupName`), `title` = VALUES(`title`), ' +
|
||||
'`unit` = VALUES(`unit`), `source` = VALUES(`source`), `frequency` = VALUES(`frequency`), ' +
|
||||
'`hidden` = VALUES(`hidden`), `updatedAt` = NOW(3)',
|
||||
[name, groupName, title, unit, source, frequency, hidden ? 1 : 0],
|
||||
)
|
||||
const indicatorId = res.insertId
|
||||
|
||||
// 明细: 先删旧点再插入新点 (全量替换, 保证与数据源一致)
|
||||
await conn.execute('DELETE FROM `IndicatorPoint` WHERE `indicatorId` = ?', [indicatorId])
|
||||
let points = 0
|
||||
for (const r of rows) {
|
||||
const period = String(r.period !== undefined ? r.period : (r.month !== undefined ? r.month : r.year))
|
||||
const value = Number(r.value)
|
||||
if (!period || Number.isNaN(value)) continue
|
||||
await conn.execute(
|
||||
'INSERT INTO `IndicatorPoint` (`indicatorId`, `period`, `value`, `createdAt`, `updatedAt`) VALUES (?, ?, ?, NOW(3), NOW(3))',
|
||||
[indicatorId, period, value],
|
||||
)
|
||||
points++
|
||||
}
|
||||
await conn.commit()
|
||||
return { indicatorId, points }
|
||||
} catch (e) {
|
||||
await conn.rollback()
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchExistingNames(conn) {
|
||||
const [rows] = await conn.execute('SELECT `name` FROM `Indicator`')
|
||||
return new Set(rows.map((r) => r.name))
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (!process.env.DATABASE_URL) {
|
||||
console.error('缺少 DATABASE_URL 环境变量')
|
||||
process.exit(1)
|
||||
}
|
||||
const force = parseForceArg(process.argv.slice(2))
|
||||
if (force.size) console.log('[Refresh] 强制刷新: ' + [...force].join(', '))
|
||||
|
||||
const conn = await mysql.createConnection(parseUrl(process.env.DATABASE_URL))
|
||||
try {
|
||||
// 1) 查 DB 已存在的指标名
|
||||
const existing = await fetchExistingNames(conn)
|
||||
console.log('[Refresh] DB 已有 ' + existing.size + ' 个指标: ' + ([...existing].join(', ') || '(空)'))
|
||||
|
||||
// 2) 跑所有 fetcher, 传入 existing + force (DB 优先, force 覆盖)
|
||||
const { runFetchers } = await import('./fetchers/index.js')
|
||||
const fetcherResults = await runFetchers({ skipNames: existing, forceNames: force })
|
||||
|
||||
// 3) 汇总: force 名单里的指标即使 DB 已有也写入
|
||||
const allItems = []
|
||||
let skipped = 0, failed = 0
|
||||
for (const r of fetcherResults) {
|
||||
if (r.skipped) { skipped++; continue }
|
||||
if (!r.ok) { failed++; continue }
|
||||
for (const it of r.items) {
|
||||
if (existing.has(it.name) && !force.has(it.name)) continue
|
||||
allItems.push(it)
|
||||
}
|
||||
}
|
||||
console.log('\n[Refresh] 汇总: ' + allItems.length + ' 个指标待写 (跳过 ' + skipped + ' 个数据源, 失败 ' + failed + ' 个, DB 已存在 ' + existing.size + ' 个)')
|
||||
|
||||
if (failed > 0) {
|
||||
console.error('[Refresh] 警告: 有数据源失败, 仅写入成功的指标')
|
||||
}
|
||||
|
||||
// 5) 同步隐藏标记: 即使指标被 DB 优先跳过, 名单内的也确保 hidden=true
|
||||
if (HIDDEN_INDICATORS.size > 0) {
|
||||
const names = [...HIDDEN_INDICATORS]
|
||||
const ph = names.map(() => '?').join(',')
|
||||
const sql = "UPDATE `Indicator` SET `hidden` = 1, `updatedAt` = NOW(3) WHERE `name` IN (" + ph + ")"
|
||||
await conn.execute(sql, names)
|
||||
console.log('[Refresh] 同步隐藏标记: ' + names.length + ' 个指标 hidden=true (数据保留, 不展示)')
|
||||
}
|
||||
|
||||
// 6) 跨源对比报警: 全量跑一次 (即便无新指标也跑, 防止上次漏检)
|
||||
try {
|
||||
const m = await import('../server/alert-engine.js')
|
||||
const a = await m.checkAfterRefresh(conn)
|
||||
console.log('[Refresh] 报警检测: 新增 ' + a.inserted + ' 条 (扫描 ' + a.scanned + ')')
|
||||
} catch (e) { console.error('[Refresh] 报警检测异常:', e.message) }
|
||||
|
||||
if (allItems.length === 0) {
|
||||
console.log('[Refresh] 无指标待写, 不写库 (DB 优先策略生效)')
|
||||
return
|
||||
}
|
||||
|
||||
// 4) 写库
|
||||
console.log('[Refresh] 写入数据库...')
|
||||
let ins = 0, upd = 0, totalPoints = 0
|
||||
for (const it of allItems) {
|
||||
const hidden = HIDDEN_INDICATORS.has(it.name)
|
||||
const { points } = await upsertIndicatorWithPoints(conn, { name: it.name, groupName: it.groupName, data: it.data, hidden })
|
||||
if (points > 0) ins++
|
||||
else upd++
|
||||
totalPoints += points
|
||||
console.log('[Refresh] ' + it.name + (hidden ? ' (hidden)' : '') + ': ' + points + ' 个明细点')
|
||||
}
|
||||
|
||||
|
||||
} finally {
|
||||
await conn.end()
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
console.error('[Refresh] 异常:', e)
|
||||
process.exit(1)
|
||||
})
|
||||
|
||||
|
||||
@@ -1,338 +0,0 @@
|
||||
import { createPool } from 'mariadb';
|
||||
import { fetch as fetchSH300Data } from '../fetchers/sh300.fetcher.js';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
let pool;
|
||||
|
||||
function loadDatabaseConfig() {
|
||||
const envPath = path.join(process.cwd(), '.env');
|
||||
if (fs.existsSync(envPath)) {
|
||||
console.log('读取 .env 文件...');
|
||||
const envContent = fs.readFileSync(envPath, 'utf-8');
|
||||
const lines = envContent.split('\n');
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmedLine = line.trim();
|
||||
if (trimmedLine.startsWith('DATABASE_URL=')) {
|
||||
const url = trimmedLine.substring('DATABASE_URL='.length).replace(/^"|"$/g, '');
|
||||
console.log('从 .env 文件获取数据库配置成功!');
|
||||
console.log(`DATABASE_URL: ${url}`);
|
||||
|
||||
// 解析URL
|
||||
const parts = url.split('://')[1].split('@');
|
||||
const auth = parts[0].split(':');
|
||||
const hostAndDb = parts[1].split('/');
|
||||
const hostAndPort = hostAndDb[0].split(':');
|
||||
|
||||
return {
|
||||
user: auth[0],
|
||||
password: auth[1],
|
||||
host: hostAndPort[0],
|
||||
port: parseInt(hostAndPort[1]),
|
||||
database: hostAndDb[1]
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log('未找到 .env 文件或 DATABASE_URL 配置，使用默认配置');
|
||||
return {
|
||||
host: '192.168.111.111',
|
||||
port: 3306,
|
||||
user: 'root',
|
||||
password: 'fullstack',
|
||||
database: 'iboard'
|
||||
};
|
||||
}
|
||||
|
||||
async function initPool() {
|
||||
if (!pool) {
|
||||
console.log('初始化数据库连接池...');
|
||||
const config = loadDatabaseConfig();
|
||||
|
||||
console.log('数据库配置:');
|
||||
console.log(` 主机: ${config.host}`);
|
||||
console.log(` 端口: ${config.port}`);
|
||||
console.log(` 用户: ${config.user}`);
|
||||
console.log(` 数据库: ${config.database}`);
|
||||
|
||||
pool = createPool({
|
||||
host: config.host,
|
||||
port: config.port,
|
||||
user: config.user,
|
||||
password: config.password,
|
||||
database: config.database,
|
||||
connectionLimit: 5
|
||||
});
|
||||
|
||||
try {
|
||||
const conn = await pool.getConnection();
|
||||
console.log('数据库连接成功!');
|
||||
conn.release();
|
||||
} catch (error) {
|
||||
console.error('数据库连接失败:', error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
return pool;
|
||||
}
|
||||
|
||||
async function checkAndFillSH300Data(startDate = null, endDate = null) {
|
||||
try {
|
||||
await initPool();
|
||||
|
||||
console.log('='.repeat(60));
|
||||
console.log('开始检查和补充沪深300指数数据...');
|
||||
console.log('='.repeat(60));
|
||||
|
||||
const currentDate = new Date();
|
||||
|
||||
if (!startDate) {
|
||||
startDate = new Date(currentDate);
|
||||
startDate.setDate(startDate.getDate() - 30);
|
||||
startDate = startDate.toISOString().split('T')[0];
|
||||
}
|
||||
|
||||
if (!endDate) {
|
||||
endDate = currentDate.toISOString().split('T')[0];
|
||||
}
|
||||
|
||||
console.log(`检查时间范围: ${startDate} 至 ${endDate}`);
|
||||
|
||||
// 1. 检查数据库中是否存在沪深300指数数据表
|
||||
const hasTable = await checkTableExists();
|
||||
if (!hasTable) {
|
||||
console.log('创建沪深300指数数据表...');
|
||||
await createSH300Table();
|
||||
}
|
||||
|
||||
// 2. 检查数据库中已有的数据
|
||||
const existingData = await getExistingSH300Data(startDate, endDate);
|
||||
console.log(`数据库中已有 ${existingData.length} 条数据`);
|
||||
|
||||
// 3. 计算缺失的日期
|
||||
const missingDates = calculateMissingDates(startDate, endDate, existingData);
|
||||
console.log(`缺失 ${missingDates.length} 天的数据`);
|
||||
|
||||
// 4. 如果有缺失，调用 fetcher 补足
|
||||
if (missingDates.length > 0) {
|
||||
console.log('\n开始获取缺失的沪深300指数数据...');
|
||||
|
||||
const firstMissingDate = missingDates[0];
|
||||
const lastMissingDate = missingDates[missingDates.length - 1];
|
||||
|
||||
console.log(`获取范围: ${firstMissingDate} 至 ${lastMissingDate}`);
|
||||
|
||||
const fetchedData = await fetchSH300Data(firstMissingDate, lastMissingDate);
|
||||
|
||||
console.log(`\n获取到 ${fetchedData.count} 条数据`);
|
||||
|
||||
// 5. 存储新数据到数据库
|
||||
const storedCount = await storeSH300Data(fetchedData.data);
|
||||
console.log(`成功存储 ${storedCount} 条数据`);
|
||||
} else {
|
||||
console.log('\n数据库数据完整，无需补充');
|
||||
}
|
||||
|
||||
// 6. 验证最终数据完整性
|
||||
const finalData = await getExistingSH300Data(startDate, endDate);
|
||||
console.log(`\n验证后的数据总量: ${finalData.length} 条`);
|
||||
|
||||
return {
|
||||
startDate,
|
||||
endDate,
|
||||
existingCount: existingData.length,
|
||||
missingCount: missingDates.length,
|
||||
finalCount: finalData.length,
|
||||
success: true
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
console.error('检查和补充沪深300指数数据失败:', error.message);
|
||||
throw error;
|
||||
} finally {
|
||||
if (pool) {
|
||||
await pool.end();
|
||||
console.log('数据库连接池已关闭');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function checkTableExists() {
|
||||
try {
|
||||
console.log('检查沪深300指数数据表是否存在...');
|
||||
const result = await pool.query('SHOW TABLES LIKE ?', ['econ_SH300Index']);
|
||||
return result.length > 0;
|
||||
} catch (error) {
|
||||
console.error('检查表存在失败:', error.message);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function createSH300Table() {
|
||||
try {
|
||||
console.log('执行创建沪深300指数数据表SQL...');
|
||||
await pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS econ_SH300Index (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
date DATE UNIQUE NOT NULL,
|
||||
symbol VARCHAR(10) NOT NULL,
|
||||
open DECIMAL(10,2) NOT NULL,
|
||||
close DECIMAL(10,2) NOT NULL,
|
||||
high DECIMAL(10,2) NOT NULL,
|
||||
low DECIMAL(10,2) NOT NULL,
|
||||
volume DECIMAL(20,2) NOT NULL,
|
||||
amount DECIMAL(20,2) NOT NULL,
|
||||
source VARCHAR(100) NOT NULL,
|
||||
createdAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updatedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||||
`);
|
||||
console.log('沪深300指数数据表创建成功');
|
||||
} catch (error) {
|
||||
console.error('创建沪深300指数数据表失败:', error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function getExistingSH300Data(startDate, endDate) {
|
||||
try {
|
||||
console.log('查询已有沪深300指数数据...');
|
||||
const result = await pool.query(
|
||||
'SELECT date FROM econ_SH300Index WHERE date BETWEEN ? AND ? ORDER BY date',
|
||||
[startDate, endDate]
|
||||
);
|
||||
const dates = result.map(row => {
|
||||
if (row.date instanceof Date) {
|
||||
return row.date.toISOString().split('T')[0];
|
||||
} else if (typeof row.date === 'string') {
|
||||
return row.date;
|
||||
}
|
||||
return null;
|
||||
}).filter(Boolean);
|
||||
console.log(`查询到 ${dates.length} 条数据`);
|
||||
return dates;
|
||||
} catch (error) {
|
||||
console.error('查询已有沪深300指数数据失败:', error.message);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function calculateMissingDates(startDate, endDate, existingDates) {
|
||||
const existingSet = new Set(existingDates);
|
||||
const missingDates = [];
|
||||
|
||||
const currentDate = new Date(startDate);
|
||||
const finalDate = new Date(endDate);
|
||||
|
||||
while (currentDate <= finalDate) {
|
||||
const dateStr = currentDate.toISOString().split('T')[0];
|
||||
const dayOfWeek = currentDate.getDay();
|
||||
|
||||
// 跳过周末（周末没有股市数据）
|
||||
if (dayOfWeek !== 0 && dayOfWeek !== 6) {
|
||||
if (!existingSet.has(dateStr)) {
|
||||
missingDates.push(dateStr);
|
||||
}
|
||||
}
|
||||
|
||||
currentDate.setDate(currentDate.getDate() + 1);
|
||||
}
|
||||
|
||||
return missingDates;
|
||||
}
|
||||
|
||||
async function storeSH300Data(data) {
|
||||
let storedCount = 0;
|
||||
|
||||
for (const item of data) {
|
||||
try {
|
||||
await pool.query(
|
||||
'INSERT INTO econ_SH300Index (date, symbol, open, close, high, low, volume, amount, source) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE open = VALUES(open), close = VALUES(close), high = VALUES(high), low = VALUES(low), volume = VALUES(volume), amount = VALUES(amount), source = VALUES(source)',
|
||||
[item.date, 'sh000300', item.open, item.close, item.high, item.low, item.volume, item.amount, item.source]
|
||||
);
|
||||
storedCount++;
|
||||
} catch (error) {
|
||||
console.error(`存储沪深300指数数据 ${item.date} 失败:`, error.message);
|
||||
}
|
||||
}
|
||||
|
||||
return storedCount;
|
||||
}
|
||||
|
||||
export async function trigger(startDate = null, endDate = null) {
|
||||
return await checkAndFillSH300Data(startDate, endDate);
|
||||
}
|
||||
|
||||
function parseCommandLineArgs() {
|
||||
const args = process.argv.slice(2);
|
||||
let startDate = null;
|
||||
let endDate = null;
|
||||
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
if (args[i] === '--start' && i + 1 < args.length) {
|
||||
startDate = args[i + 1];
|
||||
i++;
|
||||
} else if (args[i] === '--end' && i + 1 < args.length) {
|
||||
endDate = args[i + 1];
|
||||
i++;
|
||||
} else if (args[i] === '--help') {
|
||||
console.log('使用方法:');
|
||||
console.log(' bun sh300.scraper.js [选项]');
|
||||
console.log('');
|
||||
console.log('选项:');
|
||||
console.log(' --start YYYY-MM-DD 指定开始日期');
|
||||
console.log(' --end YYYY-MM-DD 指定结束日期');
|
||||
console.log(' --help 显示帮助信息');
|
||||
console.log('');
|
||||
console.log('示例:');
|
||||
console.log(' bun sh300.scraper.js');
|
||||
console.log(' bun sh300.scraper.js --start 2024-01-01 --end 2024-01-31');
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
return { startDate, endDate };
|
||||
}
|
||||
|
||||
async function runTest() {
|
||||
console.log('='.repeat(60));
|
||||
console.log('测试 sh300.scraper.js');
|
||||
console.log('='.repeat(60));
|
||||
|
||||
const { startDate, endDate } = parseCommandLineArgs();
|
||||
|
||||
if (startDate) {
|
||||
console.log(`\n使用命令行参数:`);
|
||||
console.log(` 开始日期: ${startDate}`);
|
||||
console.log(` 结束日期: ${endDate || '默认'}`);
|
||||
}
|
||||
|
||||
console.log('\n测试数据库配置读取...');
|
||||
const config = loadDatabaseConfig();
|
||||
console.log('配置读取结果:');
|
||||
console.log(` 主机: ${config.host}`);
|
||||
console.log(` 端口: ${config.port}`);
|
||||
console.log(` 用户: ${config.user}`);
|
||||
console.log(` 数据库: ${config.database}`);
|
||||
console.log('配置读取测试完成!');
|
||||
|
||||
try {
|
||||
const result = await trigger(startDate, endDate);
|
||||
console.log('\n' + '='.repeat(60));
|
||||
console.log('测试结果:');
|
||||
console.log(` 开始日期: ${result.startDate}`);
|
||||
console.log(` 结束日期: ${result.endDate}`);
|
||||
console.log(` 原有数据: ${result.existingCount} 条`);
|
||||
console.log(` 缺失数据: ${result.missingCount} 条`);
|
||||
console.log(` 最终数据: ${result.finalCount} 条`);
|
||||
console.log(` 操作成功: ${result.success}`);
|
||||
console.log('='.repeat(60));
|
||||
} catch (error) {
|
||||
console.error('测试失败:', error);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
runTest();
|
||||
@@ -1,335 +0,0 @@
|
||||
import { createPool } from 'mariadb';
|
||||
import { fetch as fetchUSDToCNYRate } from '../fetchers/usd-cny-rate.fetcher.js';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
let pool;
|
||||
|
||||
function loadDatabaseConfig() {
|
||||
const envPath = path.join(process.cwd(), '.env');
|
||||
if (fs.existsSync(envPath)) {
|
||||
console.log('读取 .env 文件...');
|
||||
const envContent = fs.readFileSync(envPath, 'utf-8');
|
||||
const lines = envContent.split('\n');
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmedLine = line.trim();
|
||||
if (trimmedLine.startsWith('DATABASE_URL=')) {
|
||||
const url = trimmedLine.substring('DATABASE_URL='.length).replace(/^"|"$/g, '');
|
||||
console.log('从 .env 文件获取数据库配置成功!');
|
||||
console.log(`DATABASE_URL: ${url}`);
|
||||
|
||||
// 解析URL
|
||||
const parts = url.split('://')[1].split('@');
|
||||
const auth = parts[0].split(':');
|
||||
const hostAndDb = parts[1].split('/');
|
||||
const hostAndPort = hostAndDb[0].split(':');
|
||||
|
||||
return {
|
||||
user: auth[0],
|
||||
password: auth[1],
|
||||
host: hostAndPort[0],
|
||||
port: parseInt(hostAndPort[1]),
|
||||
database: hostAndDb[1]
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log('未找到 .env 文件或 DATABASE_URL 配置，使用默认配置');
|
||||
return {
|
||||
host: '192.168.111.111',
|
||||
port: 3306,
|
||||
user: 'root',
|
||||
password: 'fullstack',
|
||||
database: 'iboard'
|
||||
};
|
||||
}
|
||||
|
||||
async function initPool() {
|
||||
if (!pool) {
|
||||
console.log('初始化数据库连接池...');
|
||||
const config = loadDatabaseConfig();
|
||||
|
||||
console.log('数据库配置:');
|
||||
console.log(` 主机: ${config.host}`);
|
||||
console.log(` 端口: ${config.port}`);
|
||||
console.log(` 用户: ${config.user}`);
|
||||
console.log(` 数据库: ${config.database}`);
|
||||
|
||||
pool = createPool({
|
||||
host: config.host,
|
||||
port: config.port,
|
||||
user: config.user,
|
||||
password: config.password,
|
||||
database: config.database,
|
||||
connectionLimit: 5
|
||||
});
|
||||
|
||||
try {
|
||||
const conn = await pool.getConnection();
|
||||
console.log('数据库连接成功!');
|
||||
conn.release();
|
||||
} catch (error) {
|
||||
console.error('数据库连接失败:', error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
return pool;
|
||||
}
|
||||
|
||||
async function checkAndFillUSDCNYRate(startDate = null, endDate = null) {
|
||||
try {
|
||||
await initPool();
|
||||
|
||||
console.log('='.repeat(60));
|
||||
console.log('开始检查和补充人民币美元汇率数据...');
|
||||
console.log('='.repeat(60));
|
||||
|
||||
const currentDate = new Date();
|
||||
|
||||
if (!startDate) {
|
||||
startDate = new Date(currentDate);
|
||||
startDate.setDate(startDate.getDate() - 30);
|
||||
startDate = startDate.toISOString().split('T')[0];
|
||||
}
|
||||
|
||||
if (!endDate) {
|
||||
endDate = currentDate.toISOString().split('T')[0];
|
||||
}
|
||||
|
||||
console.log(`检查时间范围: ${startDate} 至 ${endDate}`);
|
||||
|
||||
// 1. 检查数据库中是否存在汇率数据表
|
||||
const hasTable = await checkTableExists();
|
||||
if (!hasTable) {
|
||||
console.log('创建汇率数据表...');
|
||||
await createExchangeRateTable();
|
||||
}
|
||||
|
||||
// 2. 检查数据库中已有的数据
|
||||
const existingData = await getExistingExchangeRates(startDate, endDate);
|
||||
console.log(`数据库中已有 ${existingData.length} 条数据`);
|
||||
|
||||
// 3. 计算缺失的日期
|
||||
const missingDates = calculateMissingDates(startDate, endDate, existingData);
|
||||
console.log(`缺失 ${missingDates.length} 天的数据`);
|
||||
|
||||
// 4. 如果有缺失，调用 fetcher 补足
|
||||
if (missingDates.length > 0) {
|
||||
console.log('\n开始获取缺失的汇率数据...');
|
||||
|
||||
const firstMissingDate = missingDates[0];
|
||||
const lastMissingDate = missingDates[missingDates.length - 1];
|
||||
|
||||
console.log(`获取范围: ${firstMissingDate} 至 ${lastMissingDate}`);
|
||||
|
||||
const fetchedData = await fetchUSDToCNYRate(firstMissingDate, lastMissingDate);
|
||||
|
||||
console.log(`\n获取到 ${fetchedData.count} 条数据`);
|
||||
|
||||
// 5. 存储新数据到数据库
|
||||
const storedCount = await storeExchangeRates(fetchedData.data);
|
||||
console.log(`成功存储 ${storedCount} 条数据`);
|
||||
} else {
|
||||
console.log('\n数据库数据完整，无需补充');
|
||||
}
|
||||
|
||||
// 6. 验证最终数据完整性
|
||||
const finalData = await getExistingExchangeRates(startDate, endDate);
|
||||
console.log(`\n验证后的数据总量: ${finalData.length} 条`);
|
||||
|
||||
return {
|
||||
startDate,
|
||||
endDate,
|
||||
existingCount: existingData.length,
|
||||
missingCount: missingDates.length,
|
||||
finalCount: finalData.length,
|
||||
success: true
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
console.error('检查和补充汇率数据失败:', error.message);
|
||||
throw error;
|
||||
} finally {
|
||||
if (pool) {
|
||||
await pool.end();
|
||||
console.log('数据库连接池已关闭');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function checkTableExists() {
|
||||
try {
|
||||
console.log('检查汇率数据表是否存在...');
|
||||
const result = await pool.query('SHOW TABLES LIKE ?', ['econ_ExchangeRate']);
|
||||
return result.length > 0;
|
||||
} catch (error) {
|
||||
console.error('检查表存在失败:', error.message);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function createExchangeRateTable() {
|
||||
try {
|
||||
console.log('执行创建汇率数据表SQL...');
|
||||
await pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS econ_ExchangeRate (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
date DATE UNIQUE NOT NULL,
|
||||
currency VARCHAR(10) NOT NULL,
|
||||
centerPrice DECIMAL(10,4) NOT NULL,
|
||||
sellingRate DECIMAL(10,4) NOT NULL,
|
||||
buyingRate DECIMAL(10,4) NOT NULL,
|
||||
source VARCHAR(100) NOT NULL,
|
||||
createdAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updatedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||||
`);
|
||||
console.log('汇率数据表创建成功');
|
||||
} catch (error) {
|
||||
console.error('创建汇率数据表失败:', error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function getExistingExchangeRates(startDate, endDate) {
|
||||
try {
|
||||
console.log('查询已有汇率数据...');
|
||||
const result = await pool.query(
|
||||
'SELECT date FROM econ_ExchangeRate WHERE date BETWEEN ? AND ? ORDER BY date',
|
||||
[startDate, endDate]
|
||||
);
|
||||
const dates = result.map(row => {
|
||||
if (row.date instanceof Date) {
|
||||
return row.date.toISOString().split('T')[0];
|
||||
} else if (typeof row.date === 'string') {
|
||||
return row.date;
|
||||
}
|
||||
return null;
|
||||
}).filter(Boolean);
|
||||
console.log(`查询到 ${dates.length} 条数据`);
|
||||
return dates;
|
||||
} catch (error) {
|
||||
console.error('查询已有汇率数据失败:', error.message);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function calculateMissingDates(startDate, endDate, existingDates) {
|
||||
const existingSet = new Set(existingDates);
|
||||
const missingDates = [];
|
||||
|
||||
const currentDate = new Date(startDate);
|
||||
const finalDate = new Date(endDate);
|
||||
|
||||
while (currentDate <= finalDate) {
|
||||
const dateStr = currentDate.toISOString().split('T')[0];
|
||||
const dayOfWeek = currentDate.getDay();
|
||||
|
||||
// 跳过周末（周末没有汇率数据）
|
||||
if (dayOfWeek !== 0 && dayOfWeek !== 6) {
|
||||
if (!existingSet.has(dateStr)) {
|
||||
missingDates.push(dateStr);
|
||||
}
|
||||
}
|
||||
|
||||
currentDate.setDate(currentDate.getDate() + 1);
|
||||
}
|
||||
|
||||
return missingDates;
|
||||
}
|
||||
|
||||
async function storeExchangeRates(rates) {
|
||||
let storedCount = 0;
|
||||
|
||||
for (const rate of rates) {
|
||||
try {
|
||||
await pool.query(
|
||||
'INSERT INTO econ_ExchangeRate (date, currency, centerPrice, sellingRate, buyingRate, source) VALUES (?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE centerPrice = VALUES(centerPrice), sellingRate = VALUES(sellingRate), buyingRate = VALUES(buyingRate), source = VALUES(source)',
|
||||
[rate.date, rate.currency, rate.centerPrice, rate.sellingRate, rate.buyingRate, rate.source]
|
||||
);
|
||||
storedCount++;
|
||||
} catch (error) {
|
||||
console.error(`存储汇率数据 ${rate.date} 失败:`, error.message);
|
||||
}
|
||||
}
|
||||
|
||||
return storedCount;
|
||||
}
|
||||
|
||||
export async function trigger(startDate = null, endDate = null) {
|
||||
return await checkAndFillUSDCNYRate(startDate, endDate);
|
||||
}
|
||||
|
||||
function parseCommandLineArgs() {
|
||||
const args = process.argv.slice(2);
|
||||
let startDate = null;
|
||||
let endDate = null;
|
||||
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
if (args[i] === '--start' && i + 1 < args.length) {
|
||||
startDate = args[i + 1];
|
||||
i++;
|
||||
} else if (args[i] === '--end' && i + 1 < args.length) {
|
||||
endDate = args[i + 1];
|
||||
i++;
|
||||
} else if (args[i] === '--help') {
|
||||
console.log('使用方法:');
|
||||
console.log(' bun usd-cny-rate.scraper.js [选项]');
|
||||
console.log('');
|
||||
console.log('选项:');
|
||||
console.log(' --start YYYY-MM-DD 指定开始日期');
|
||||
console.log(' --end YYYY-MM-DD 指定结束日期');
|
||||
console.log(' --help 显示帮助信息');
|
||||
console.log('');
|
||||
console.log('示例:');
|
||||
console.log(' bun usd-cny-rate.scraper.js');
|
||||
console.log(' bun usd-cny-rate.scraper.js --start 2024-01-01 --end 2024-01-31');
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
return { startDate, endDate };
|
||||
}
|
||||
|
||||
async function runTest() {
|
||||
console.log('='.repeat(60));
|
||||
console.log('测试 usd-cny-rate.scraper.js');
|
||||
console.log('='.repeat(60));
|
||||
|
||||
const { startDate, endDate } = parseCommandLineArgs();
|
||||
|
||||
if (startDate) {
|
||||
console.log(`\n使用命令行参数:`);
|
||||
console.log(` 开始日期: ${startDate}`);
|
||||
console.log(` 结束日期: ${endDate || '默认'}`);
|
||||
}
|
||||
|
||||
console.log('\n测试数据库配置读取...');
|
||||
const config = loadDatabaseConfig();
|
||||
console.log('配置读取结果:');
|
||||
console.log(` 主机: ${config.host}`);
|
||||
console.log(` 端口: ${config.port}`);
|
||||
console.log(` 用户: ${config.user}`);
|
||||
console.log(` 数据库: ${config.database}`);
|
||||
console.log('配置读取测试完成!');
|
||||
|
||||
try {
|
||||
const result = await trigger(startDate, endDate);
|
||||
console.log('\n' + '='.repeat(60));
|
||||
console.log('测试结果:');
|
||||
console.log(` 开始日期: ${result.startDate}`);
|
||||
console.log(` 结束日期: ${result.endDate}`);
|
||||
console.log(` 原有数据: ${result.existingCount} 条`);
|
||||
console.log(` 缺失数据: ${result.missingCount} 条`);
|
||||
console.log(` 最终数据: ${result.finalCount} 条`);
|
||||
console.log(` 操作成功: ${result.success}`);
|
||||
console.log('='.repeat(60));
|
||||
} catch (error) {
|
||||
console.error('测试失败:', error);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
runTest();
|
||||
File diff suppressed because one or more lines are too long
+86
-132
@@ -1,133 +1,87 @@
|
||||
/**
|
||||
* iBoard 基础数据初始化脚本
|
||||
* 用法: npm run db:seed
|
||||
* 幂等: 主键/唯一键冲突跳过, Strategy/Indicator 按 name 存在则更新
|
||||
*/
|
||||
|
||||
require('dotenv').config();
|
||||
const crypto = require('crypto');
|
||||
const mysql = require('mysql2/promise');
|
||||
|
||||
function hashPassword(plain) {
|
||||
const salt = crypto.randomBytes(16).toString('hex');
|
||||
const hash = crypto.scryptSync(plain, salt, 64).toString('hex');
|
||||
return 'scrypt$' + salt + '$' + hash;
|
||||
}
|
||||
|
||||
function parseUrl(url) {
|
||||
const u = new URL(url);
|
||||
return {
|
||||
host: u.hostname,
|
||||
port: Number(u.port) || 3306,
|
||||
user: decodeURIComponent(u.username),
|
||||
password: decodeURIComponent(u.password),
|
||||
database: u.pathname.replace(/^\//, ''),
|
||||
};
|
||||
}
|
||||
|
||||
async function insertIgnore(conn, sql, params) {
|
||||
try {
|
||||
const [r] = await conn.execute(sql, params);
|
||||
return r.affectedRows;
|
||||
} catch (e) {
|
||||
if (e.code === 'ER_DUP_ENTRY') return 0;
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async function upsertByName(conn, table, payload, keyField) {
|
||||
const fields = Object.keys(payload);
|
||||
const placeholders = fields.map(() => '?').join(', ');
|
||||
const updateClause = fields
|
||||
.filter((f) => f !== keyField)
|
||||
.map((f) => '`' + f + '` = VALUES(`' + f + '`)')
|
||||
.join(', ');
|
||||
const sql = 'INSERT INTO `' + table + '` (' + fields.map((f) => '`' + f + '`').join(', ') + ') VALUES (' + placeholders + ') ON DUPLICATE KEY UPDATE ' + updateClause;
|
||||
const params = fields.map((f) => payload[f]);
|
||||
const [r] = await conn.execute(sql, params);
|
||||
return r.affectedRows;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const cfg = parseUrl(process.env.DATABASE_URL);
|
||||
if (!cfg.host) throw new Error('DATABASE_URL 未配置或解析失败');
|
||||
const conn = await mysql.createConnection(cfg);
|
||||
console.log('已连接:', cfg.host + ':' + cfg.port + '/' + cfg.database);
|
||||
|
||||
const adminPwd = hashPassword('admin123');
|
||||
const n1 = await insertIgnore(
|
||||
conn,
|
||||
'INSERT INTO `User` (name, email, password, createdAt, updatedAt) VALUES (?, ?, ?, NOW(3), NOW(3))',
|
||||
['Admin', 'admin@iboard.local', adminPwd],
|
||||
);
|
||||
console.log('[User] 新增管理员: ' + n1 + ' 行 (admin@iboard.local / admin123)');
|
||||
|
||||
const strategies = [
|
||||
{ name: 'us-debt-gdp', description: '美国政府债务/GDP分析',
|
||||
data: { source: 'strategy.js', unit: '万亿美元', rows: [
|
||||
{ year: '2019', debt: 22.7, gdp: 21.4, debtToGdp: 106.1, interestToGdp: 1.8 },
|
||||
{ year: '2020', debt: 26.9, gdp: 20.9, debtToGdp: 128.7, interestToGdp: 1.6 },
|
||||
{ year: '2021', debt: 28.4, gdp: 23.0, debtToGdp: 123.5, interestToGdp: 1.5 },
|
||||
{ year: '2022', debt: 30.9, gdp: 25.4, debtToGdp: 121.6, interestToGdp: 1.6 },
|
||||
{ year: '2023', debt: 33.1, gdp: 26.9, debtToGdp: 123.0, interestToGdp: 1.9 },
|
||||
{ year: '2024', debt: 35.3, gdp: 28.5, debtToGdp: 123.9, interestToGdp: 2.1 },
|
||||
]},
|
||||
},
|
||||
{ name: 'china-debt-gdp', description: '中国政府债务/GDP分析',
|
||||
data: { source: 'strategy.js', unit: '万亿美元', rows: [
|
||||
{ year: '2019', debt: 8.8, gdp: 14.3, debtToGdp: 61.5, interestToGdp: 1.9 },
|
||||
{ year: '2020', debt: 10.3, gdp: 14.7, debtToGdp: 70.1, interestToGdp: 1.8 },
|
||||
{ year: '2021', debt: 11.7, gdp: 16.1, debtToGdp: 72.7, interestToGdp: 1.7 },
|
||||
{ year: '2022', debt: 13.1, gdp: 17.9, debtToGdp: 73.2, interestToGdp: 1.6 },
|
||||
{ year: '2023', debt: 14.5, gdp: 19.4, debtToGdp: 74.7, interestToGdp: 1.6 },
|
||||
{ year: '2024', debt: 15.9, gdp: 21.0, debtToGdp: 75.7, interestToGdp: 1.5 },
|
||||
]},
|
||||
},
|
||||
];
|
||||
let n2 = 0;
|
||||
for (const s of strategies) {
|
||||
n2 += await upsertByName(conn, 'Strategy',
|
||||
{ name: s.name, description: s.description, data: JSON.stringify(s.data), updatedAt: new Date() },
|
||||
'name');
|
||||
}
|
||||
console.log('[Strategy] upsert 策略: ' + n2 + ' 行 (1=新增, 2=更新)');
|
||||
|
||||
// Indicator 表: 改由 scripts/refresh-indicators.cjs 管理 (2026-08 重构后数据存 IndicatorPoint 明细表)
|
||||
// 本脚本不再写入 Indicator, 如需初始化指标请运行 npm run data:refresh
|
||||
|
||||
const article = {
|
||||
title: '欢迎使用 iBoard',
|
||||
slug: 'welcome-to-iboard',
|
||||
content: 'iBoard 用于收集和整理经济数据，以策略为主体重新组织。\n\n本文为示例条目，正式文章仍由 content/articles/*.md 文件承载。',
|
||||
date: '2026-08-03',
|
||||
};
|
||||
const n4 = await insertIgnore(
|
||||
conn,
|
||||
'INSERT INTO `Article` (title, slug, content, date, createdAt, updatedAt) VALUES (?, ?, ?, ?, NOW(3), NOW(3))',
|
||||
[article.title, article.slug, article.content, article.date],
|
||||
);
|
||||
console.log('[Article] 新增示例文章: ' + n4 + ' 行');
|
||||
|
||||
const sowRows = [
|
||||
{ month: '2024-01', inventory: 4067 },
|
||||
{ month: '2024-02', inventory: 4042 },
|
||||
{ month: '2024-03', inventory: 3992 },
|
||||
{ month: '2024-04', inventory: 3986 },
|
||||
{ month: '2024-05', inventory: 3996 },
|
||||
{ month: '2024-06', inventory: 4038 },
|
||||
];
|
||||
let n5 = 0;
|
||||
for (const r of sowRows) {
|
||||
n5 += await insertIgnore(
|
||||
conn,
|
||||
'INSERT INTO `Econ_SowInventory` (month, inventory, createdAt, updatedAt) VALUES (?, ?, NOW(3), NOW(3))',
|
||||
[r.month, r.inventory],
|
||||
);
|
||||
}
|
||||
console.log('[Econ_SowInventory] 新增: ' + n5 + ' 行');
|
||||
|
||||
await conn.end();
|
||||
console.log('\nSeed 完成.');
|
||||
}
|
||||
|
||||
/**
|
||||
* iBoard 基础数据初始化脚本
|
||||
* 用法: npm run db:seed
|
||||
* 幂等: 主键/唯一键冲突跳过
|
||||
*/
|
||||
|
||||
require('dotenv').config();
|
||||
const crypto = require('crypto');
|
||||
const mysql = require('mysql2/promise');
|
||||
|
||||
function hashPassword(plain) {
|
||||
const salt = crypto.randomBytes(16).toString('hex');
|
||||
const hash = crypto.scryptSync(plain, salt, 64).toString('hex');
|
||||
return 'scrypt$' + salt + '$' + hash;
|
||||
}
|
||||
|
||||
function parseUrl(url) {
|
||||
const u = new URL(url);
|
||||
return {
|
||||
host: u.hostname,
|
||||
port: Number(u.port) || 3306,
|
||||
user: decodeURIComponent(u.username),
|
||||
password: decodeURIComponent(u.password),
|
||||
database: u.pathname.replace(/^\//, ''),
|
||||
};
|
||||
}
|
||||
|
||||
async function insertIgnore(conn, sql, params) {
|
||||
try {
|
||||
const [r] = await conn.execute(sql, params);
|
||||
return r.affectedRows;
|
||||
} catch (e) {
|
||||
if (e.code === 'ER_DUP_ENTRY') return 0;
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const cfg = parseUrl(process.env.DATABASE_URL);
|
||||
if (!cfg.host) throw new Error('DATABASE_URL 未配置或解析失败');
|
||||
const conn = await mysql.createConnection(cfg);
|
||||
console.log('已连接:', cfg.host + ':' + cfg.port + '/' + cfg.database);
|
||||
|
||||
const adminPwd = hashPassword('admin123');
|
||||
const n1 = await insertIgnore(
|
||||
conn,
|
||||
'INSERT INTO `User` (name, email, password, createdAt, updatedAt) VALUES (?, ?, ?, NOW(3), NOW(3))',
|
||||
['Admin', 'admin@iboard.local', adminPwd],
|
||||
);
|
||||
console.log('[User] 新增管理员: ' + n1 + ' 行 (admin@iboard.local / admin123)');
|
||||
|
||||
const article = {
|
||||
title: '欢迎使用 iBoard',
|
||||
slug: 'welcome-to-iboard',
|
||||
content: 'iBoard 用于收集和整理经济数据，以策略为主体重新组织。\n\n本文为示例条目，正式文章仍由 content/articles/*.md 文件承载。',
|
||||
date: '2026-08-03',
|
||||
};
|
||||
const n4 = await insertIgnore(
|
||||
conn,
|
||||
'INSERT INTO `Article` (title, slug, content, date, createdAt, updatedAt) VALUES (?, ?, ?, ?, NOW(3), NOW(3))',
|
||||
[article.title, article.slug, article.content, article.date],
|
||||
);
|
||||
console.log('[Article] 新增示例文章: ' + n4 + ' 行');
|
||||
|
||||
const sowRows = [
|
||||
{ month: '2024-01', inventory: 4067 },
|
||||
{ month: '2024-02', inventory: 4042 },
|
||||
{ month: '2024-03', inventory: 3992 },
|
||||
{ month: '2024-04', inventory: 3986 },
|
||||
{ month: '2024-05', inventory: 3996 },
|
||||
{ month: '2024-06', inventory: 4038 },
|
||||
];
|
||||
let n5 = 0;
|
||||
for (const r of sowRows) {
|
||||
n5 += await insertIgnore(
|
||||
conn,
|
||||
'INSERT INTO `Econ_SowInventory` (month, inventory, createdAt, updatedAt) VALUES (?, ?, NOW(3), NOW(3))',
|
||||
[r.month, r.inventory],
|
||||
);
|
||||
}
|
||||
console.log('[Econ_SowInventory] 新增: ' + n5 + ' 行');
|
||||
|
||||
await conn.end();
|
||||
console.log('\nSeed 完成.');
|
||||
}
|
||||
|
||||
main().catch((e) => { console.error('Seed 失败:', e); process.exit(1); });
|
||||
@@ -1,22 +0,0 @@
|
||||
require('dotenv').config();
|
||||
const mysql = require('mysql2/promise');
|
||||
(async () => {
|
||||
const u = new URL(process.env.DATABASE_URL);
|
||||
const c = await mysql.createConnection({
|
||||
host: u.hostname, port: Number(u.port),
|
||||
user: decodeURIComponent(u.username),
|
||||
password: decodeURIComponent(u.password),
|
||||
database: u.pathname.replace(/^\//, ''),
|
||||
});
|
||||
const [users] = await c.query('SELECT id,name,email,LEFT(password,20) AS pwd_prefix,createdAt FROM User');
|
||||
console.log('User:'); console.table(users);
|
||||
const [sow] = await c.query('SELECT * FROM Econ_SowInventory ORDER BY month');
|
||||
console.log('Econ_SowInventory:'); console.table(sow);
|
||||
const [str] = await c.query('SELECT id,name,description,JSON_LENGTH(data,"$.rows") AS row_count FROM Strategy');
|
||||
console.log('Strategy:'); console.table(str);
|
||||
const [ind] = await c.query('SELECT id,name,groupName,JSON_LENGTH(data,"$.rows") AS row_count FROM Indicator');
|
||||
console.log('Indicator:'); console.table(ind);
|
||||
const [art] = await c.query('SELECT id,title,slug,date FROM Article');
|
||||
console.log('Article:'); console.table(art);
|
||||
await c.end();
|
||||
})().catch(e=>{console.error(e);process.exit(1)});
|
||||
@@ -1,31 +0,0 @@
|
||||
import 'dotenv/config'
|
||||
import { PrismaMariaDb } from '@prisma/adapter-mariadb'
|
||||
import { PrismaClient } from '../generated/prisma/client.ts'
|
||||
|
||||
function parseDatabaseUrl(url) {
|
||||
const u = new URL(url)
|
||||
return {
|
||||
host: u.hostname,
|
||||
port: Number(u.port) || 3306,
|
||||
user: decodeURIComponent(u.username),
|
||||
password: decodeURIComponent(u.password),
|
||||
database: u.pathname.replace(/^\//, ''),
|
||||
}
|
||||
}
|
||||
|
||||
const p = new PrismaClient({ adapter: new PrismaMariaDb(parseDatabaseUrl(process.env.DATABASE_URL)) })
|
||||
try {
|
||||
const users = await p.user.findMany()
|
||||
console.log('User count:', users.length, '->', users.map((u) => u.email).join(','))
|
||||
const strategies = await p.strategy.findMany({ select: { id: true, name: true } })
|
||||
console.log('Strategy count:', strategies.length, '->', strategies.map((s) => s.name).join(','))
|
||||
const indicators = await p.indicator.findMany({ select: { id: true, name: true } })
|
||||
console.log('Indicator count:', indicators.length, '->', indicators.map((i) => i.name).join(','))
|
||||
const sow = await p.econ_SowInventory.findMany({ orderBy: { month: 'asc' } })
|
||||
console.log('SowInventory count:', sow.length, '->', sow.map((s) => s.month + '=' + s.inventory).join(','))
|
||||
} catch (e) {
|
||||
console.error('ERR:', e.message)
|
||||
process.exitCode = 1
|
||||
} finally {
|
||||
await p.$disconnect()
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
require('dotenv').config();
|
||||
const mysql = require('mysql2/promise');
|
||||
|
||||
(async () => {
|
||||
const url = new URL(process.env.DATABASE_URL);
|
||||
const conn = await mysql.createConnection({
|
||||
host: url.hostname,
|
||||
port: Number(url.port),
|
||||
user: decodeURIComponent(url.username),
|
||||
password: decodeURIComponent(url.password),
|
||||
database: url.pathname.replace(/^\//, ''),
|
||||
});
|
||||
const [tables] = await conn.query('SHOW TABLES');
|
||||
console.log('TABLES:');
|
||||
for (const row of tables) console.log(' -', Object.values(row)[0]);
|
||||
const tableNames = ['User', 'Article', 'Strategy', 'Indicator', 'Econ_SowInventory'];
|
||||
for (const t of tableNames) {
|
||||
const [r] = await conn.query('SELECT COUNT(*) AS c FROM ' + t + '');
|
||||
console.log(t + ': ' + r[0].c + ' rows');
|
||||
}
|
||||
await conn.end();
|
||||
})().catch((e) => { console.error('ERR:', e.message); process.exit(1); });
|
||||
@@ -1,15 +0,0 @@
|
||||
const { PrismaClient } = require('../generated/prisma');
|
||||
(async () => {
|
||||
const p = new PrismaClient();
|
||||
const tables = await p.$queryRawUnsafe('SHOW TABLES');
|
||||
console.log('TABLES:', JSON.stringify(tables, null, 2));
|
||||
const counts = {
|
||||
User: await p.user.count(),
|
||||
Article: await p.article.count(),
|
||||
Strategy: await p.strategy.count(),
|
||||
Indicator: await p.indicator.count(),
|
||||
Econ_SowInventory: await p.econ_SowInventory.count(),
|
||||
};
|
||||
console.log('COUNTS:', JSON.stringify(counts, null, 2));
|
||||
await p.$disconnect();
|
||||
})().catch((e) => { console.error(e); process.exit(1); });
|
||||
@@ -1,22 +0,0 @@
|
||||
import 'dotenv/config';
|
||||
import { PrismaClient } from '../generated/prisma/client.ts';
|
||||
|
||||
const url = process.env.DATABASE_URL;
|
||||
const p = new PrismaClient({ datasourceUrl: url });
|
||||
try {
|
||||
const tables = await p.$queryRawUnsafe('SHOW TABLES');
|
||||
console.log('TABLES:', JSON.stringify(tables, null, 2));
|
||||
const counts = {
|
||||
User: await p.user.count(),
|
||||
Article: await p.article.count(),
|
||||
Strategy: await p.strategy.count(),
|
||||
Indicator: await p.indicator.count(),
|
||||
Econ_SowInventory: await p.econ_SowInventory.count(),
|
||||
};
|
||||
console.log('COUNTS:', JSON.stringify(counts, null, 2));
|
||||
} catch (e) {
|
||||
console.error('ERR:', e.message);
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
await p.$disconnect();
|
||||
}
|
||||
Reference in New Issue
Block a user