修改数据获取方式
This commit is contained in:
cheney
parent
4da9489662
commit
b10b1c44db
3
.gitignore
vendored
3
.gitignore
vendored
@ -31,3 +31,6 @@ lerna-debug.log*
|
||||
*.swo
|
||||
*~
|
||||
dist/*
|
||||
|
||||
# local env (contains API keys)
|
||||
/.env
|
||||
|
||||
@ -12,6 +12,8 @@
|
||||
"start": "node --experimental-strip-types --no-warnings server/index.js",
|
||||
"db:migrate": "prisma migrate deploy",
|
||||
"db:seed": "node scripts/seed.cjs",
|
||||
"data:refresh": "node --no-warnings scripts/refresh-indicators.cjs",
|
||||
"data:discover": "node --no-warnings scripts/discover-mcp-tools.cjs",
|
||||
"icons": "node scripts/generate-icons.mjs",
|
||||
"tauri:dev": "tauri dev",
|
||||
"tauri:build": "tauri build",
|
||||
|
||||
17
readme
17
readme
@ -4,3 +4,20 @@
|
||||
- 数据主要是放各种经济数据和图表。
|
||||
- 策略是放按照经济理论组织的相关数据和图表的对比。
|
||||
|
||||
|
||||
|
||||
|
||||
## 新闻模块
|
||||
增加一个新闻模块，数据来源是每日夜间 3点，通过妙想 MCP 获取国内外最新经济政治新闻，按影响大小排序10条。
|
||||
存入数据库。
|
||||
|
||||
|
||||
## 报警模块
|
||||
World Bank 中的数据只用来对比妙想 MCP 拉取的数据正确性。当同一时间的同一项数据不同时，记录一个报警。
|
||||
|
||||
|
||||
## 首页
|
||||
首页显示新闻模块和报警模块的数据，按照时间最新优先，无限滚动，直到数据库中没有数据为止。
|
||||
|
||||
|
||||
|
||||
|
||||
44
scripts/debug-mcp.cjs
Normal file
44
scripts/debug-mcp.cjs
Normal file
@ -0,0 +1,44 @@
|
||||
require('dotenv').config()
|
||||
const axios = require('axios')
|
||||
|
||||
async function call(tool, query) {
|
||||
const base = process.env.EASTMONEY_MCP_BASE || 'https://mxapi.eastmoney.com/mxds/mcp'
|
||||
const payload = {
|
||||
jsonrpc: '2.0', id: 1, method: 'tools/call',
|
||||
params: { name: tool, arguments: { query } },
|
||||
}
|
||||
const resp = await axios.post(base, payload, {
|
||||
timeout: 60000,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Accept': 'application/json, text/event-stream',
|
||||
'em_api_key': process.env.EM_API_KEY,
|
||||
},
|
||||
validateStatus: () => true,
|
||||
responseType: 'text',
|
||||
transformResponse: [(d) => d],
|
||||
})
|
||||
const body = String(resp.data)
|
||||
try {
|
||||
const j = JSON.parse(body)
|
||||
const text = j.result.content[0].text
|
||||
const parsed = JSON.parse(text)
|
||||
console.log(`\n=== ${tool}: "${query}" ===`)
|
||||
console.log('sheets 数量:', parsed.data.length)
|
||||
parsed.data.forEach((s, i) => {
|
||||
console.log(`\nsheet[${i}]:`)
|
||||
console.log(' sheetName:', s.sheetName)
|
||||
console.log(' columns[0..3]:', s.columns.slice(0, 4))
|
||||
console.log(' items[0][0..3]:', s.items[0] ? s.items[0].slice(0, 4) : '(无)')
|
||||
})
|
||||
} catch (e) {
|
||||
console.log('解析失败:', e.message)
|
||||
console.log(body.slice(0, 1000))
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
await call('mx_macro_data', '查询中国 2020-2025 年 CPI 月度同比数据')
|
||||
await call('mx_macro_data', '查询中国 2015-2024 年 GDP 同比增速年度数据')
|
||||
}
|
||||
main().catch(console.error)
|
||||
32
scripts/discover-mcp-tools.cjs
Normal file
32
scripts/discover-mcp-tools.cjs
Normal file
@ -0,0 +1,32 @@
|
||||
/**
|
||||
* 列出东方财富妙想 MCP 暴露的所有工具
|
||||
* 用法: npm run data:discover
|
||||
* 输出: tool 名称 + 描述 + 输入 schema
|
||||
*/
|
||||
require('dotenv').config()
|
||||
|
||||
async function main() {
|
||||
if (!process.env.EM_API_KEY) {
|
||||
console.error('缺少 EM_API_KEY (在 .env 配)')
|
||||
process.exit(1)
|
||||
}
|
||||
const { listAvailableTools } = await import('./fetchers/eastmoney-mcp.fetcher.js')
|
||||
console.log('[Discover] 调妙想 MCP tools/list...')
|
||||
const tools = await listAvailableTools()
|
||||
console.log(`[Discover] 找到 ${tools.length} 个工具\n`)
|
||||
for (const t of tools) {
|
||||
console.log(`── ${t.name} ${'─'.repeat(Math.max(0, 60 - t.name.length))}`)
|
||||
if (t.description) console.log(` ${t.description}`)
|
||||
if (t.inputSchema) {
|
||||
console.log(' inputSchema:')
|
||||
const s = JSON.stringify(t.inputSchema, null, 2)
|
||||
s.split('\n').forEach((line) => console.log(' ' + line))
|
||||
}
|
||||
console.log('')
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
console.error('[Discover] 失败:', e.message)
|
||||
process.exit(1)
|
||||
})
|
||||
210
scripts/fetchers/eastmoney-mcp.fetcher.js
Normal file
210
scripts/fetchers/eastmoney-mcp.fetcher.js
Normal file
@ -0,0 +1,210 @@
|
||||
// 东方财富妙想 MCP 数据源
|
||||
// 协议: StreamableHttp (MCP 2025-03-26 spec), HTTP POST + JSON-RPC 2.0
|
||||
// - 认证: header `em_api_key: <EM_API_KEY>`
|
||||
// - 妙想 MCP 是"自然语言驱动": 传 query 字符串, 内部 LLM 解析 + 调底层数据接口
|
||||
// - 重要: 每次调用 5-20s + LLM token 成本, 必须配合"DB 优先" (refresh 时已存在的指标跳过)
|
||||
//
|
||||
// 需要的 .env:
|
||||
// EM_API_KEY=xxx 必填
|
||||
// EASTMONEY_MCP_BASE=... 可选, 默认 https://mxapi.eastmoney.com/mxds/mcp
|
||||
//
|
||||
// INDICATORS 数组: 配置要拉的指标
|
||||
// 每项:
|
||||
// name → Indicator.name (主键)
|
||||
// groupName → Indicator.groupName
|
||||
// title → data.title
|
||||
// unit → data.unit
|
||||
// tool → MCP tool 名
|
||||
// query → 自然语言查询
|
||||
// pickSheet(sheets) → 选一张 sheet (默认取第一张)
|
||||
// parseSheet(sheet) → [{period, value}, ...] 默认按宽表转长表 (列=日期, 行=值)
|
||||
|
||||
import axios from 'axios'
|
||||
|
||||
const DEFAULT_BASE = 'https://mxapi.eastmoney.com/mxds/mcp'
|
||||
const TIMEOUT = 60000
|
||||
|
||||
// ====== INDICATORS 配置 ======
|
||||
// 跑 npm run data:discover 看可用 tool, 再加新指标
|
||||
const INDICATORS = [
|
||||
{
|
||||
name: 'china-cpi-yoy',
|
||||
groupName: 'china-macro',
|
||||
title: '中国 CPI 同比',
|
||||
unit: '%',
|
||||
tool: 'mx_macro_data',
|
||||
query: '查询中国 2015 年至今 CPI 月度同比数据，每一个月的数值',
|
||||
pickSheet: (sheets) =>
|
||||
sheets.find((s) => s.items?.[0]?.[0]?.includes('CPI') && s.items[0][0].includes('同比'))
|
||||
|| sheets.find((s) => s.sheetName?.includes('宏观数据'))
|
||||
|| sheets[0],
|
||||
},
|
||||
{
|
||||
name: 'china-gdp-yoy',
|
||||
groupName: 'china-macro',
|
||||
title: '中国 GDP 同比',
|
||||
unit: '%',
|
||||
tool: 'mx_macro_data',
|
||||
query: '查询中国 2015-2025 年 GDP 不变价同比增速，季度数据，每一季度的数值',
|
||||
pickSheet: (sheets) =>
|
||||
sheets.find((s) => s.items?.[0]?.[0]?.includes('GDP') && s.items[0][0].includes('同比'))
|
||||
|| sheets[0],
|
||||
},
|
||||
]
|
||||
|
||||
export const META = {
|
||||
name: 'eastmoney-mcp',
|
||||
description: '东方财富妙想 MCP (StreamableHttp, NL 查询)',
|
||||
docs: 'https://mxapi.eastmoney.com/mxds/doc/mcp-install.md',
|
||||
}
|
||||
|
||||
// ====== StreamableHttp 客户端 ======
|
||||
|
||||
class McpClient {
|
||||
constructor(baseUrl, apiKey) {
|
||||
this.baseUrl = baseUrl
|
||||
this.apiKey = apiKey
|
||||
this.id = 0
|
||||
}
|
||||
async _request(method, params) {
|
||||
const payload = { jsonrpc: '2.0', id: ++this.id, method, ...(params ? { params } : {}) }
|
||||
const resp = await axios.post(this.baseUrl, payload, {
|
||||
timeout: TIMEOUT,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Accept': 'application/json, text/event-stream',
|
||||
'em_api_key': this.apiKey,
|
||||
},
|
||||
responseType: 'text',
|
||||
transformResponse: [(d) => d],
|
||||
validateStatus: () => true,
|
||||
})
|
||||
if (resp.status >= 400) {
|
||||
throw new Error(`HTTP ${resp.status}: ${String(resp.data).slice(0, 300)}`)
|
||||
}
|
||||
const parsed = parseMcpResponse(String(resp.data || ''))
|
||||
// JSON-RPC 信封 {jsonrpc,id,result|error}, 统一解包 result
|
||||
if (parsed.error) {
|
||||
throw new Error(`MCP error ${parsed.error.code}: ${parsed.error.message}`)
|
||||
}
|
||||
return parsed.result || parsed
|
||||
}
|
||||
async listTools() {
|
||||
const r = await this._request('tools/list')
|
||||
return r.tools || []
|
||||
}
|
||||
async callTool(name, args = {}) {
|
||||
const r = await this._request('tools/call', { name, arguments: args })
|
||||
if (r.isError) {
|
||||
const msg = (r.content || []).map((c) => c.text || '').join('; ')
|
||||
throw new Error(`tool ${name} error: ${msg || JSON.stringify(r).slice(0, 200)}`)
|
||||
}
|
||||
return r
|
||||
}
|
||||
}
|
||||
|
||||
// 解析响应: JSON 或 SSE
|
||||
function parseMcpResponse(text) {
|
||||
const t = text.trim()
|
||||
if (t.startsWith('{')) return JSON.parse(t)
|
||||
if (t.startsWith('data:')) {
|
||||
const lines = t.split('\n').filter((l) => l.startsWith('data:'))
|
||||
let last = null
|
||||
for (const line of lines) {
|
||||
const json = line.slice(5).trim()
|
||||
if (!json || json === '[DONE]') continue
|
||||
try { last = JSON.parse(json) } catch { /* skip */ }
|
||||
}
|
||||
if (!last) throw new Error('SSE 无有效 data 块: ' + text.slice(0, 200))
|
||||
return last
|
||||
}
|
||||
throw new Error('未知响应格式: ' + t.slice(0, 200))
|
||||
}
|
||||
|
||||
// 默认 sheet 解析: 宽表 (列=日期, 行=值) → 长表
|
||||
// 例: { columns:["指标","来源","2024-12","2024-11"], items:[["CPI","统计局","0.1","0.2"]] }
|
||||
// → [{period:"2024-12", value:0.1}, {period:"2024-11", value:0.2}]
|
||||
// 返回时反转成从老到新
|
||||
function defaultParseSheet(sheet) {
|
||||
if (!sheet || !sheet.columns || !sheet.items || !sheet.items[0]) return []
|
||||
const cols = sheet.columns
|
||||
const row = sheet.items[0]
|
||||
const dates = cols.slice(2) // 跳过 [指标名, 来源]
|
||||
const values = row.slice(2)
|
||||
const out = []
|
||||
for (let i = 0; i < dates.length; i++) {
|
||||
const v = parseFloat(values[i])
|
||||
if (!isNaN(v)) out.push({ period: dates[i], value: v })
|
||||
}
|
||||
return out.reverse() // 从老到新
|
||||
}
|
||||
|
||||
// ====== fetcher 入口 ======
|
||||
|
||||
export async function listAvailableTools() {
|
||||
const key = process.env.EM_API_KEY
|
||||
if (!key) throw new Error('缺少 EM_API_KEY')
|
||||
const base = process.env.EASTMONEY_MCP_BASE || DEFAULT_BASE
|
||||
const client = new McpClient(base, key)
|
||||
return client.listTools()
|
||||
}
|
||||
|
||||
// opts.skipNames: Set<string> — DB 已存在的指标名, 跳过不调 MCP (省 token)
|
||||
// opts.onlyNames: Set<string> — 白名单, 只跑这些 (用于补跑单个)
|
||||
export async function fetchAll(opts = {}) {
|
||||
if (INDICATORS.length === 0) {
|
||||
throw new Error('eastmoney-mcp: INDICATORS 数组为空, 加新指标见文件顶部注释')
|
||||
}
|
||||
const skip = opts.skipNames || new Set()
|
||||
const only = opts.onlyNames || null
|
||||
|
||||
const key = process.env.EM_API_KEY
|
||||
if (!key) throw new Error('缺少 EM_API_KEY')
|
||||
const base = process.env.EASTMONEY_MCP_BASE || DEFAULT_BASE
|
||||
const client = new McpClient(base, key)
|
||||
|
||||
const results = []
|
||||
for (const cfg of INDICATORS) {
|
||||
if (skip.has(cfg.name)) {
|
||||
console.log(`[MCP] ${cfg.name} 跳过 (DB 已存在)`)
|
||||
continue
|
||||
}
|
||||
if (only && !only.has(cfg.name)) continue
|
||||
try {
|
||||
console.log(`[MCP] ${cfg.name} 调用 ${cfg.tool}: ${cfg.query}`)
|
||||
const r = await client.callTool(cfg.tool, { query: cfg.query })
|
||||
const text = (r.content || []).map((c) => c.text || '').join('')
|
||||
let payload
|
||||
try { payload = JSON.parse(text) } catch { payload = { raw: text } }
|
||||
const sheets = payload.data || (Array.isArray(payload) ? payload : [payload])
|
||||
const sheet = cfg.pickSheet ? cfg.pickSheet(sheets) : sheets[0]
|
||||
const parsed = cfg.parseSheet ? cfg.parseSheet(sheet) : defaultParseSheet(sheet)
|
||||
if (parsed.length === 0) {
|
||||
console.log(`[MCP] ${cfg.name} 返回空 rows, 跳过`)
|
||||
continue
|
||||
}
|
||||
results.push({
|
||||
name: cfg.name,
|
||||
groupName: cfg.groupName,
|
||||
data: {
|
||||
unit: cfg.unit,
|
||||
title: cfg.title,
|
||||
source: 'eastmoney-mcp',
|
||||
rows: parsed,
|
||||
},
|
||||
})
|
||||
console.log(`[MCP] ${cfg.name} OK, ${parsed.length} rows (${parsed[0]?.period}..${parsed.at(-1)?.period})`)
|
||||
} catch (e) {
|
||||
console.error(`[MCP] ${cfg.name} FAIL: ${e.message}`)
|
||||
}
|
||||
}
|
||||
// 全部被 DB 优先跳过时, 返回显式标记而不是空数组 (避免被误判为失败)
|
||||
if (results.length === 0 && INDICATORS.every((cfg) => skip.has(cfg.name))) {
|
||||
return { skipped: true, reason: 'DB 已存在全部指标' }
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
82
scripts/fetchers/index.js
Normal file
82
scripts/fetchers/index.js
Normal file
@ -0,0 +1,82 @@
|
||||
// Fetcher 注册表: 列出所有启用的数据源
|
||||
// 加新数据源只要:
|
||||
// 1. 写一个 scripts/fetchers/<source>.fetcher.js, 导出 META + fetchAll
|
||||
// 2. 在本文件 FETCHERS 数组里加一行
|
||||
// 3. 完事, refresh 脚本会自动跑
|
||||
|
||||
// 每项的 schema:
|
||||
// name: 数据源标识 (用于日志)
|
||||
// module: fetcher 模块路径 (相对 scripts/fetchers/)
|
||||
// enabled: true/false, 可由 env 关闭 (如 ENABLE_EASTMONEY_MCP=0)
|
||||
// env: 需要的环境变量名列表, 缺一即跳过
|
||||
export const FETCHERS = [
|
||||
{
|
||||
name: 'wb-indicators',
|
||||
module: './wb-indicators.fetcher.js',
|
||||
enabled: true,
|
||||
env: [],
|
||||
},
|
||||
{
|
||||
name: 'eastmoney-mcp',
|
||||
module: './eastmoney-mcp.fetcher.js',
|
||||
// 默认开启, 有 APIKey 才真正能跑 (fetcher 内部会自检)
|
||||
enabled: process.env.ENABLE_EASTMONEY_MCP !== '0',
|
||||
env: ['EM_API_KEY'],
|
||||
},
|
||||
// 后续在这里追加:
|
||||
// { name: 'fred', module: './fred.fetcher.js', enabled: true, env: ['FRED_API_KEY'] },
|
||||
// { name: 'tushare', module: './tushare.fetcher.js', enabled: true, env: ['TUSHARE_TOKEN'] },
|
||||
]
|
||||
|
||||
// 校验环境变量, 返回 { ok, missing[] }
|
||||
export function checkEnv(required) {
|
||||
const missing = required.filter((k) => !process.env[k] || !process.env[k].trim())
|
||||
return { ok: missing.length === 0, missing }
|
||||
}
|
||||
|
||||
// 加载并运行所有启用的 fetcher, 返回 [{ name, ok, items, error, skipped }]
|
||||
// 错误隔离: 一个 fetcher 挂了不影响其他
|
||||
// opts.skipNames: Set<string> — DB 优先, 已存在的指标名, 透传给 fetcher
|
||||
export async function runFetchers(opts = {}) {
|
||||
const results = []
|
||||
for (const meta of FETCHERS) {
|
||||
if (!meta.enabled) {
|
||||
results.push({ name: meta.name, skipped: true, reason: 'disabled' })
|
||||
console.log(`[Fetcher] ${meta.name} 跳过 (disabled)`)
|
||||
continue
|
||||
}
|
||||
const envCheck = checkEnv(meta.env)
|
||||
if (!envCheck.ok) {
|
||||
results.push({ name: meta.name, skipped: true, reason: `缺少 env: ${envCheck.missing.join(', ')}` })
|
||||
console.log(`[Fetcher] ${meta.name} 跳过 (${envCheck.missing.join(', ')})`)
|
||||
continue
|
||||
}
|
||||
try {
|
||||
const mod = await import(meta.module)
|
||||
if (!mod.META || !mod.fetchAll) {
|
||||
throw new Error('fetcher 必须导出 META 和 fetchAll')
|
||||
}
|
||||
console.log(`[Fetcher] ${meta.name} 开始...`)
|
||||
const out = await mod.fetchAll({ skipNames: opts.skipNames, onlyNames: opts.onlyNames })
|
||||
// fetcher 可返回 { skipped: true, reason } 表示"全部被 DB 优先跳过", 不算失败
|
||||
if (out && out.skipped) {
|
||||
results.push({ name: meta.name, skipped: true, reason: out.reason })
|
||||
console.log(`[Fetcher] ${meta.name} 跳过 (${out.reason})`)
|
||||
continue
|
||||
}
|
||||
const items = Array.isArray(out) ? out : []
|
||||
// 校验返回结构
|
||||
const valid = items.filter((it) => it && it.name && it.data && Array.isArray(it.data.rows))
|
||||
if (valid.length === 0) {
|
||||
throw new Error('fetcher 未返回任何有效指标 (需包含 name/data.rows)')
|
||||
}
|
||||
console.log(`[Fetcher] ${meta.name} 拉取 ${valid.length} 个指标`)
|
||||
results.push({ name: meta.name, ok: true, items: valid })
|
||||
} catch (e) {
|
||||
results.push({ name: meta.name, ok: false, error: e.message })
|
||||
console.error(`[Fetcher] ${meta.name} 失败: ${e.message}`)
|
||||
}
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
113
scripts/fetchers/wb-indicators.fetcher.js
Normal file
113
scripts/fetchers/wb-indicators.fetcher.js
Normal file
@ -0,0 +1,113 @@
|
||||
import axios from 'axios'
|
||||
|
||||
// World Bank API 数据源
|
||||
// 文档: https://datahelpdesk.worldbank.org/knowledgebase/articles/898581
|
||||
// 优点: 官方、JSON、免 key、稳定, 中国数据滞后约 1 年 (这是全球宏观数据物理上限)
|
||||
// 接口: GET /v2/country/{code}/indicator/{id}?format=json&date={startYear}:{endYear}&per_page=100
|
||||
|
||||
const WB_BASE = 'https://api.worldbank.org/v2'
|
||||
const TIMEOUT = 15000
|
||||
|
||||
// 6 个核心宏观指标
|
||||
// - 中国 GDP (现价美元) - 国民经济总规模
|
||||
// - 中国 GDP 增速 - 增长动能
|
||||
// - 中国 CPI 同比 - 通胀
|
||||
// - 中国失业率 - 就业
|
||||
// - 美国 GDP (现价美元) - 对标
|
||||
// - 美国 CPI 同比 - 对标
|
||||
export const WB_INDICATORS = [
|
||||
{ id: 'NY.GDP.MKTP.CD', name: 'china-gdp-usd', groupName: 'china-macro', title: '中国 GDP', unit: '万亿美元', scale: 1e-8, digits: 2 },
|
||||
{ id: 'NY.GDP.MKTP.KD.ZG',name: 'china-gdp-growth', groupName: 'china-macro', title: '中国 GDP 增速', unit: '%', scale: 1, digits: 2 },
|
||||
{ id: 'FP.CPI.TOTL.ZG', name: 'china-cpi', groupName: 'china-macro', title: '中国 CPI', unit: '%', scale: 1, digits: 2 },
|
||||
{ id: 'SL.UEM.TOTL.ZS', name: 'china-unemploy', groupName: 'china-macro', title: '中国失业率', unit: '%', scale: 1, digits: 2 },
|
||||
{ id: 'NY.GDP.MKTP.CD', name: 'usa-gdp-usd', groupName: 'usa-macro', title: '美国 GDP', unit: '万亿美元', scale: 1e-8, digits: 2 },
|
||||
{ id: 'FP.CPI.TOTL.ZG', name: 'usa-cpi', groupName: 'usa-macro', title: '美国 CPI', unit: '%', scale: 1, digits: 2 },
|
||||
]
|
||||
|
||||
// World Bank 中国是 CHN, 美国是 USA
|
||||
function countryCodeOf(name) {
|
||||
if (name.startsWith('china-')) return 'CHN'
|
||||
if (name.startsWith('usa-')) return 'USA'
|
||||
throw new Error('Unknown country for ' + name)
|
||||
}
|
||||
|
||||
// 拉单个指标从 startYear 到当前年
|
||||
// 返回 { name, groupName, title, unit, rows: [{ year, value }] }
|
||||
// 注意: WB 对连续请求会限流返回 400 Invalid value, 加 3 次退避重试
|
||||
export async function fetchIndicator(meta, startYear = 2015) {
|
||||
const country = countryCodeOf(meta.name)
|
||||
const currentYear = new Date().getFullYear()
|
||||
const url = `${WB_BASE}/country/${country}/indicator/${meta.id}?format=json&date=${startYear}:${currentYear}&per_page=200`
|
||||
|
||||
let resp
|
||||
const delays = [0, 1500, 4000]
|
||||
for (let attempt = 0; attempt < delays.length; attempt++) {
|
||||
if (delays[attempt]) await new Promise((r) => setTimeout(r, delays[attempt]))
|
||||
try {
|
||||
resp = await axios.get(url, { timeout: TIMEOUT, headers: { 'User-Agent': 'iboard-refresh/1.0' } })
|
||||
break
|
||||
} catch (e) {
|
||||
if (attempt === delays.length - 1) throw e
|
||||
console.log(`[WB] ${meta.name} 请求失败, ${delays[attempt + 1] / 1000}s 后重试 (${e.message})`)
|
||||
}
|
||||
}
|
||||
// WB 返回: [{page,pages,per_page,total,...}, [{indicator, country, value, date}, ...]]
|
||||
const data = resp.data
|
||||
if (!Array.isArray(data) || data.length < 2 || !Array.isArray(data[1])) {
|
||||
throw new Error('World Bank 返回结构异常: ' + JSON.stringify(data).slice(0, 200))
|
||||
}
|
||||
const points = data[1]
|
||||
.filter((p) => p.value !== null && p.value !== undefined)
|
||||
.map((p) => ({
|
||||
year: String(p.date),
|
||||
value: Number((Number(p.value) * meta.scale).toFixed(meta.digits)),
|
||||
}))
|
||||
.sort((a, b) => Number(a.year) - Number(b.year))
|
||||
|
||||
return {
|
||||
name: meta.name,
|
||||
groupName: meta.groupName,
|
||||
data: {
|
||||
unit: meta.unit,
|
||||
title: meta.title,
|
||||
source: 'World Bank API',
|
||||
rows: points,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export const META = {
|
||||
name: 'wb-indicators',
|
||||
description: 'World Bank 官方 API 宏观数据',
|
||||
docs: 'https://datahelpdesk.worldbank.org/knowledgebase/articles/898581',
|
||||
}
|
||||
|
||||
// 拉全部 (返回符合 fetcher 契约的 [{name, groupName, data}, ...])
|
||||
// 错误隔离: 单个指标失败 -> 跳过 + 记日志, 不影响其他
|
||||
// opts.skipNames: Set<string> — DB 已存在的指标名, 跳过不拉 (WB 免费但也要省请求)
|
||||
export async function fetchAll(opts = {}) {
|
||||
const skip = opts.skipNames || new Set()
|
||||
const startYear = 2015
|
||||
const results = []
|
||||
for (const m of WB_INDICATORS) {
|
||||
if (skip.has(m.name)) {
|
||||
console.log(`[WB] ${m.name} 跳过 (DB 已存在)`)
|
||||
continue
|
||||
}
|
||||
try {
|
||||
const r = await fetchIndicator(m, startYear)
|
||||
console.log(`[WB] ${r.name} OK, ${r.data.rows.length} rows (${r.data.rows[0]?.year}..${r.data.rows.at(-1)?.year})`)
|
||||
results.push(r)
|
||||
} catch (e) {
|
||||
console.error(`[WB] ${m.name} FAIL: ${e.message}`)
|
||||
}
|
||||
}
|
||||
// 全部被 DB 优先跳过时, 返回显式标记而不是空数组 (避免被误判为失败)
|
||||
if (results.length === 0 && WB_INDICATORS.every((m) => skip.has(m.name))) {
|
||||
return { skipped: true, reason: 'DB 已存在全部指标' }
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
|
||||
|
||||
97
scripts/refresh-indicators.cjs
Normal file
97
scripts/refresh-indicators.cjs
Normal file
@ -0,0 +1,97 @@
|
||||
/**
|
||||
* 拉所有启用的数据源, upsert 到 Indicator 表
|
||||
* 用法: npm run data:refresh
|
||||
* 幂等: 同 name 覆盖 data, 跨源独立, 错误隔离
|
||||
*
|
||||
* DB 优先策略:
|
||||
* 1. 先从 DB 查已存在的 name 集合
|
||||
* 2. 把集合传给每个 fetcher, fetcher 跳过已存在的
|
||||
* 3. 只对 DB 没有的指标调远程拉取, 节省 token (尤其妙想 MCP 这种 LLM 接口)
|
||||
*
|
||||
* 加新数据源: 写 scripts/fetchers/<name>.fetcher.js 导出 META + fetchAll(opts),
|
||||
* 然后在 scripts/fetchers/index.js 的 FETCHERS 注册
|
||||
*/
|
||||
require('dotenv').config()
|
||||
const mysql = require('mysql2/promise')
|
||||
|
||||
function parseUrl(url) {
|
||||
const u = new URL(url)
|
||||
return {
|
||||
host: u.hostname,
|
||||
port: Number(u.port) || 3306,
|
||||
user: decodeURIComponent(u.username),
|
||||
password: decodeURIComponent(u.password),
|
||||
database: u.pathname.replace(/^\//, ''),
|
||||
}
|
||||
}
|
||||
|
||||
async function upsertIndicator(conn, payload) {
|
||||
const sql =
|
||||
'INSERT INTO `Indicator` (`name`, `groupName`, `data`, `createdAt`, `updatedAt`) ' +
|
||||
'VALUES (?, ?, ?, NOW(3), NOW(3)) ' +
|
||||
'ON DUPLICATE KEY UPDATE `groupName` = VALUES(`groupName`), `data` = VALUES(`data`), `updatedAt` = NOW(3)'
|
||||
const [r] = await conn.execute(sql, [payload.name, payload.groupName, JSON.stringify(payload.data)])
|
||||
return r.affectedRows
|
||||
}
|
||||
|
||||
async function fetchExistingNames(conn) {
|
||||
const [rows] = await conn.execute('SELECT `name` FROM `Indicator`')
|
||||
return new Set(rows.map((r) => r.name))
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (!process.env.DATABASE_URL) {
|
||||
console.error('缺少 DATABASE_URL 环境变量')
|
||||
process.exit(1)
|
||||
}
|
||||
const conn = await mysql.createConnection(parseUrl(process.env.DATABASE_URL))
|
||||
try {
|
||||
// 1) 查 DB 已存在的指标名
|
||||
const existing = await fetchExistingNames(conn)
|
||||
console.log(`[Refresh] DB 已有 ${existing.size} 个指标: ${[...existing].join(', ') || '(空)'}`)
|
||||
|
||||
// 2) 跑所有 fetcher, 传入 existing (DB 优先)
|
||||
const { runFetchers } = await import('./fetchers/index.js')
|
||||
const fetcherResults = await runFetchers({ skipNames: existing })
|
||||
|
||||
// 3) 汇总
|
||||
const allItems = []
|
||||
let skipped = 0, failed = 0
|
||||
for (const r of fetcherResults) {
|
||||
if (r.skipped) { skipped++; continue }
|
||||
if (!r.ok) { failed++; continue }
|
||||
// fetcher 内部已经按 skipNames 过滤过, 这里再做一次防御性去重
|
||||
for (const it of r.items) {
|
||||
if (existing.has(it.name)) continue
|
||||
allItems.push(it)
|
||||
}
|
||||
}
|
||||
console.log(`\n[Refresh] 汇总: ${allItems.length} 个新指标 (跳过 ${skipped} 个数据源, 失败 ${failed} 个, DB 已存在 ${existing.size} 个)`)
|
||||
|
||||
if (failed > 0) {
|
||||
console.error('[Refresh] 警告: 有数据源失败, 仅写入成功的指标')
|
||||
}
|
||||
|
||||
if (allItems.length === 0) {
|
||||
console.log('[Refresh] 无新指标, 不写库 (DB 优先策略生效)')
|
||||
return
|
||||
}
|
||||
|
||||
// 4) 写库
|
||||
console.log('[Refresh] 写入数据库...')
|
||||
let ins = 0, upd = 0
|
||||
for (const it of allItems) {
|
||||
const n = await upsertIndicator(conn, { name: it.name, groupName: it.groupName, data: it.data })
|
||||
if (n === 1) ins++
|
||||
else if (n === 2) upd++
|
||||
}
|
||||
console.log(`[Refresh] 写入完成: ${ins} 新增, ${upd} 更新\n`)
|
||||
} finally {
|
||||
await conn.end()
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
console.error('[Refresh] 异常:', e)
|
||||
process.exit(1)
|
||||
})
|
||||
@ -92,32 +92,9 @@ async function main() {
|
||||
}
|
||||
console.log('[Strategy] upsert 策略: ' + n2 + ' 行 (1=新增, 2=更新)');
|
||||
|
||||
const indicators = [
|
||||
{ name: 'china-bond-1y', groupName: 'china-bond',
|
||||
data: { unit: '%', title: '中国国债1年', rows: [
|
||||
{ month: '2024-01', value: 1.85 }, { month: '2024-02', value: 1.82 },
|
||||
{ month: '2024-03', value: 1.78 }, { month: '2024-04', value: 1.75 },
|
||||
{ month: '2024-05', value: 1.72 }, { month: '2024-06', value: 1.68 },
|
||||
]}},
|
||||
{ name: 'china-bond-10y', groupName: 'china-bond',
|
||||
data: { unit: '%', title: '中国国债10年', rows: [
|
||||
{ month: '2024-01', value: 2.65 }, { month: '2024-02', value: 2.62 },
|
||||
{ month: '2024-03', value: 2.58 }, { month: '2024-04', value: 2.55 },
|
||||
{ month: '2024-05', value: 2.52 }, { month: '2024-06', value: 2.48 },
|
||||
]}},
|
||||
{ name: 'us-bond-1y', groupName: 'us-bond',
|
||||
data: { unit: '%', title: '美国国债1年', rows: [
|
||||
{ month: '2024-01', value: 4.85 }, { month: '2024-02', value: 4.82 },
|
||||
{ month: '2024-03', value: 4.78 }, { month: '2024-04', value: 4.65 },
|
||||
{ month: '2024-05', value: 4.55 }, { month: '2024-06', value: 4.45 },
|
||||
]}},
|
||||
{ name: 'china-lpr-1y', groupName: 'china-lpr',
|
||||
data: { unit: '%', title: '中国LPR 1年期', rows: [
|
||||
{ month: '2024-01', value: 3.45 }, { month: '2024-02', value: 3.45 },
|
||||
{ month: '2024-03', value: 3.45 }, { month: '2024-04', value: 3.40 },
|
||||
{ month: '2024-05', value: 3.40 }, { month: '2024-06', value: 3.40 },
|
||||
]}},
|
||||
];
|
||||
// Indicator 表: 改由 scripts/refresh-indicators.cjs 从 World Bank API 拉取最新数据
|
||||
// 旧 4 个金融指标 (国债/LPR) 无稳定 JSON 数据源, 暂不维护
|
||||
const indicators = [];
|
||||
let n3 = 0;
|
||||
for (const i of indicators) {
|
||||
n3 += await upsertByName(conn, 'Indicator',
|
||||
|
||||
@ -7,6 +7,11 @@ import TrendBadge, { calcTrend, lastPoint } from '../../components/TrendBadge.js
|
||||
// 经济数据
|
||||
// 列表态 (默认): 列出所有指标, 每行显示最新值 + 趋势徽标, 点击进入详情
|
||||
// 详情态: 顶部 NavBar (返回) + 折线图 + 数据表
|
||||
// 统一数据行的时间字段: 兼容 {month}/{year}/{period} 三种存储形状
|
||||
function periodOf(r) {
|
||||
return (r && (r.period || r.month || r.year)) || ''
|
||||
}
|
||||
|
||||
export default function DataView() {
|
||||
const [indicators, setIndicators] = useState([])
|
||||
const [err, setErr] = useState('')
|
||||
@ -51,7 +56,7 @@ export default function DataView() {
|
||||
<List.Item
|
||||
key={i.name}
|
||||
title={labelOf(i)}
|
||||
description={lp ? <><b>{lp.value}{unit}</b> · {lp.month}</> : '暂无数据'}
|
||||
description={lp ? <><b>{lp.value}{unit}</b> · {periodOf(lp)}</> : '暂无数据'}
|
||||
extra={
|
||||
<div className="row-meta">
|
||||
{lp && Math.abs(change) >= 1 && (
|
||||
@ -77,6 +82,8 @@ export default function DataView() {
|
||||
function DataDetail({ indicator, onBack }) {
|
||||
const i = indicator
|
||||
const rows = (i.data && i.data.rows) || []
|
||||
// 图表 X 轴统一用 period 字段
|
||||
const chartRows = rows.map((r) => ({ ...r, period: periodOf(r) }))
|
||||
const lp = lastPoint(i.data)
|
||||
const { trend } = calcTrend(rows)
|
||||
const title = (i.data && i.data.title) || i.name
|
||||
@ -86,14 +93,14 @@ function DataDetail({ indicator, onBack }) {
|
||||
<Card className="detail-card">
|
||||
<div className="detail-meta">
|
||||
单位 {i.data.unit || '-'} · 来源 {i.data.source || 'seed'} ·
|
||||
{lp && <> 最新 <b style={{ color: 'var(--color-text)' }}>{lp.value}{i.data.unit || ''}</b> ({lp.month}) </>}
|
||||
{lp && <> 最新 <b style={{ color: 'var(--color-text)' }}>{lp.value}{i.data.unit || ''}</b> ({periodOf(lp)}) </>}
|
||||
<TrendBadge trend={trend} />
|
||||
</div>
|
||||
<div className="chart-wrap">
|
||||
<ResponsiveContainer width="100%" height={260}>
|
||||
<LineChart data={rows} margin={{ top: 10, right: 16, left: 0, bottom: 0 }}>
|
||||
<LineChart data={chartRows} margin={{ top: 10, right: 16, left: 0, bottom: 0 }}>
|
||||
<CartesianGrid strokeDasharray="3 3" />
|
||||
<XAxis dataKey="month" tick={{ fontSize: 11 }} />
|
||||
<XAxis dataKey="period" tick={{ fontSize: 11 }} />
|
||||
<YAxis tick={{ fontSize: 11 }} />
|
||||
<Tooltip />
|
||||
<Legend />
|
||||
@ -108,7 +115,7 @@ function DataDetail({ indicator, onBack }) {
|
||||
</thead>
|
||||
<tbody>
|
||||
{rows.map((r) => (
|
||||
<tr key={r.month}><td>{r.month}</td><td style={{ textAlign: 'right' }}>{r.value}</td></tr>
|
||||
<tr key={periodOf(r)}><td>{periodOf(r)}</td><td style={{ textAlign: 'right' }}>{r.value}</td></tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
Loading…
Reference in New Issue
Block a user