修改数据获取方式

2026-08-06 14:00:11 +08:00
parent 4da9489662
commit b10b1c44db
11 changed files with 615 additions and 31 deletions
@@ -0,0 +1,210 @@
+// 东方财富妙想 MCP 数据源
+// 协议: StreamableHttp (MCP 2025-03-26 spec), HTTP POST + JSON-RPC 2.0
+//   - 认证: header `em_api_key: <EM_API_KEY>`
+//   - 妙想 MCP 是"自然语言驱动": 传 query 字符串, 内部 LLM 解析 + 调底层数据接口
+//   - 重要: 每次调用 5-20s + LLM token 成本, 必须配合"DB 优先" (refresh 时已存在的指标跳过)
+//
+// 需要的 .env:
+//   EM_API_KEY=xxx                  必填
+//   EASTMONEY_MCP_BASE=...          可选, 默认 https://mxapi.eastmoney.com/mxds/mcp
+//
+// INDICATORS 数组: 配置要拉的指标
+//   每项:
+//     name      → Indicator.name (主键)
+//     groupName → Indicator.groupName
+//     title     → data.title
+//     unit      → data.unit
+//     tool      → MCP tool 名
+//     query     → 自然语言查询
+//     pickSheet(sheets) → 选一张 sheet (默认取第一张)
+//     parseSheet(sheet) → [{period, value}, ...]   默认按宽表转长表 (列=日期, 行=值)
+
+import axios from 'axios'
+
+const DEFAULT_BASE = 'https://mxapi.eastmoney.com/mxds/mcp'
+const TIMEOUT = 60000
+
+// ====== INDICATORS 配置 ======
+// 跑 npm run data:discover 看可用 tool, 再加新指标
+const INDICATORS = [
+  {
+    name: 'china-cpi-yoy',
+    groupName: 'china-macro',
+    title: '中国 CPI 同比',
+    unit: '%',
+    tool: 'mx_macro_data',
+    query: '查询中国 2015 年至今 CPI 月度同比数据，每一个月的数值',
+    pickSheet: (sheets) =>
+      sheets.find((s) => s.items?.[0]?.[0]?.includes('CPI') && s.items[0][0].includes('同比'))
+      || sheets.find((s) => s.sheetName?.includes('宏观数据'))
+      || sheets[0],
+  },
+  {
+    name: 'china-gdp-yoy',
+    groupName: 'china-macro',
+    title: '中国 GDP 同比',
+    unit: '%',
+    tool: 'mx_macro_data',
+    query: '查询中国 2015-2025 年 GDP 不变价同比增速，季度数据，每一季度的数值',
+    pickSheet: (sheets) =>
+      sheets.find((s) => s.items?.[0]?.[0]?.includes('GDP') && s.items[0][0].includes('同比'))
+      || sheets[0],
+  },
+]
+
+export const META = {
+  name: 'eastmoney-mcp',
+  description: '东方财富妙想 MCP (StreamableHttp, NL 查询)',
+  docs: 'https://mxapi.eastmoney.com/mxds/doc/mcp-install.md',
+}
+
+// ====== StreamableHttp 客户端 ======
+
+class McpClient {
+  constructor(baseUrl, apiKey) {
+    this.baseUrl = baseUrl
+    this.apiKey = apiKey
+    this.id = 0
+  }
+  async _request(method, params) {
+    const payload = { jsonrpc: '2.0', id: ++this.id, method, ...(params ? { params } : {}) }
+    const resp = await axios.post(this.baseUrl, payload, {
+      timeout: TIMEOUT,
+      headers: {
+        'Content-Type': 'application/json',
+        'Accept': 'application/json, text/event-stream',
+        'em_api_key': this.apiKey,
+      },
+      responseType: 'text',
+      transformResponse: [(d) => d],
+      validateStatus: () => true,
+    })
+    if (resp.status >= 400) {
+      throw new Error(`HTTP ${resp.status}: ${String(resp.data).slice(0, 300)}`)
+    }
+    const parsed = parseMcpResponse(String(resp.data || ''))
+    // JSON-RPC 信封 {jsonrpc,id,result|error}, 统一解包 result
+    if (parsed.error) {
+      throw new Error(`MCP error ${parsed.error.code}: ${parsed.error.message}`)
+    }
+    return parsed.result || parsed
+  }
+  async listTools() {
+    const r = await this._request('tools/list')
+    return r.tools || []
+  }
+  async callTool(name, args = {}) {
+    const r = await this._request('tools/call', { name, arguments: args })
+    if (r.isError) {
+      const msg = (r.content || []).map((c) => c.text || '').join('; ')
+      throw new Error(`tool ${name} error: ${msg || JSON.stringify(r).slice(0, 200)}`)
+    }
+    return r
+  }
+}
+
+// 解析响应: JSON 或 SSE
+function parseMcpResponse(text) {
+  const t = text.trim()
+  if (t.startsWith('{')) return JSON.parse(t)
+  if (t.startsWith('data:')) {
+    const lines = t.split('\n').filter((l) => l.startsWith('data:'))
+    let last = null
+    for (const line of lines) {
+      const json = line.slice(5).trim()
+      if (!json || json === '[DONE]') continue
+      try { last = JSON.parse(json) } catch { /* skip */ }
+    }
+    if (!last) throw new Error('SSE 无有效 data 块: ' + text.slice(0, 200))
+    return last
+  }
+  throw new Error('未知响应格式: ' + t.slice(0, 200))
+}
+
+// 默认 sheet 解析: 宽表 (列=日期, 行=值) → 长表
+// 例: { columns:["指标","来源","2024-12","2024-11"], items:[["CPI","统计局","0.1","0.2"]] }
+//   → [{period:"2024-12", value:0.1}, {period:"2024-11", value:0.2}]
+// 返回时反转成从老到新
+function defaultParseSheet(sheet) {
+  if (!sheet || !sheet.columns || !sheet.items || !sheet.items[0]) return []
+  const cols = sheet.columns
+  const row = sheet.items[0]
+  const dates = cols.slice(2)         // 跳过 [指标名, 来源]
+  const values = row.slice(2)
+  const out = []
+  for (let i = 0; i < dates.length; i++) {
+    const v = parseFloat(values[i])
+    if (!isNaN(v)) out.push({ period: dates[i], value: v })
+  }
+  return out.reverse()  // 从老到新
+}
+
+// ====== fetcher 入口 ======
+
+export async function listAvailableTools() {
+  const key = process.env.EM_API_KEY
+  if (!key) throw new Error('缺少 EM_API_KEY')
+  const base = process.env.EASTMONEY_MCP_BASE || DEFAULT_BASE
+  const client = new McpClient(base, key)
+  return client.listTools()
+}
+
+// opts.skipNames: Set<string> — DB 已存在的指标名, 跳过不调 MCP (省 token)
+// opts.onlyNames: Set<string> — 白名单, 只跑这些 (用于补跑单个)
+export async function fetchAll(opts = {}) {
+  if (INDICATORS.length === 0) {
+    throw new Error('eastmoney-mcp: INDICATORS 数组为空, 加新指标见文件顶部注释')
+  }
+  const skip = opts.skipNames || new Set()
+  const only = opts.onlyNames || null
+
+  const key = process.env.EM_API_KEY
+  if (!key) throw new Error('缺少 EM_API_KEY')
+  const base = process.env.EASTMONEY_MCP_BASE || DEFAULT_BASE
+  const client = new McpClient(base, key)
+
+  const results = []
+  for (const cfg of INDICATORS) {
+    if (skip.has(cfg.name)) {
+      console.log(`[MCP] ${cfg.name} 跳过 (DB 已存在)`)
+      continue
+    }
+    if (only && !only.has(cfg.name)) continue
+    try {
+      console.log(`[MCP] ${cfg.name} 调用 ${cfg.tool}: ${cfg.query}`)
+      const r = await client.callTool(cfg.tool, { query: cfg.query })
+      const text = (r.content || []).map((c) => c.text || '').join('')
+      let payload
+      try { payload = JSON.parse(text) } catch { payload = { raw: text } }
+      const sheets = payload.data || (Array.isArray(payload) ? payload : [payload])
+      const sheet = cfg.pickSheet ? cfg.pickSheet(sheets) : sheets[0]
+      const parsed = cfg.parseSheet ? cfg.parseSheet(sheet) : defaultParseSheet(sheet)
+      if (parsed.length === 0) {
+        console.log(`[MCP] ${cfg.name} 返回空 rows, 跳过`)
+        continue
+      }
+      results.push({
+        name: cfg.name,
+        groupName: cfg.groupName,
+        data: {
+          unit: cfg.unit,
+          title: cfg.title,
+          source: 'eastmoney-mcp',
+          rows: parsed,
+        },
+      })
+      console.log(`[MCP] ${cfg.name} OK, ${parsed.length} rows (${parsed[0]?.period}..${parsed.at(-1)?.period})`)
+    } catch (e) {
+      console.error(`[MCP] ${cfg.name} FAIL: ${e.message}`)
+    }
+  }
+  // 全部被 DB 优先跳过时, 返回显式标记而不是空数组 (避免被误判为失败)
+  if (results.length === 0 && INDICATORS.every((cfg) => skip.has(cfg.name))) {
+    return { skipped: true, reason: 'DB 已存在全部指标' }
+  }
+  return results
+}
+
+
+
+
@@ -0,0 +1,82 @@
+// Fetcher 注册表: 列出所有启用的数据源
+// 加新数据源只要:
+//   1. 写一个 scripts/fetchers/<source>.fetcher.js, 导出 META + fetchAll
+//   2. 在本文件 FETCHERS 数组里加一行
+//   3. 完事, refresh 脚本会自动跑
+
+// 每项的 schema:
+//   name:    数据源标识 (用于日志)
+//   module:  fetcher 模块路径 (相对 scripts/fetchers/)
+//   enabled: true/false, 可由 env 关闭 (如 ENABLE_EASTMONEY_MCP=0)
+//   env:     需要的环境变量名列表, 缺一即跳过
+export const FETCHERS = [
+  {
+    name: 'wb-indicators',
+    module: './wb-indicators.fetcher.js',
+    enabled: true,
+    env: [],
+  },
+  {
+    name: 'eastmoney-mcp',
+    module: './eastmoney-mcp.fetcher.js',
+    // 默认开启, 有 APIKey 才真正能跑 (fetcher 内部会自检)
+    enabled: process.env.ENABLE_EASTMONEY_MCP !== '0',
+    env: ['EM_API_KEY'],
+  },
+  // 后续在这里追加:
+  // { name: 'fred',     module: './fred.fetcher.js',          enabled: true, env: ['FRED_API_KEY'] },
+  // { name: 'tushare',  module: './tushare.fetcher.js',       enabled: true, env: ['TUSHARE_TOKEN'] },
+]
+
+// 校验环境变量, 返回 { ok, missing[] }
+export function checkEnv(required) {
+  const missing = required.filter((k) => !process.env[k] || !process.env[k].trim())
+  return { ok: missing.length === 0, missing }
+}
+
+// 加载并运行所有启用的 fetcher, 返回 [{ name, ok, items, error, skipped }]
+// 错误隔离: 一个 fetcher 挂了不影响其他
+// opts.skipNames: Set<string> — DB 优先, 已存在的指标名, 透传给 fetcher
+export async function runFetchers(opts = {}) {
+  const results = []
+  for (const meta of FETCHERS) {
+    if (!meta.enabled) {
+      results.push({ name: meta.name, skipped: true, reason: 'disabled' })
+      console.log(`[Fetcher] ${meta.name} 跳过 (disabled)`)
+      continue
+    }
+    const envCheck = checkEnv(meta.env)
+    if (!envCheck.ok) {
+      results.push({ name: meta.name, skipped: true, reason: `缺少 env: ${envCheck.missing.join(', ')}` })
+      console.log(`[Fetcher] ${meta.name} 跳过 (${envCheck.missing.join(', ')})`)
+      continue
+    }
+    try {
+      const mod = await import(meta.module)
+      if (!mod.META || !mod.fetchAll) {
+        throw new Error('fetcher 必须导出 META 和 fetchAll')
+      }
+      console.log(`[Fetcher] ${meta.name} 开始...`)
+      const out = await mod.fetchAll({ skipNames: opts.skipNames, onlyNames: opts.onlyNames })
+      // fetcher 可返回 { skipped: true, reason } 表示"全部被 DB 优先跳过", 不算失败
+      if (out && out.skipped) {
+        results.push({ name: meta.name, skipped: true, reason: out.reason })
+        console.log(`[Fetcher] ${meta.name} 跳过 (${out.reason})`)
+        continue
+      }
+      const items = Array.isArray(out) ? out : []
+      // 校验返回结构
+      const valid = items.filter((it) => it && it.name && it.data && Array.isArray(it.data.rows))
+      if (valid.length === 0) {
+        throw new Error('fetcher 未返回任何有效指标 (需包含 name/data.rows)')
+      }
+      console.log(`[Fetcher] ${meta.name} 拉取 ${valid.length} 个指标`)
+      results.push({ name: meta.name, ok: true, items: valid })
+    } catch (e) {
+      results.push({ name: meta.name, ok: false, error: e.message })
+      console.error(`[Fetcher] ${meta.name} 失败: ${e.message}`)
+    }
+  }
+  return results
+}
+
@@ -0,0 +1,113 @@
+import axios from 'axios'
+
+// World Bank API 数据源
+// 文档: https://datahelpdesk.worldbank.org/knowledgebase/articles/898581
+// 优点: 官方、JSON、免 key、稳定, 中国数据滞后约 1 年 (这是全球宏观数据物理上限)
+// 接口: GET /v2/country/{code}/indicator/{id}?format=json&date={startYear}:{endYear}&per_page=100
+
+const WB_BASE = 'https://api.worldbank.org/v2'
+const TIMEOUT = 15000
+
+// 6 个核心宏观指标
+// - 中国 GDP (现价美元)  - 国民经济总规模
+// - 中国 GDP 增速      - 增长动能
+// - 中国 CPI 同比      - 通胀
+// - 中国失业率         - 就业
+// - 美国 GDP (现价美元) - 对标
+// - 美国 CPI 同比      - 对标
+export const WB_INDICATORS = [
+  { id: 'NY.GDP.MKTP.CD',   name: 'china-gdp-usd',    groupName: 'china-macro', title: '中国 GDP',           unit: '万亿美元', scale: 1e-8,   digits: 2 },
+  { id: 'NY.GDP.MKTP.KD.ZG',name: 'china-gdp-growth', groupName: 'china-macro', title: '中国 GDP 增速',      unit: '%',       scale: 1,      digits: 2 },
+  { id: 'FP.CPI.TOTL.ZG',   name: 'china-cpi',        groupName: 'china-macro', title: '中国 CPI',           unit: '%',       scale: 1,      digits: 2 },
+  { id: 'SL.UEM.TOTL.ZS',   name: 'china-unemploy',   groupName: 'china-macro', title: '中国失业率',         unit: '%',       scale: 1,      digits: 2 },
+  { id: 'NY.GDP.MKTP.CD',   name: 'usa-gdp-usd',      groupName: 'usa-macro',   title: '美国 GDP',           unit: '万亿美元', scale: 1e-8,   digits: 2 },
+  { id: 'FP.CPI.TOTL.ZG',   name: 'usa-cpi',          groupName: 'usa-macro',   title: '美国 CPI',           unit: '%',       scale: 1,      digits: 2 },
+]
+
+// World Bank 中国是 CHN, 美国是 USA
+function countryCodeOf(name) {
+  if (name.startsWith('china-')) return 'CHN'
+  if (name.startsWith('usa-')) return 'USA'
+  throw new Error('Unknown country for ' + name)
+}
+
+// 拉单个指标从 startYear 到当前年
+// 返回 { name, groupName, title, unit, rows: [{ year, value }] }
+// 注意: WB 对连续请求会限流返回 400 Invalid value, 加 3 次退避重试
+export async function fetchIndicator(meta, startYear = 2015) {
+  const country = countryCodeOf(meta.name)
+  const currentYear = new Date().getFullYear()
+  const url = `${WB_BASE}/country/${country}/indicator/${meta.id}?format=json&date=${startYear}:${currentYear}&per_page=200`
+
+  let resp
+  const delays = [0, 1500, 4000]
+  for (let attempt = 0; attempt < delays.length; attempt++) {
+    if (delays[attempt]) await new Promise((r) => setTimeout(r, delays[attempt]))
+    try {
+      resp = await axios.get(url, { timeout: TIMEOUT, headers: { 'User-Agent': 'iboard-refresh/1.0' } })
+      break
+    } catch (e) {
+      if (attempt === delays.length - 1) throw e
+      console.log(`[WB] ${meta.name} 请求失败, ${delays[attempt + 1] / 1000}s 后重试 (${e.message})`)
+    }
+  }
+  // WB 返回: [{page,pages,per_page,total,...}, [{indicator, country, value, date}, ...]]
+  const data = resp.data
+  if (!Array.isArray(data) || data.length < 2 || !Array.isArray(data[1])) {
+    throw new Error('World Bank 返回结构异常: ' + JSON.stringify(data).slice(0, 200))
+  }
+  const points = data[1]
+    .filter((p) => p.value !== null && p.value !== undefined)
+    .map((p) => ({
+      year: String(p.date),
+      value: Number((Number(p.value) * meta.scale).toFixed(meta.digits)),
+    }))
+    .sort((a, b) => Number(a.year) - Number(b.year))
+
+  return {
+    name: meta.name,
+    groupName: meta.groupName,
+    data: {
+      unit: meta.unit,
+      title: meta.title,
+      source: 'World Bank API',
+      rows: points,
+    },
+  }
+}
+
+export const META = {
+  name: 'wb-indicators',
+  description: 'World Bank 官方 API 宏观数据',
+  docs: 'https://datahelpdesk.worldbank.org/knowledgebase/articles/898581',
+}
+
+// 拉全部 (返回符合 fetcher 契约的 [{name, groupName, data}, ...])
+// 错误隔离: 单个指标失败 -> 跳过 + 记日志, 不影响其他
+// opts.skipNames: Set<string> — DB 已存在的指标名, 跳过不拉 (WB 免费但也要省请求)
+export async function fetchAll(opts = {}) {
+  const skip = opts.skipNames || new Set()
+  const startYear = 2015
+  const results = []
+  for (const m of WB_INDICATORS) {
+    if (skip.has(m.name)) {
+      console.log(`[WB] ${m.name} 跳过 (DB 已存在)`)
+      continue
+    }
+    try {
+      const r = await fetchIndicator(m, startYear)
+      console.log(`[WB] ${r.name} OK, ${r.data.rows.length} rows (${r.data.rows[0]?.year}..${r.data.rows.at(-1)?.year})`)
+      results.push(r)
+    } catch (e) {
+      console.error(`[WB] ${m.name} FAIL: ${e.message}`)
+    }
+  }
+  // 全部被 DB 优先跳过时, 返回显式标记而不是空数组 (避免被误判为失败)
+  if (results.length === 0 && WB_INDICATORS.every((m) => skip.has(m.name))) {
+    return { skipped: true, reason: 'DB 已存在全部指标' }
+  }
+  return results
+}
+
+
+