添加汇率 scraper

readme

@@ -4,3 +4,16 @@
 - 数据主要是放各种经济数据和图表。
 - 策略是放按照经济理论组织的相关数据和图表的对比。
 
+
+## 爬虫
+- 基于 bun 的定时任务
+- 统一获取接口
+- 统一爬取接口
+```
+async function fetch(startDate = null, endDate = null)
+async function trigger(startDate = null, endDate = null)
+```
+不传递startDate和endDate，默认，默认获取最近30天的数据。
+
+fecher 负责从目标网站获取原始数据
+scraper 负责检查数据库中的数据是否完整，调用 fecher 将数据存储到数据库。

scripts/scraper/usd-cny-rate.scraper.js

@@ -0,0 +1,234 @@
+import { createPool } from 'mariadb';
+import { fetch as fetchUSDToCNYRate } from '../fetchers/usd-cny-rate.fetcher.js';
+
+let pool;
+
+async function initPool() {
+  if (!pool) {
+    console.log('初始化数据库连接池...');
+    pool = createPool({
+      host: '192.168.111.111',
+      port: 3306,
+      user: 'root',
+      password: 'fullstack',
+      database: 'iboard',
+      connectionLimit: 5
+    });
+    
+    try {
+      const conn = await pool.getConnection();
+      console.log('数据库连接成功!');
+      conn.release();
+    } catch (error) {
+      console.error('数据库连接失败:', error.message);
+      throw error;
+    }
+  }
+  return pool;
+}
+
+async function checkAndFillUSDCNYRate(startDate = null, endDate = null) {
+  try {
+    await initPool();
+    
+    console.log('='.repeat(60));
+    console.log('开始检查和补充人民币美元汇率数据...');
+    console.log('='.repeat(60));
+
+    const currentDate = new Date();
+    
+    if (!startDate) {
+      startDate = new Date(currentDate);
+      startDate.setDate(startDate.getDate() - 30);
+      startDate = startDate.toISOString().split('T')[0];
+    }
+
+    if (!endDate) {
+      endDate = currentDate.toISOString().split('T')[0];
+    }
+
+    console.log(`检查时间范围: ${startDate} 至 ${endDate}`);
+
+    // 1. 检查数据库中是否存在汇率数据表
+    const hasTable = await checkTableExists();
+    if (!hasTable) {
+      console.log('创建汇率数据表...');
+      await createExchangeRateTable();
+    }
+
+    // 2. 检查数据库中已有的数据
+    const existingData = await getExistingExchangeRates(startDate, endDate);
+    console.log(`数据库中已有 ${existingData.length} 条数据`);
+
+    // 3. 计算缺失的日期
+    const missingDates = calculateMissingDates(startDate, endDate, existingData);
+    console.log(`缺失 ${missingDates.length} 天的数据`);
+
+    // 4. 如果有缺失，调用 fetcher 补足
+    if (missingDates.length > 0) {
+      console.log('\n开始获取缺失的汇率数据...');
+      
+      const firstMissingDate = missingDates[0];
+      const lastMissingDate = missingDates[missingDates.length - 1];
+      
+      console.log(`获取范围: ${firstMissingDate} 至 ${lastMissingDate}`);
+      
+      const fetchedData = await fetchUSDToCNYRate(firstMissingDate, lastMissingDate);
+      
+      console.log(`\n获取到 ${fetchedData.count} 条数据`);
+      
+      // 5. 存储新数据到数据库
+      const storedCount = await storeExchangeRates(fetchedData.data);
+      console.log(`成功存储 ${storedCount} 条数据`);
+    } else {
+      console.log('\n数据库数据完整，无需补充');
+    }
+
+    // 6. 验证最终数据完整性
+    const finalData = await getExistingExchangeRates(startDate, endDate);
+    console.log(`\n验证后的数据总量: ${finalData.length} 条`);
+
+    return {
+      startDate,
+      endDate,
+      existingCount: existingData.length,
+      missingCount: missingDates.length,
+      finalCount: finalData.length,
+      success: true
+    };
+
+  } catch (error) {
+    console.error('检查和补充汇率数据失败:', error.message);
+    throw error;
+  } finally {
+    if (pool) {
+      await pool.end();
+      console.log('数据库连接池已关闭');
+    }
+  }
+}
+
+async function checkTableExists() {
+  try {
+    console.log('检查汇率数据表是否存在...');
+    const result = await pool.query('SHOW TABLES LIKE ?', ['econ_ExchangeRate']);
+    return result.length > 0;
+  } catch (error) {
+    console.error('检查表存在失败:', error.message);
+    return false;
+  }
+}
+
+async function createExchangeRateTable() {
+  try {
+    console.log('执行创建汇率数据表SQL...');
+    await pool.query(`
+      CREATE TABLE IF NOT EXISTS econ_ExchangeRate (
+        id INT AUTO_INCREMENT PRIMARY KEY,
+        date DATE UNIQUE NOT NULL,
+        currency VARCHAR(10) NOT NULL,
+        centerPrice DECIMAL(10,4) NOT NULL,
+        sellingRate DECIMAL(10,4) NOT NULL,
+        buyingRate DECIMAL(10,4) NOT NULL,
+        source VARCHAR(100) NOT NULL,
+        createdAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+        updatedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
+      ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+    `);
+    console.log('汇率数据表创建成功');
+  } catch (error) {
+    console.error('创建汇率数据表失败:', error.message);
+    throw error;
+  }
+}
+
+async function getExistingExchangeRates(startDate, endDate) {
+  try {
+    console.log('查询已有汇率数据...');
+    const result = await pool.query(
+      'SELECT date FROM econ_ExchangeRate WHERE date BETWEEN ? AND ? ORDER BY date',
+      [startDate, endDate]
+    );
+    const dates = result.map(row => {
+      if (row.date instanceof Date) {
+        return row.date.toISOString().split('T')[0];
+      } else if (typeof row.date === 'string') {
+        return row.date;
+      }
+      return null;
+    }).filter(Boolean);
+    console.log(`查询到 ${dates.length} 条数据`);
+    return dates;
+  } catch (error) {
+    console.error('查询已有汇率数据失败:', error.message);
+    return [];
+  }
+}
+
+function calculateMissingDates(startDate, endDate, existingDates) {
+  const existingSet = new Set(existingDates);
+  const missingDates = [];
+  
+  const currentDate = new Date(startDate);
+  const finalDate = new Date(endDate);
+  
+  while (currentDate <= finalDate) {
+    const dateStr = currentDate.toISOString().split('T')[0];
+    const dayOfWeek = currentDate.getDay();
+    
+    // 跳过周末（周末没有汇率数据）
+    if (dayOfWeek !== 0 && dayOfWeek !== 6) {
+      if (!existingSet.has(dateStr)) {
+        missingDates.push(dateStr);
+      }
+    }
+    
+    currentDate.setDate(currentDate.getDate() + 1);
+  }
+  
+  return missingDates;
+}
+
+async function storeExchangeRates(rates) {
+  let storedCount = 0;
+  
+  for (const rate of rates) {
+    try {
+      await pool.query(
+        'INSERT INTO econ_ExchangeRate (date, currency, centerPrice, sellingRate, buyingRate, source) VALUES (?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE centerPrice = VALUES(centerPrice), sellingRate = VALUES(sellingRate), buyingRate = VALUES(buyingRate), source = VALUES(source)',
+        [rate.date, rate.currency, rate.centerPrice, rate.sellingRate, rate.buyingRate, rate.source]
+      );
+      storedCount++;
+    } catch (error) {
+      console.error(`存储汇率数据 ${rate.date} 失败:`, error.message);
+    }
+  }
+  
+  return storedCount;
+}
+
+export async function trigger(startDate = null, endDate = null) {
+  return await checkAndFillUSDCNYRate(startDate, endDate);
+}
+
+// 直接运行测试
+console.log('='.repeat(60));
+console.log('测试 usd-cny-rate.scraper.js');
+console.log('='.repeat(60));
+
+trigger()
+  .then(result => {
+    console.log('\n' + '='.repeat(60));
+    console.log('测试结果:');
+    console.log(`  开始日期: ${result.startDate}`);
+    console.log(`  结束日期: ${result.endDate}`);
+    console.log(`  原有数据: ${result.existingCount} 条`);
+    console.log(`  缺失数据: ${result.missingCount} 条`);
+    console.log(`  最终数据: ${result.finalCount} 条`);
+    console.log(`  操作成功: ${result.success}`);
+    console.log('='.repeat(60));
+  })
+  .catch(error => {
+    console.error('测试失败:', error);
+    process.exit(1);
+  });