diff --git a/readme b/readme index 415d140..304121f 100644 --- a/readme +++ b/readme @@ -4,3 +4,16 @@ - 数据主要是放各种经济数据和图表。 - 策略是放按照经济理论组织的相关数据和图表的对比。 + +## 爬虫 +- 基于 bun 的定时任务 +- 统一获取接口 +- 统一爬取接口 +``` +async function fetch(startDate = null, endDate = null) +async function trigger(startDate = null, endDate = null) +``` +不传递startDate和endDate,默认,默认获取最近30天的数据。 + +fecher 负责从目标网站获取原始数据 +scraper 负责检查数据库中的数据是否完整,调用 fecher 将数据存储到数据库。 diff --git a/scripts/scraper/usd-cny-rate.scraper.js b/scripts/scraper/usd-cny-rate.scraper.js new file mode 100644 index 0000000..eec7765 --- /dev/null +++ b/scripts/scraper/usd-cny-rate.scraper.js @@ -0,0 +1,234 @@ +import { createPool } from 'mariadb'; +import { fetch as fetchUSDToCNYRate } from '../fetchers/usd-cny-rate.fetcher.js'; + +let pool; + +async function initPool() { + if (!pool) { + console.log('初始化数据库连接池...'); + pool = createPool({ + host: '192.168.111.111', + port: 3306, + user: 'root', + password: 'fullstack', + database: 'iboard', + connectionLimit: 5 + }); + + try { + const conn = await pool.getConnection(); + console.log('数据库连接成功!'); + conn.release(); + } catch (error) { + console.error('数据库连接失败:', error.message); + throw error; + } + } + return pool; +} + +async function checkAndFillUSDCNYRate(startDate = null, endDate = null) { + try { + await initPool(); + + console.log('='.repeat(60)); + console.log('开始检查和补充人民币美元汇率数据...'); + console.log('='.repeat(60)); + + const currentDate = new Date(); + + if (!startDate) { + startDate = new Date(currentDate); + startDate.setDate(startDate.getDate() - 30); + startDate = startDate.toISOString().split('T')[0]; + } + + if (!endDate) { + endDate = currentDate.toISOString().split('T')[0]; + } + + console.log(`检查时间范围: ${startDate} 至 ${endDate}`); + + // 1. 检查数据库中是否存在汇率数据表 + const hasTable = await checkTableExists(); + if (!hasTable) { + console.log('创建汇率数据表...'); + await createExchangeRateTable(); + } + + // 2. 检查数据库中已有的数据 + const existingData = await getExistingExchangeRates(startDate, endDate); + console.log(`数据库中已有 ${existingData.length} 条数据`); + + // 3. 计算缺失的日期 + const missingDates = calculateMissingDates(startDate, endDate, existingData); + console.log(`缺失 ${missingDates.length} 天的数据`); + + // 4. 如果有缺失,调用 fetcher 补足 + if (missingDates.length > 0) { + console.log('\n开始获取缺失的汇率数据...'); + + const firstMissingDate = missingDates[0]; + const lastMissingDate = missingDates[missingDates.length - 1]; + + console.log(`获取范围: ${firstMissingDate} 至 ${lastMissingDate}`); + + const fetchedData = await fetchUSDToCNYRate(firstMissingDate, lastMissingDate); + + console.log(`\n获取到 ${fetchedData.count} 条数据`); + + // 5. 存储新数据到数据库 + const storedCount = await storeExchangeRates(fetchedData.data); + console.log(`成功存储 ${storedCount} 条数据`); + } else { + console.log('\n数据库数据完整,无需补充'); + } + + // 6. 验证最终数据完整性 + const finalData = await getExistingExchangeRates(startDate, endDate); + console.log(`\n验证后的数据总量: ${finalData.length} 条`); + + return { + startDate, + endDate, + existingCount: existingData.length, + missingCount: missingDates.length, + finalCount: finalData.length, + success: true + }; + + } catch (error) { + console.error('检查和补充汇率数据失败:', error.message); + throw error; + } finally { + if (pool) { + await pool.end(); + console.log('数据库连接池已关闭'); + } + } +} + +async function checkTableExists() { + try { + console.log('检查汇率数据表是否存在...'); + const result = await pool.query('SHOW TABLES LIKE ?', ['econ_ExchangeRate']); + return result.length > 0; + } catch (error) { + console.error('检查表存在失败:', error.message); + return false; + } +} + +async function createExchangeRateTable() { + try { + console.log('执行创建汇率数据表SQL...'); + await pool.query(` + CREATE TABLE IF NOT EXISTS econ_ExchangeRate ( + id INT AUTO_INCREMENT PRIMARY KEY, + date DATE UNIQUE NOT NULL, + currency VARCHAR(10) NOT NULL, + centerPrice DECIMAL(10,4) NOT NULL, + sellingRate DECIMAL(10,4) NOT NULL, + buyingRate DECIMAL(10,4) NOT NULL, + source VARCHAR(100) NOT NULL, + createdAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updatedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + `); + console.log('汇率数据表创建成功'); + } catch (error) { + console.error('创建汇率数据表失败:', error.message); + throw error; + } +} + +async function getExistingExchangeRates(startDate, endDate) { + try { + console.log('查询已有汇率数据...'); + const result = await pool.query( + 'SELECT date FROM econ_ExchangeRate WHERE date BETWEEN ? AND ? ORDER BY date', + [startDate, endDate] + ); + const dates = result.map(row => { + if (row.date instanceof Date) { + return row.date.toISOString().split('T')[0]; + } else if (typeof row.date === 'string') { + return row.date; + } + return null; + }).filter(Boolean); + console.log(`查询到 ${dates.length} 条数据`); + return dates; + } catch (error) { + console.error('查询已有汇率数据失败:', error.message); + return []; + } +} + +function calculateMissingDates(startDate, endDate, existingDates) { + const existingSet = new Set(existingDates); + const missingDates = []; + + const currentDate = new Date(startDate); + const finalDate = new Date(endDate); + + while (currentDate <= finalDate) { + const dateStr = currentDate.toISOString().split('T')[0]; + const dayOfWeek = currentDate.getDay(); + + // 跳过周末(周末没有汇率数据) + if (dayOfWeek !== 0 && dayOfWeek !== 6) { + if (!existingSet.has(dateStr)) { + missingDates.push(dateStr); + } + } + + currentDate.setDate(currentDate.getDate() + 1); + } + + return missingDates; +} + +async function storeExchangeRates(rates) { + let storedCount = 0; + + for (const rate of rates) { + try { + await pool.query( + 'INSERT INTO econ_ExchangeRate (date, currency, centerPrice, sellingRate, buyingRate, source) VALUES (?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE centerPrice = VALUES(centerPrice), sellingRate = VALUES(sellingRate), buyingRate = VALUES(buyingRate), source = VALUES(source)', + [rate.date, rate.currency, rate.centerPrice, rate.sellingRate, rate.buyingRate, rate.source] + ); + storedCount++; + } catch (error) { + console.error(`存储汇率数据 ${rate.date} 失败:`, error.message); + } + } + + return storedCount; +} + +export async function trigger(startDate = null, endDate = null) { + return await checkAndFillUSDCNYRate(startDate, endDate); +} + +// 直接运行测试 +console.log('='.repeat(60)); +console.log('测试 usd-cny-rate.scraper.js'); +console.log('='.repeat(60)); + +trigger() + .then(result => { + console.log('\n' + '='.repeat(60)); + console.log('测试结果:'); + console.log(` 开始日期: ${result.startDate}`); + console.log(` 结束日期: ${result.endDate}`); + console.log(` 原有数据: ${result.existingCount} 条`); + console.log(` 缺失数据: ${result.missingCount} 条`); + console.log(` 最终数据: ${result.finalCount} 条`); + console.log(` 操作成功: ${result.success}`); + console.log('='.repeat(60)); + }) + .catch(error => { + console.error('测试失败:', error); + process.exit(1); + });