diff --git a/scripts/fetchers/sh300.fetcher.js b/scripts/fetchers/sh300.fetcher.js new file mode 100644 index 0000000..138c572 --- /dev/null +++ b/scripts/fetchers/sh300.fetcher.js @@ -0,0 +1,168 @@ +import axios from 'axios'; +import { fileURLToPath } from 'url'; + +async function fetch(startDate = null, endDate = null, retryCount = 3) { + try { + const currentDate = new Date(); + + if (!startDate) { + startDate = new Date(currentDate); + startDate.setDate(startDate.getDate() - 30); + startDate = startDate.toISOString().split('T')[0]; + } + + if (!endDate) { + endDate = currentDate.toISOString().split('T')[0]; + } + + console.log('正在获取沪深300指数数据...'); + console.log(`时间范围: ${startDate} 至 ${endDate}`); + console.log(`数据源: 腾讯财经API`); + + // 沪深300指数代码 + const symbol = '000300'; + + // 使用腾讯财经的API端点获取历史数据 + const response = await axios.get('https://web.ifzq.gtimg.cn/appstock/app/kline/kline?param=sh' + symbol + ',day,' + startDate + ',' + endDate + ',640', { + headers: { + 'Accept': 'application/json', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' + }, + timeout: 10000 // 设置10秒超时 + }); + + console.log(`\nAPI状态: ${response.status}`); + + const data = response.data; + console.log(`响应数据:`, JSON.stringify(data, null, 2)); + + const allData = []; + + if (data && data.data && data.data['sh' + symbol] && data.data['sh' + symbol].day) { + const klineData = data.data['sh' + symbol].day; + console.log(`数据条数: ${klineData.length}`); + + for (const item of klineData) { + allData.push({ + date: item[0], + open: parseFloat(item[1]), + close: parseFloat(item[2]), + high: parseFloat(item[3]), + low: parseFloat(item[4]), + volume: parseFloat(item[5]), + amount: 0, // 腾讯财经API没有返回成交额数据,设置为0 + source: '腾讯财经API', + fetchTime: new Date().toISOString() + }); + } + } else { + console.log('数据条数: 0'); + } + + allData.sort((a, b) => a.date.localeCompare(b.date)); + + console.log(`\n共获取到 ${allData.length} 条沪深300指数数据`); + + return { + symbol: 'sh000300', + name: '沪深300指数', + startDate, + endDate, + count: allData.length, + data: allData, + fetchTime: new Date().toISOString(), + note: '数据来源: 腾讯财经API' + }; + + } catch (error) { + console.error('获取沪深300指数数据失败:', error.message); + if (error.response) { + console.error(`HTTP状态码: ${error.response.status}`); + console.error(`响应数据:`, error.response.data); + } + + // 重试机制 + if (retryCount > 0) { + console.log(`正在重试... (剩余重试次数: ${retryCount - 1})`); + // 等待1秒后重试 + await new Promise(resolve => setTimeout(resolve, 1000)); + return fetch(startDate, endDate, retryCount - 1); + } + + throw error; + } +} + +export { + fetch +}; + +const isMainModule = () => { + const currentFile = fileURLToPath(import.meta.url); + const mainFile = process.argv[1]; + return currentFile === mainFile; +}; + +function parseCommandLineArgs() { + const args = process.argv.slice(2); + let startDate = null; + let endDate = null; + + for (let i = 0; i < args.length; i++) { + if (args[i] === '--start' && i + 1 < args.length) { + startDate = args[i + 1]; + i++; + } else if (args[i] === '--end' && i + 1 < args.length) { + endDate = args[i + 1]; + i++; + } else if (args[i] === '--help') { + console.log('使用方法:'); + console.log(' bun sh300.fetcher.js [选项]'); + console.log(''); + console.log('选项:'); + console.log(' --start YYYY-MM-DD 指定开始日期(默认:30天前)'); + console.log(' --end YYYY-MM-DD 指定结束日期(默认:今天)'); + console.log(' --help 显示帮助信息'); + console.log(''); + console.log('示例:'); + console.log(' bun sh300.fetcher.js'); + console.log(' bun sh300.fetcher.js --start 2024-01-01 --end 2024-01-31'); + process.exit(0); + } + } + + return { startDate, endDate }; +} + +if (isMainModule()) { + const { startDate, endDate } = parseCommandLineArgs(); + + console.log('='.repeat(50)); + console.log('沪深300指数数据获取脚本'); + console.log('='.repeat(50)); + + fetch(startDate, endDate) + .then(result => { + console.log('\n' + '='.repeat(50)); + console.log('获取结果:'); + console.log(` 指数代码: ${result.symbol}`); + console.log(` 指数名称: ${result.name}`); + console.log(` 开始日期: ${result.startDate}`); + console.log(` 结束日期: ${result.endDate}`); + console.log(` 数据条数: ${result.count}`); + console.log(` 获取时间: ${result.fetchTime}`); + console.log(` 提示: ${result.note}`); + if (result.count > 0) { + console.log('-'.repeat(50)); + console.log('指数数据:'); + result.data.forEach(item => { + console.log(` ${item.date} 开盘: ${item.open} 最高: ${item.high} 最低: ${item.low} 收盘: ${item.close}`); + }); + } + console.log('='.repeat(50)); + }) + .catch(error => { + console.error('执行失败:', error); + process.exit(1); + }); +} diff --git a/scripts/scraper/sh300.scraper.js b/scripts/scraper/sh300.scraper.js new file mode 100644 index 0000000..959b54b --- /dev/null +++ b/scripts/scraper/sh300.scraper.js @@ -0,0 +1,302 @@ +import { createPool } from 'mariadb'; +import { fetch as fetchSH300Data } from '../fetchers/sh300.fetcher.js'; +import fs from 'fs'; +import path from 'path'; + +let pool; + +function loadDatabaseConfig() { + const envPath = path.join(process.cwd(), '.env'); + if (fs.existsSync(envPath)) { + console.log('读取 .env 文件...'); + const envContent = fs.readFileSync(envPath, 'utf-8'); + const lines = envContent.split('\n'); + + for (const line of lines) { + const trimmedLine = line.trim(); + if (trimmedLine.startsWith('DATABASE_URL=')) { + const url = trimmedLine.substring('DATABASE_URL='.length).replace(/^"|"$/g, ''); + console.log('从 .env 文件获取数据库配置成功!'); + console.log(`DATABASE_URL: ${url}`); + + // 解析URL + const parts = url.split('://')[1].split('@'); + const auth = parts[0].split(':'); + const hostAndDb = parts[1].split('/'); + const hostAndPort = hostAndDb[0].split(':'); + + return { + user: auth[0], + password: auth[1], + host: hostAndPort[0], + port: parseInt(hostAndPort[1]), + database: hostAndDb[1] + }; + } + } + } + + console.log('未找到 .env 文件或 DATABASE_URL 配置,使用默认配置'); + return { + host: '192.168.111.111', + port: 3306, + user: 'root', + password: 'fullstack', + database: 'iboard' + }; +} + +async function initPool() { + if (!pool) { + console.log('初始化数据库连接池...'); + const config = loadDatabaseConfig(); + + console.log('数据库配置:'); + console.log(` 主机: ${config.host}`); + console.log(` 端口: ${config.port}`); + console.log(` 用户: ${config.user}`); + console.log(` 数据库: ${config.database}`); + + pool = createPool({ + host: config.host, + port: config.port, + user: config.user, + password: config.password, + database: config.database, + connectionLimit: 5 + }); + + try { + const conn = await pool.getConnection(); + console.log('数据库连接成功!'); + conn.release(); + } catch (error) { + console.error('数据库连接失败:', error.message); + throw error; + } + } + return pool; +} + +async function checkAndFillSH300Data(startDate = null, endDate = null) { + try { + await initPool(); + + console.log('='.repeat(60)); + console.log('开始检查和补充沪深300指数数据...'); + console.log('='.repeat(60)); + + const currentDate = new Date(); + + if (!startDate) { + startDate = new Date(currentDate); + startDate.setDate(startDate.getDate() - 30); + startDate = startDate.toISOString().split('T')[0]; + } + + if (!endDate) { + endDate = currentDate.toISOString().split('T')[0]; + } + + console.log(`检查时间范围: ${startDate} 至 ${endDate}`); + + // 1. 检查数据库中是否存在沪深300指数数据表 + const hasTable = await checkTableExists(); + if (!hasTable) { + console.log('创建沪深300指数数据表...'); + await createSH300Table(); + } + + // 2. 检查数据库中已有的数据 + const existingData = await getExistingSH300Data(startDate, endDate); + console.log(`数据库中已有 ${existingData.length} 条数据`); + + // 3. 计算缺失的日期 + const missingDates = calculateMissingDates(startDate, endDate, existingData); + console.log(`缺失 ${missingDates.length} 天的数据`); + + // 4. 如果有缺失,调用 fetcher 补足 + if (missingDates.length > 0) { + console.log('\n开始获取缺失的沪深300指数数据...'); + + const firstMissingDate = missingDates[0]; + const lastMissingDate = missingDates[missingDates.length - 1]; + + console.log(`获取范围: ${firstMissingDate} 至 ${lastMissingDate}`); + + const fetchedData = await fetchSH300Data(firstMissingDate, lastMissingDate); + + console.log(`\n获取到 ${fetchedData.count} 条数据`); + + // 5. 存储新数据到数据库 + const storedCount = await storeSH300Data(fetchedData.data); + console.log(`成功存储 ${storedCount} 条数据`); + } else { + console.log('\n数据库数据完整,无需补充'); + } + + // 6. 验证最终数据完整性 + const finalData = await getExistingSH300Data(startDate, endDate); + console.log(`\n验证后的数据总量: ${finalData.length} 条`); + + return { + startDate, + endDate, + existingCount: existingData.length, + missingCount: missingDates.length, + finalCount: finalData.length, + success: true + }; + + } catch (error) { + console.error('检查和补充沪深300指数数据失败:', error.message); + throw error; + } finally { + if (pool) { + await pool.end(); + console.log('数据库连接池已关闭'); + } + } +} + +async function checkTableExists() { + try { + console.log('检查沪深300指数数据表是否存在...'); + const result = await pool.query('SHOW TABLES LIKE ?', ['econ_SH300Index']); + return result.length > 0; + } catch (error) { + console.error('检查表存在失败:', error.message); + return false; + } +} + +async function createSH300Table() { + try { + console.log('执行创建沪深300指数数据表SQL...'); + await pool.query(` + CREATE TABLE IF NOT EXISTS econ_SH300Index ( + id INT AUTO_INCREMENT PRIMARY KEY, + date DATE UNIQUE NOT NULL, + symbol VARCHAR(10) NOT NULL, + open DECIMAL(10,2) NOT NULL, + close DECIMAL(10,2) NOT NULL, + high DECIMAL(10,2) NOT NULL, + low DECIMAL(10,2) NOT NULL, + volume DECIMAL(20,2) NOT NULL, + amount DECIMAL(20,2) NOT NULL, + source VARCHAR(100) NOT NULL, + createdAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updatedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + `); + console.log('沪深300指数数据表创建成功'); + } catch (error) { + console.error('创建沪深300指数数据表失败:', error.message); + throw error; + } +} + +async function getExistingSH300Data(startDate, endDate) { + try { + console.log('查询已有沪深300指数数据...'); + const result = await pool.query( + 'SELECT date FROM econ_SH300Index WHERE date BETWEEN ? AND ? ORDER BY date', + [startDate, endDate] + ); + const dates = result.map(row => { + if (row.date instanceof Date) { + return row.date.toISOString().split('T')[0]; + } else if (typeof row.date === 'string') { + return row.date; + } + return null; + }).filter(Boolean); + console.log(`查询到 ${dates.length} 条数据`); + return dates; + } catch (error) { + console.error('查询已有沪深300指数数据失败:', error.message); + return []; + } +} + +function calculateMissingDates(startDate, endDate, existingDates) { + const existingSet = new Set(existingDates); + const missingDates = []; + + const currentDate = new Date(startDate); + const finalDate = new Date(endDate); + + while (currentDate <= finalDate) { + const dateStr = currentDate.toISOString().split('T')[0]; + const dayOfWeek = currentDate.getDay(); + + // 跳过周末(周末没有股市数据) + if (dayOfWeek !== 0 && dayOfWeek !== 6) { + if (!existingSet.has(dateStr)) { + missingDates.push(dateStr); + } + } + + currentDate.setDate(currentDate.getDate() + 1); + } + + return missingDates; +} + +async function storeSH300Data(data) { + let storedCount = 0; + + for (const item of data) { + try { + await pool.query( + 'INSERT INTO econ_SH300Index (date, symbol, open, close, high, low, volume, amount, source) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE open = VALUES(open), close = VALUES(close), high = VALUES(high), low = VALUES(low), volume = VALUES(volume), amount = VALUES(amount), source = VALUES(source)', + [item.date, 'sh000300', item.open, item.close, item.high, item.low, item.volume, item.amount, item.source] + ); + storedCount++; + } catch (error) { + console.error(`存储沪深300指数数据 ${item.date} 失败:`, error.message); + } + } + + return storedCount; +} + +export async function trigger(startDate = null, endDate = null) { + return await checkAndFillSH300Data(startDate, endDate); +} + +// 直接运行测试 +async function runTest() { + console.log('='.repeat(60)); + console.log('测试 sh300.scraper.js'); + console.log('='.repeat(60)); + + // 测试配置读取 + console.log('\n测试数据库配置读取...'); + const config = loadDatabaseConfig(); + console.log('配置读取结果:'); + console.log(` 主机: ${config.host}`); + console.log(` 端口: ${config.port}`); + console.log(` 用户: ${config.user}`); + console.log(` 数据库: ${config.database}`); + console.log('配置读取测试完成!'); + + try { + const result = await trigger(); + console.log('\n' + '='.repeat(60)); + console.log('测试结果:'); + console.log(` 开始日期: ${result.startDate}`); + console.log(` 结束日期: ${result.endDate}`); + console.log(` 原有数据: ${result.existingCount} 条`); + console.log(` 缺失数据: ${result.missingCount} 条`); + console.log(` 最终数据: ${result.finalCount} 条`); + console.log(` 操作成功: ${result.success}`); + console.log('='.repeat(60)); + } catch (error) { + console.error('测试失败:', error); + process.exit(1); + } +} + +// 运行测试 +runTest();