获取沪深 300 数据
All checks were successful
Docker Build and Push / build-image (push) Successful in 4m44s
All checks were successful
Docker Build and Push / build-image (push) Successful in 4m44s
This commit is contained in:
parent
306af38fdc
commit
f2824d88f7
168
scripts/fetchers/sh300.fetcher.js
Normal file
168
scripts/fetchers/sh300.fetcher.js
Normal file
@ -0,0 +1,168 @@
|
||||
import axios from 'axios';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
async function fetch(startDate = null, endDate = null, retryCount = 3) {
|
||||
try {
|
||||
const currentDate = new Date();
|
||||
|
||||
if (!startDate) {
|
||||
startDate = new Date(currentDate);
|
||||
startDate.setDate(startDate.getDate() - 30);
|
||||
startDate = startDate.toISOString().split('T')[0];
|
||||
}
|
||||
|
||||
if (!endDate) {
|
||||
endDate = currentDate.toISOString().split('T')[0];
|
||||
}
|
||||
|
||||
console.log('正在获取沪深300指数数据...');
|
||||
console.log(`时间范围: ${startDate} 至 ${endDate}`);
|
||||
console.log(`数据源: 腾讯财经API`);
|
||||
|
||||
// 沪深300指数代码
|
||||
const symbol = '000300';
|
||||
|
||||
// 使用腾讯财经的API端点获取历史数据
|
||||
const response = await axios.get('https://web.ifzq.gtimg.cn/appstock/app/kline/kline?param=sh' + symbol + ',day,' + startDate + ',' + endDate + ',640', {
|
||||
headers: {
|
||||
'Accept': 'application/json',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
|
||||
},
|
||||
timeout: 10000 // 设置10秒超时
|
||||
});
|
||||
|
||||
console.log(`\nAPI状态: ${response.status}`);
|
||||
|
||||
const data = response.data;
|
||||
console.log(`响应数据:`, JSON.stringify(data, null, 2));
|
||||
|
||||
const allData = [];
|
||||
|
||||
if (data && data.data && data.data['sh' + symbol] && data.data['sh' + symbol].day) {
|
||||
const klineData = data.data['sh' + symbol].day;
|
||||
console.log(`数据条数: ${klineData.length}`);
|
||||
|
||||
for (const item of klineData) {
|
||||
allData.push({
|
||||
date: item[0],
|
||||
open: parseFloat(item[1]),
|
||||
close: parseFloat(item[2]),
|
||||
high: parseFloat(item[3]),
|
||||
low: parseFloat(item[4]),
|
||||
volume: parseFloat(item[5]),
|
||||
amount: 0, // 腾讯财经API没有返回成交额数据，设置为0
|
||||
source: '腾讯财经API',
|
||||
fetchTime: new Date().toISOString()
|
||||
});
|
||||
}
|
||||
} else {
|
||||
console.log('数据条数: 0');
|
||||
}
|
||||
|
||||
allData.sort((a, b) => a.date.localeCompare(b.date));
|
||||
|
||||
console.log(`\n共获取到 ${allData.length} 条沪深300指数数据`);
|
||||
|
||||
return {
|
||||
symbol: 'sh000300',
|
||||
name: '沪深300指数',
|
||||
startDate,
|
||||
endDate,
|
||||
count: allData.length,
|
||||
data: allData,
|
||||
fetchTime: new Date().toISOString(),
|
||||
note: '数据来源: 腾讯财经API'
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
console.error('获取沪深300指数数据失败:', error.message);
|
||||
if (error.response) {
|
||||
console.error(`HTTP状态码: ${error.response.status}`);
|
||||
console.error(`响应数据:`, error.response.data);
|
||||
}
|
||||
|
||||
// 重试机制
|
||||
if (retryCount > 0) {
|
||||
console.log(`正在重试... (剩余重试次数: ${retryCount - 1})`);
|
||||
// 等待1秒后重试
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
return fetch(startDate, endDate, retryCount - 1);
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export {
|
||||
fetch
|
||||
};
|
||||
|
||||
const isMainModule = () => {
|
||||
const currentFile = fileURLToPath(import.meta.url);
|
||||
const mainFile = process.argv[1];
|
||||
return currentFile === mainFile;
|
||||
};
|
||||
|
||||
function parseCommandLineArgs() {
|
||||
const args = process.argv.slice(2);
|
||||
let startDate = null;
|
||||
let endDate = null;
|
||||
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
if (args[i] === '--start' && i + 1 < args.length) {
|
||||
startDate = args[i + 1];
|
||||
i++;
|
||||
} else if (args[i] === '--end' && i + 1 < args.length) {
|
||||
endDate = args[i + 1];
|
||||
i++;
|
||||
} else if (args[i] === '--help') {
|
||||
console.log('使用方法:');
|
||||
console.log(' bun sh300.fetcher.js [选项]');
|
||||
console.log('');
|
||||
console.log('选项:');
|
||||
console.log(' --start YYYY-MM-DD 指定开始日期（默认：30天前）');
|
||||
console.log(' --end YYYY-MM-DD 指定结束日期（默认：今天）');
|
||||
console.log(' --help 显示帮助信息');
|
||||
console.log('');
|
||||
console.log('示例:');
|
||||
console.log(' bun sh300.fetcher.js');
|
||||
console.log(' bun sh300.fetcher.js --start 2024-01-01 --end 2024-01-31');
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
return { startDate, endDate };
|
||||
}
|
||||
|
||||
if (isMainModule()) {
|
||||
const { startDate, endDate } = parseCommandLineArgs();
|
||||
|
||||
console.log('='.repeat(50));
|
||||
console.log('沪深300指数数据获取脚本');
|
||||
console.log('='.repeat(50));
|
||||
|
||||
fetch(startDate, endDate)
|
||||
.then(result => {
|
||||
console.log('\n' + '='.repeat(50));
|
||||
console.log('获取结果:');
|
||||
console.log(` 指数代码: ${result.symbol}`);
|
||||
console.log(` 指数名称: ${result.name}`);
|
||||
console.log(` 开始日期: ${result.startDate}`);
|
||||
console.log(` 结束日期: ${result.endDate}`);
|
||||
console.log(` 数据条数: ${result.count}`);
|
||||
console.log(` 获取时间: ${result.fetchTime}`);
|
||||
console.log(` 提示: ${result.note}`);
|
||||
if (result.count > 0) {
|
||||
console.log('-'.repeat(50));
|
||||
console.log('指数数据:');
|
||||
result.data.forEach(item => {
|
||||
console.log(` ${item.date} 开盘: ${item.open} 最高: ${item.high} 最低: ${item.low} 收盘: ${item.close}`);
|
||||
});
|
||||
}
|
||||
console.log('='.repeat(50));
|
||||
})
|
||||
.catch(error => {
|
||||
console.error('执行失败:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
302
scripts/scraper/sh300.scraper.js
Normal file
302
scripts/scraper/sh300.scraper.js
Normal file
@ -0,0 +1,302 @@
|
||||
import { createPool } from 'mariadb';
|
||||
import { fetch as fetchSH300Data } from '../fetchers/sh300.fetcher.js';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
let pool;
|
||||
|
||||
function loadDatabaseConfig() {
|
||||
const envPath = path.join(process.cwd(), '.env');
|
||||
if (fs.existsSync(envPath)) {
|
||||
console.log('读取 .env 文件...');
|
||||
const envContent = fs.readFileSync(envPath, 'utf-8');
|
||||
const lines = envContent.split('\n');
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmedLine = line.trim();
|
||||
if (trimmedLine.startsWith('DATABASE_URL=')) {
|
||||
const url = trimmedLine.substring('DATABASE_URL='.length).replace(/^"|"$/g, '');
|
||||
console.log('从 .env 文件获取数据库配置成功!');
|
||||
console.log(`DATABASE_URL: ${url}`);
|
||||
|
||||
// 解析URL
|
||||
const parts = url.split('://')[1].split('@');
|
||||
const auth = parts[0].split(':');
|
||||
const hostAndDb = parts[1].split('/');
|
||||
const hostAndPort = hostAndDb[0].split(':');
|
||||
|
||||
return {
|
||||
user: auth[0],
|
||||
password: auth[1],
|
||||
host: hostAndPort[0],
|
||||
port: parseInt(hostAndPort[1]),
|
||||
database: hostAndDb[1]
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log('未找到 .env 文件或 DATABASE_URL 配置，使用默认配置');
|
||||
return {
|
||||
host: '192.168.111.111',
|
||||
port: 3306,
|
||||
user: 'root',
|
||||
password: 'fullstack',
|
||||
database: 'iboard'
|
||||
};
|
||||
}
|
||||
|
||||
async function initPool() {
|
||||
if (!pool) {
|
||||
console.log('初始化数据库连接池...');
|
||||
const config = loadDatabaseConfig();
|
||||
|
||||
console.log('数据库配置:');
|
||||
console.log(` 主机: ${config.host}`);
|
||||
console.log(` 端口: ${config.port}`);
|
||||
console.log(` 用户: ${config.user}`);
|
||||
console.log(` 数据库: ${config.database}`);
|
||||
|
||||
pool = createPool({
|
||||
host: config.host,
|
||||
port: config.port,
|
||||
user: config.user,
|
||||
password: config.password,
|
||||
database: config.database,
|
||||
connectionLimit: 5
|
||||
});
|
||||
|
||||
try {
|
||||
const conn = await pool.getConnection();
|
||||
console.log('数据库连接成功!');
|
||||
conn.release();
|
||||
} catch (error) {
|
||||
console.error('数据库连接失败:', error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
return pool;
|
||||
}
|
||||
|
||||
async function checkAndFillSH300Data(startDate = null, endDate = null) {
|
||||
try {
|
||||
await initPool();
|
||||
|
||||
console.log('='.repeat(60));
|
||||
console.log('开始检查和补充沪深300指数数据...');
|
||||
console.log('='.repeat(60));
|
||||
|
||||
const currentDate = new Date();
|
||||
|
||||
if (!startDate) {
|
||||
startDate = new Date(currentDate);
|
||||
startDate.setDate(startDate.getDate() - 30);
|
||||
startDate = startDate.toISOString().split('T')[0];
|
||||
}
|
||||
|
||||
if (!endDate) {
|
||||
endDate = currentDate.toISOString().split('T')[0];
|
||||
}
|
||||
|
||||
console.log(`检查时间范围: ${startDate} 至 ${endDate}`);
|
||||
|
||||
// 1. 检查数据库中是否存在沪深300指数数据表
|
||||
const hasTable = await checkTableExists();
|
||||
if (!hasTable) {
|
||||
console.log('创建沪深300指数数据表...');
|
||||
await createSH300Table();
|
||||
}
|
||||
|
||||
// 2. 检查数据库中已有的数据
|
||||
const existingData = await getExistingSH300Data(startDate, endDate);
|
||||
console.log(`数据库中已有 ${existingData.length} 条数据`);
|
||||
|
||||
// 3. 计算缺失的日期
|
||||
const missingDates = calculateMissingDates(startDate, endDate, existingData);
|
||||
console.log(`缺失 ${missingDates.length} 天的数据`);
|
||||
|
||||
// 4. 如果有缺失，调用 fetcher 补足
|
||||
if (missingDates.length > 0) {
|
||||
console.log('\n开始获取缺失的沪深300指数数据...');
|
||||
|
||||
const firstMissingDate = missingDates[0];
|
||||
const lastMissingDate = missingDates[missingDates.length - 1];
|
||||
|
||||
console.log(`获取范围: ${firstMissingDate} 至 ${lastMissingDate}`);
|
||||
|
||||
const fetchedData = await fetchSH300Data(firstMissingDate, lastMissingDate);
|
||||
|
||||
console.log(`\n获取到 ${fetchedData.count} 条数据`);
|
||||
|
||||
// 5. 存储新数据到数据库
|
||||
const storedCount = await storeSH300Data(fetchedData.data);
|
||||
console.log(`成功存储 ${storedCount} 条数据`);
|
||||
} else {
|
||||
console.log('\n数据库数据完整，无需补充');
|
||||
}
|
||||
|
||||
// 6. 验证最终数据完整性
|
||||
const finalData = await getExistingSH300Data(startDate, endDate);
|
||||
console.log(`\n验证后的数据总量: ${finalData.length} 条`);
|
||||
|
||||
return {
|
||||
startDate,
|
||||
endDate,
|
||||
existingCount: existingData.length,
|
||||
missingCount: missingDates.length,
|
||||
finalCount: finalData.length,
|
||||
success: true
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
console.error('检查和补充沪深300指数数据失败:', error.message);
|
||||
throw error;
|
||||
} finally {
|
||||
if (pool) {
|
||||
await pool.end();
|
||||
console.log('数据库连接池已关闭');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function checkTableExists() {
|
||||
try {
|
||||
console.log('检查沪深300指数数据表是否存在...');
|
||||
const result = await pool.query('SHOW TABLES LIKE ?', ['econ_SH300Index']);
|
||||
return result.length > 0;
|
||||
} catch (error) {
|
||||
console.error('检查表存在失败:', error.message);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function createSH300Table() {
|
||||
try {
|
||||
console.log('执行创建沪深300指数数据表SQL...');
|
||||
await pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS econ_SH300Index (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
date DATE UNIQUE NOT NULL,
|
||||
symbol VARCHAR(10) NOT NULL,
|
||||
open DECIMAL(10,2) NOT NULL,
|
||||
close DECIMAL(10,2) NOT NULL,
|
||||
high DECIMAL(10,2) NOT NULL,
|
||||
low DECIMAL(10,2) NOT NULL,
|
||||
volume DECIMAL(20,2) NOT NULL,
|
||||
amount DECIMAL(20,2) NOT NULL,
|
||||
source VARCHAR(100) NOT NULL,
|
||||
createdAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updatedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||||
`);
|
||||
console.log('沪深300指数数据表创建成功');
|
||||
} catch (error) {
|
||||
console.error('创建沪深300指数数据表失败:', error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function getExistingSH300Data(startDate, endDate) {
|
||||
try {
|
||||
console.log('查询已有沪深300指数数据...');
|
||||
const result = await pool.query(
|
||||
'SELECT date FROM econ_SH300Index WHERE date BETWEEN ? AND ? ORDER BY date',
|
||||
[startDate, endDate]
|
||||
);
|
||||
const dates = result.map(row => {
|
||||
if (row.date instanceof Date) {
|
||||
return row.date.toISOString().split('T')[0];
|
||||
} else if (typeof row.date === 'string') {
|
||||
return row.date;
|
||||
}
|
||||
return null;
|
||||
}).filter(Boolean);
|
||||
console.log(`查询到 ${dates.length} 条数据`);
|
||||
return dates;
|
||||
} catch (error) {
|
||||
console.error('查询已有沪深300指数数据失败:', error.message);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function calculateMissingDates(startDate, endDate, existingDates) {
|
||||
const existingSet = new Set(existingDates);
|
||||
const missingDates = [];
|
||||
|
||||
const currentDate = new Date(startDate);
|
||||
const finalDate = new Date(endDate);
|
||||
|
||||
while (currentDate <= finalDate) {
|
||||
const dateStr = currentDate.toISOString().split('T')[0];
|
||||
const dayOfWeek = currentDate.getDay();
|
||||
|
||||
// 跳过周末（周末没有股市数据）
|
||||
if (dayOfWeek !== 0 && dayOfWeek !== 6) {
|
||||
if (!existingSet.has(dateStr)) {
|
||||
missingDates.push(dateStr);
|
||||
}
|
||||
}
|
||||
|
||||
currentDate.setDate(currentDate.getDate() + 1);
|
||||
}
|
||||
|
||||
return missingDates;
|
||||
}
|
||||
|
||||
async function storeSH300Data(data) {
|
||||
let storedCount = 0;
|
||||
|
||||
for (const item of data) {
|
||||
try {
|
||||
await pool.query(
|
||||
'INSERT INTO econ_SH300Index (date, symbol, open, close, high, low, volume, amount, source) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE open = VALUES(open), close = VALUES(close), high = VALUES(high), low = VALUES(low), volume = VALUES(volume), amount = VALUES(amount), source = VALUES(source)',
|
||||
[item.date, 'sh000300', item.open, item.close, item.high, item.low, item.volume, item.amount, item.source]
|
||||
);
|
||||
storedCount++;
|
||||
} catch (error) {
|
||||
console.error(`存储沪深300指数数据 ${item.date} 失败:`, error.message);
|
||||
}
|
||||
}
|
||||
|
||||
return storedCount;
|
||||
}
|
||||
|
||||
export async function trigger(startDate = null, endDate = null) {
|
||||
return await checkAndFillSH300Data(startDate, endDate);
|
||||
}
|
||||
|
||||
// 直接运行测试
|
||||
async function runTest() {
|
||||
console.log('='.repeat(60));
|
||||
console.log('测试 sh300.scraper.js');
|
||||
console.log('='.repeat(60));
|
||||
|
||||
// 测试配置读取
|
||||
console.log('\n测试数据库配置读取...');
|
||||
const config = loadDatabaseConfig();
|
||||
console.log('配置读取结果:');
|
||||
console.log(` 主机: ${config.host}`);
|
||||
console.log(` 端口: ${config.port}`);
|
||||
console.log(` 用户: ${config.user}`);
|
||||
console.log(` 数据库: ${config.database}`);
|
||||
console.log('配置读取测试完成!');
|
||||
|
||||
try {
|
||||
const result = await trigger();
|
||||
console.log('\n' + '='.repeat(60));
|
||||
console.log('测试结果:');
|
||||
console.log(` 开始日期: ${result.startDate}`);
|
||||
console.log(` 结束日期: ${result.endDate}`);
|
||||
console.log(` 原有数据: ${result.existingCount} 条`);
|
||||
console.log(` 缺失数据: ${result.missingCount} 条`);
|
||||
console.log(` 最终数据: ${result.finalCount} 条`);
|
||||
console.log(` 操作成功: ${result.success}`);
|
||||
console.log('='.repeat(60));
|
||||
} catch (error) {
|
||||
console.error('测试失败:', error);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// 运行测试
|
||||
runTest();
|
||||
Loading…
Reference in New Issue
Block a user