339 lines
10 KiB
JavaScript
339 lines
10 KiB
JavaScript
import { createPool } from 'mariadb';
|
||
import { fetch as fetchSH300Data } from '../fetchers/sh300.fetcher.js';
|
||
import fs from 'fs';
|
||
import path from 'path';
|
||
|
||
let pool;
|
||
|
||
function loadDatabaseConfig() {
|
||
const envPath = path.join(process.cwd(), '.env');
|
||
if (fs.existsSync(envPath)) {
|
||
console.log('读取 .env 文件...');
|
||
const envContent = fs.readFileSync(envPath, 'utf-8');
|
||
const lines = envContent.split('\n');
|
||
|
||
for (const line of lines) {
|
||
const trimmedLine = line.trim();
|
||
if (trimmedLine.startsWith('DATABASE_URL=')) {
|
||
const url = trimmedLine.substring('DATABASE_URL='.length).replace(/^"|"$/g, '');
|
||
console.log('从 .env 文件获取数据库配置成功!');
|
||
console.log(`DATABASE_URL: ${url}`);
|
||
|
||
// 解析URL
|
||
const parts = url.split('://')[1].split('@');
|
||
const auth = parts[0].split(':');
|
||
const hostAndDb = parts[1].split('/');
|
||
const hostAndPort = hostAndDb[0].split(':');
|
||
|
||
return {
|
||
user: auth[0],
|
||
password: auth[1],
|
||
host: hostAndPort[0],
|
||
port: parseInt(hostAndPort[1]),
|
||
database: hostAndDb[1]
|
||
};
|
||
}
|
||
}
|
||
}
|
||
|
||
console.log('未找到 .env 文件或 DATABASE_URL 配置，使用默认配置');
|
||
return {
|
||
host: '192.168.111.111',
|
||
port: 3306,
|
||
user: 'root',
|
||
password: 'fullstack',
|
||
database: 'iboard'
|
||
};
|
||
}
|
||
|
||
async function initPool() {
|
||
if (!pool) {
|
||
console.log('初始化数据库连接池...');
|
||
const config = loadDatabaseConfig();
|
||
|
||
console.log('数据库配置:');
|
||
console.log(` 主机: ${config.host}`);
|
||
console.log(` 端口: ${config.port}`);
|
||
console.log(` 用户: ${config.user}`);
|
||
console.log(` 数据库: ${config.database}`);
|
||
|
||
pool = createPool({
|
||
host: config.host,
|
||
port: config.port,
|
||
user: config.user,
|
||
password: config.password,
|
||
database: config.database,
|
||
connectionLimit: 5
|
||
});
|
||
|
||
try {
|
||
const conn = await pool.getConnection();
|
||
console.log('数据库连接成功!');
|
||
conn.release();
|
||
} catch (error) {
|
||
console.error('数据库连接失败:', error.message);
|
||
throw error;
|
||
}
|
||
}
|
||
return pool;
|
||
}
|
||
|
||
async function checkAndFillSH300Data(startDate = null, endDate = null) {
|
||
try {
|
||
await initPool();
|
||
|
||
console.log('='.repeat(60));
|
||
console.log('开始检查和补充沪深300指数数据...');
|
||
console.log('='.repeat(60));
|
||
|
||
const currentDate = new Date();
|
||
|
||
if (!startDate) {
|
||
startDate = new Date(currentDate);
|
||
startDate.setDate(startDate.getDate() - 30);
|
||
startDate = startDate.toISOString().split('T')[0];
|
||
}
|
||
|
||
if (!endDate) {
|
||
endDate = currentDate.toISOString().split('T')[0];
|
||
}
|
||
|
||
console.log(`检查时间范围: ${startDate} 至 ${endDate}`);
|
||
|
||
// 1. 检查数据库中是否存在沪深300指数数据表
|
||
const hasTable = await checkTableExists();
|
||
if (!hasTable) {
|
||
console.log('创建沪深300指数数据表...');
|
||
await createSH300Table();
|
||
}
|
||
|
||
// 2. 检查数据库中已有的数据
|
||
const existingData = await getExistingSH300Data(startDate, endDate);
|
||
console.log(`数据库中已有 ${existingData.length} 条数据`);
|
||
|
||
// 3. 计算缺失的日期
|
||
const missingDates = calculateMissingDates(startDate, endDate, existingData);
|
||
console.log(`缺失 ${missingDates.length} 天的数据`);
|
||
|
||
// 4. 如果有缺失，调用 fetcher 补足
|
||
if (missingDates.length > 0) {
|
||
console.log('\n开始获取缺失的沪深300指数数据...');
|
||
|
||
const firstMissingDate = missingDates[0];
|
||
const lastMissingDate = missingDates[missingDates.length - 1];
|
||
|
||
console.log(`获取范围: ${firstMissingDate} 至 ${lastMissingDate}`);
|
||
|
||
const fetchedData = await fetchSH300Data(firstMissingDate, lastMissingDate);
|
||
|
||
console.log(`\n获取到 ${fetchedData.count} 条数据`);
|
||
|
||
// 5. 存储新数据到数据库
|
||
const storedCount = await storeSH300Data(fetchedData.data);
|
||
console.log(`成功存储 ${storedCount} 条数据`);
|
||
} else {
|
||
console.log('\n数据库数据完整，无需补充');
|
||
}
|
||
|
||
// 6. 验证最终数据完整性
|
||
const finalData = await getExistingSH300Data(startDate, endDate);
|
||
console.log(`\n验证后的数据总量: ${finalData.length} 条`);
|
||
|
||
return {
|
||
startDate,
|
||
endDate,
|
||
existingCount: existingData.length,
|
||
missingCount: missingDates.length,
|
||
finalCount: finalData.length,
|
||
success: true
|
||
};
|
||
|
||
} catch (error) {
|
||
console.error('检查和补充沪深300指数数据失败:', error.message);
|
||
throw error;
|
||
} finally {
|
||
if (pool) {
|
||
await pool.end();
|
||
console.log('数据库连接池已关闭');
|
||
}
|
||
}
|
||
}
|
||
|
||
async function checkTableExists() {
|
||
try {
|
||
console.log('检查沪深300指数数据表是否存在...');
|
||
const result = await pool.query('SHOW TABLES LIKE ?', ['econ_SH300Index']);
|
||
return result.length > 0;
|
||
} catch (error) {
|
||
console.error('检查表存在失败:', error.message);
|
||
return false;
|
||
}
|
||
}
|
||
|
||
async function createSH300Table() {
|
||
try {
|
||
console.log('执行创建沪深300指数数据表SQL...');
|
||
await pool.query(`
|
||
CREATE TABLE IF NOT EXISTS econ_SH300Index (
|
||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||
date DATE UNIQUE NOT NULL,
|
||
symbol VARCHAR(10) NOT NULL,
|
||
open DECIMAL(10,2) NOT NULL,
|
||
close DECIMAL(10,2) NOT NULL,
|
||
high DECIMAL(10,2) NOT NULL,
|
||
low DECIMAL(10,2) NOT NULL,
|
||
volume DECIMAL(20,2) NOT NULL,
|
||
amount DECIMAL(20,2) NOT NULL,
|
||
source VARCHAR(100) NOT NULL,
|
||
createdAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||
updatedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
|
||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||
`);
|
||
console.log('沪深300指数数据表创建成功');
|
||
} catch (error) {
|
||
console.error('创建沪深300指数数据表失败:', error.message);
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
async function getExistingSH300Data(startDate, endDate) {
|
||
try {
|
||
console.log('查询已有沪深300指数数据...');
|
||
const result = await pool.query(
|
||
'SELECT date FROM econ_SH300Index WHERE date BETWEEN ? AND ? ORDER BY date',
|
||
[startDate, endDate]
|
||
);
|
||
const dates = result.map(row => {
|
||
if (row.date instanceof Date) {
|
||
return row.date.toISOString().split('T')[0];
|
||
} else if (typeof row.date === 'string') {
|
||
return row.date;
|
||
}
|
||
return null;
|
||
}).filter(Boolean);
|
||
console.log(`查询到 ${dates.length} 条数据`);
|
||
return dates;
|
||
} catch (error) {
|
||
console.error('查询已有沪深300指数数据失败:', error.message);
|
||
return [];
|
||
}
|
||
}
|
||
|
||
function calculateMissingDates(startDate, endDate, existingDates) {
|
||
const existingSet = new Set(existingDates);
|
||
const missingDates = [];
|
||
|
||
const currentDate = new Date(startDate);
|
||
const finalDate = new Date(endDate);
|
||
|
||
while (currentDate <= finalDate) {
|
||
const dateStr = currentDate.toISOString().split('T')[0];
|
||
const dayOfWeek = currentDate.getDay();
|
||
|
||
// 跳过周末（周末没有股市数据）
|
||
if (dayOfWeek !== 0 && dayOfWeek !== 6) {
|
||
if (!existingSet.has(dateStr)) {
|
||
missingDates.push(dateStr);
|
||
}
|
||
}
|
||
|
||
currentDate.setDate(currentDate.getDate() + 1);
|
||
}
|
||
|
||
return missingDates;
|
||
}
|
||
|
||
async function storeSH300Data(data) {
|
||
let storedCount = 0;
|
||
|
||
for (const item of data) {
|
||
try {
|
||
await pool.query(
|
||
'INSERT INTO econ_SH300Index (date, symbol, open, close, high, low, volume, amount, source) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE open = VALUES(open), close = VALUES(close), high = VALUES(high), low = VALUES(low), volume = VALUES(volume), amount = VALUES(amount), source = VALUES(source)',
|
||
[item.date, 'sh000300', item.open, item.close, item.high, item.low, item.volume, item.amount, item.source]
|
||
);
|
||
storedCount++;
|
||
} catch (error) {
|
||
console.error(`存储沪深300指数数据 ${item.date} 失败:`, error.message);
|
||
}
|
||
}
|
||
|
||
return storedCount;
|
||
}
|
||
|
||
export async function trigger(startDate = null, endDate = null) {
|
||
return await checkAndFillSH300Data(startDate, endDate);
|
||
}
|
||
|
||
function parseCommandLineArgs() {
|
||
const args = process.argv.slice(2);
|
||
let startDate = null;
|
||
let endDate = null;
|
||
|
||
for (let i = 0; i < args.length; i++) {
|
||
if (args[i] === '--start' && i + 1 < args.length) {
|
||
startDate = args[i + 1];
|
||
i++;
|
||
} else if (args[i] === '--end' && i + 1 < args.length) {
|
||
endDate = args[i + 1];
|
||
i++;
|
||
} else if (args[i] === '--help') {
|
||
console.log('使用方法:');
|
||
console.log(' bun sh300.scraper.js [选项]');
|
||
console.log('');
|
||
console.log('选项:');
|
||
console.log(' --start YYYY-MM-DD 指定开始日期');
|
||
console.log(' --end YYYY-MM-DD 指定结束日期');
|
||
console.log(' --help 显示帮助信息');
|
||
console.log('');
|
||
console.log('示例:');
|
||
console.log(' bun sh300.scraper.js');
|
||
console.log(' bun sh300.scraper.js --start 2024-01-01 --end 2024-01-31');
|
||
process.exit(0);
|
||
}
|
||
}
|
||
|
||
return { startDate, endDate };
|
||
}
|
||
|
||
async function runTest() {
|
||
console.log('='.repeat(60));
|
||
console.log('测试 sh300.scraper.js');
|
||
console.log('='.repeat(60));
|
||
|
||
const { startDate, endDate } = parseCommandLineArgs();
|
||
|
||
if (startDate) {
|
||
console.log(`\n使用命令行参数:`);
|
||
console.log(` 开始日期: ${startDate}`);
|
||
console.log(` 结束日期: ${endDate || '默认'}`);
|
||
}
|
||
|
||
console.log('\n测试数据库配置读取...');
|
||
const config = loadDatabaseConfig();
|
||
console.log('配置读取结果:');
|
||
console.log(` 主机: ${config.host}`);
|
||
console.log(` 端口: ${config.port}`);
|
||
console.log(` 用户: ${config.user}`);
|
||
console.log(` 数据库: ${config.database}`);
|
||
console.log('配置读取测试完成!');
|
||
|
||
try {
|
||
const result = await trigger(startDate, endDate);
|
||
console.log('\n' + '='.repeat(60));
|
||
console.log('测试结果:');
|
||
console.log(` 开始日期: ${result.startDate}`);
|
||
console.log(` 结束日期: ${result.endDate}`);
|
||
console.log(` 原有数据: ${result.existingCount} 条`);
|
||
console.log(` 缺失数据: ${result.missingCount} 条`);
|
||
console.log(` 最终数据: ${result.finalCount} 条`);
|
||
console.log(` 操作成功: ${result.success}`);
|
||
console.log('='.repeat(60));
|
||
} catch (error) {
|
||
console.error('测试失败:', error);
|
||
process.exit(1);
|
||
}
|
||
}
|
||
|
||
runTest();
|