Fullstack/iboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

添加汇率数据

Browse Source
This commit is contained in:
cheney
2026-04-25 19:22:57 +08:00
parent b61cf9628e
commit d232591a38
10 changed files with 480 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/scraper/sow-inventory-scraper.js
+102 -53
View File
@@ -1,23 +1,82 @@
import axios from 'axios';
import * as cheerio from 'cheerio';
import { PrismaClient } from '../../generated/prisma/client.js';
import { PrismaMariaDb } from '@prisma/adapter-mariadb';
import { createPool } from 'mariadb';
import fs from 'fs';
import path from 'path';
function loadDatabaseConfig() {
const envPath = path.join(process.cwd(), '.env');
if (fs.existsSync(envPath)) {
console.log('读取 .env 文件...');
const envContent = fs.readFileSync(envPath, 'utf-8');
const lines = envContent.split('\n');
for (const line of lines) {
const trimmedLine = line.trim();
if (trimmedLine.startsWith('DATABASE_URL=')) {
const url = trimmedLine.substring('DATABASE_URL='.length).replace(/^"|"$/g, '');
console.log('从 .env 文件获取数据库配置成功!');
console.log(`DATABASE_URL: ${url}`);
// 解析URL
const parts = url.split('://')[1].split('@');
const auth = parts[0].split(':');
const hostAndDb = parts[1].split('/');
const hostAndPort = hostAndDb[0].split(':');
return {
user: auth[0],
password: auth[1],
host: hostAndPort[0],
port: parseInt(hostAndPort[1]),
database: hostAndDb[1]
};
}
}
}
console.log('未找到 .env 文件或 DATABASE_URL 配置，使用默认配置');
return {
host: '192.168.111.111',
port: 3306,
user: 'root',
password: 'fullstack',
database: 'iboard'
};
}
// 创建数据库连接池
const pool = createPool({
host: '192.168.111.111',
port: 3306,
user: 'root',
password: 'fullstack',
database: 'iboard',
connectionLimit: 5
});
const config = loadDatabaseConfig();
console.log('数据库配置:');
console.log(` 主机: ${config.host}`);
console.log(` 端口: ${config.port}`);
console.log(` 用户: ${config.user}`);
console.log(` 数据库: ${config.database}`);
// 构造 PrismaClient，提供 adapter
const prisma = new PrismaClient({
adapter: new PrismaMariaDb(pool)
});
let pool;
async function initPool() {
if (!pool) {
pool = createPool({
host: config.host,
port: config.port,
user: config.user,
password: config.password,
database: config.database,
connectionLimit: 5
});
try {
const conn = await pool.getConnection();
console.log('数据库连接成功!');
conn.release();
} catch (error) {
console.error('数据库连接失败:', error.message);
throw error;
}
}
return pool;
}
// 浙江数据开放网站URL（示例，实际需要根据真实网站修改）
const ZHEJIANG_OPEN_DATA_URL = 'https://data.zj.gov.cn/';
@@ -31,10 +90,9 @@ const MIN_UPDATE_INTERVAL = 30 * 60 * 1000;
// 从数据库读取缓存
async function readCache() {
try {
const cache = await prisma.scraperCache.findUnique({
where: { scraperName: SCRAPER_NAME }
});
return cache || { scraperName: SCRAPER_NAME, lastDataHash: null, lastUpdateAt: null };
await initPool();
const result = await pool.query('SELECT * FROM ScraperCache WHERE scraperName = ?', [SCRAPER_NAME]);
return result.length > 0 ? result[0] : { scraperName: SCRAPER_NAME, lastDataHash: null, lastUpdateAt: null };
} catch (error) {
console.error('读取数据库缓存失败:', error.message);
return { scraperName: SCRAPER_NAME, lastDataHash: null, lastUpdateAt: null };
@@ -44,18 +102,11 @@ async function readCache() {
// 写入数据库缓存
async function writeCache(lastDataHash) {
try {
await prisma.scraperCache.upsert({
where: { scraperName: SCRAPER_NAME },
update: {
lastDataHash,
lastUpdateAt: new Date()
},
create: {
scraperName: SCRAPER_NAME,
lastDataHash,
lastUpdateAt: new Date()
}
});
await initPool();
await pool.query(
'INSERT INTO ScraperCache (scraperName, lastDataHash, lastUpdateAt) VALUES (?, ?, ?) ON DUPLICATE KEY UPDATE lastDataHash = VALUES(lastDataHash), lastUpdateAt = VALUES(lastUpdateAt)',
[SCRAPER_NAME, lastDataHash, new Date()]
);
} catch (error) {
console.error('写入数据库缓存失败:', error.message);
}
@@ -76,12 +127,9 @@ function computeDataHash(data) {
// 获取数据库中最新的月份
async function getLatestMonth() {
try {
const latest = await prisma.econ_SowInventory.findFirst({
orderBy: {
month: 'desc'
}
});
return latest ? latest.month : null;
await initPool();
const result = await pool.query('SELECT MAX(month) as latestMonth FROM Econ_SowInventory');
return result[0].latestMonth || null;
} catch (error) {
console.error('查询最新月份失败:', error.message);
return null;
@@ -108,6 +156,7 @@ function shouldSkipRequest(cache) {
// 能繁母猪数量数据爬取函数（增量更新版本）
async function scrapeSowInventory(incremental = true) {
try {
await initPool();
console.log('='.repeat(50));
console.log('开始爬取浙江省能繁母猪数量数据...');
console.log(`增量更新模式: ${incremental ? '开启' : '关闭'}`);
@@ -224,26 +273,25 @@ async function scrapeSowInventory(incremental = true) {
for (const item of dataToUpdate) {
try {
// 检查数据是否已存在
const existing = await prisma.econ_SowInventory.findUnique({
where: { month: item.month }
});
if (existing) {
const existing = await pool.query('SELECT * FROM Econ_SowInventory WHERE month = ?', [item.month]);
if (existing.length > 0) {
// 检查数据是否真的需要更新（值变化了才更新）
if (existing.inventory !== item.inventory) {
await prisma.econ_SowInventory.update({
where: { month: item.month },
data: { inventory: item.inventory }
});
if (existing[0].inventory !== item.inventory) {
await pool.query(
'UPDATE Econ_SowInventory SET inventory = ? WHERE month = ?',
[item.inventory, item.month]
);
updateCount++;
console.log(`更新数据: ${item.month} - ${item.inventory} (原值: ${existing.inventory})`);
console.log(`更新数据: ${item.month} - ${item.inventory} (原值: ${existing[0].inventory})`);
} else {
console.log(`数据未变化，跳过: ${item.month}`);
}
} else {
await prisma.econ_SowInventory.create({
data: item
});
await pool.query(
'INSERT INTO Econ_SowInventory (month, inventory) VALUES (?, ?)',
[item.month, item.inventory]
);
newCount++;
console.log(`新增数据: ${item.month} - ${item.inventory}`);
}
@@ -272,10 +320,11 @@ async function scrapeSowInventory(incremental = true) {
console.error('爬取数据时出错:', error.message);
throw error;
} finally {
// 关闭Prisma连接
await prisma.$disconnect();
// 关闭数据库连接池
await pool.end();
if (pool) {
await pool.end();
console.log('数据库连接池已关闭');
}
}
}
scripts/scraper/usd-cny-rate.scraper.js
+79 -14
View File
@@ -1,17 +1,68 @@
import { createPool } from 'mariadb';
import { fetch as fetchUSDToCNYRate } from '../fetchers/usd-cny-rate.fetcher.js';
import fs from 'fs';
import path from 'path';
let pool;
function loadDatabaseConfig() {
const envPath = path.join(process.cwd(), '.env');
if (fs.existsSync(envPath)) {
console.log('读取 .env 文件...');
const envContent = fs.readFileSync(envPath, 'utf-8');
const lines = envContent.split('\n');
for (const line of lines) {
const trimmedLine = line.trim();
if (trimmedLine.startsWith('DATABASE_URL=')) {
const url = trimmedLine.substring('DATABASE_URL='.length).replace(/^"|"$/g, '');
console.log('从 .env 文件获取数据库配置成功!');
console.log(`DATABASE_URL: ${url}`);
// 解析URL
const parts = url.split('://')[1].split('@');
const auth = parts[0].split(':');
const hostAndDb = parts[1].split('/');
const hostAndPort = hostAndDb[0].split(':');
return {
user: auth[0],
password: auth[1],
host: hostAndPort[0],
port: parseInt(hostAndPort[1]),
database: hostAndDb[1]
};
}
}
}
console.log('未找到 .env 文件或 DATABASE_URL 配置，使用默认配置');
return {
host: '192.168.111.111',
port: 3306,
user: 'root',
password: 'fullstack',
database: 'iboard'
};
}
async function initPool() {
if (!pool) {
console.log('初始化数据库连接池...');
const config = loadDatabaseConfig();
console.log('数据库配置:');
console.log(` 主机: ${config.host}`);
console.log(` 端口: ${config.port}`);
console.log(` 用户: ${config.user}`);
console.log(` 数据库: ${config.database}`);
pool = createPool({
host: '192.168.111.111',
port: 3306,
user: 'root',
password: 'fullstack',
database: 'iboard',
host: config.host,
port: config.port,
user: config.user,
password: config.password,
database: config.database,
connectionLimit: 5
});
@@ -212,12 +263,23 @@ export async function trigger(startDate = null, endDate = null) {
}
// 直接运行测试
console.log('='.repeat(60));
console.log('测试 usd-cny-rate.scraper.js');
console.log('='.repeat(60));
trigger()
.then(result => {
async function runTest() {
console.log('='.repeat(60));
console.log('测试 usd-cny-rate.scraper.js');
console.log('='.repeat(60));
// 测试配置读取
console.log('\n测试数据库配置读取...');
const config = loadDatabaseConfig();
console.log('配置读取结果:');
console.log(` 主机: ${config.host}`);
console.log(` 端口: ${config.port}`);
console.log(` 用户: ${config.user}`);
console.log(` 数据库: ${config.database}`);
console.log('配置读取测试完成!');
try {
const result = await trigger();
console.log('\n' + '='.repeat(60));
console.log('测试结果:');
console.log(` 开始日期: ${result.startDate}`);
@@ -227,8 +289,11 @@ trigger()
console.log(` 最终数据: ${result.finalCount} 条`);
console.log(` 操作成功: ${result.success}`);
console.log('='.repeat(60));
})
.catch(error => {
} catch (error) {
console.error('测试失败:', error);
process.exit(1);
});
}
}
// 运行测试
runTest();