79341bb8cc
All checks were successful
Docker Build and Push / build-image (push) Successful in 3m16s
96 lines
2.9 KiB
JavaScript
96 lines
2.9 KiB
JavaScript
const axios = require('axios');
|
||
const cheerio = require('cheerio');
|
||
const fs = require('fs');
|
||
const path = require('path');
|
||
const { PrismaClient } = require('../../generated/prisma');
|
||
|
||
const prisma = new PrismaClient();
|
||
|
||
// 浙江数据开放网站URL（示例，实际需要根据真实网站修改）
|
||
const ZHEJIANG_OPEN_DATA_URL = 'https://data.zj.gov.cn/';
|
||
|
||
// 能繁母猪数量数据爬取函数
|
||
async function scrapeSowInventory() {
|
||
try {
|
||
console.log('开始爬取浙江省能繁母猪数量数据...');
|
||
|
||
// 1. 发送HTTP请求获取页面内容
|
||
const response = await axios.get(ZHEJIANG_OPEN_DATA_URL, {
|
||
headers: {
|
||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
|
||
}
|
||
});
|
||
|
||
// 2. 解析HTML内容
|
||
const $ = cheerio.load(response.data);
|
||
|
||
// 3. 提取数据（这里需要根据实际网站结构修改选择器）
|
||
// 示例：假设数据在表格中
|
||
const data = [];
|
||
|
||
$('table').each((tableIndex, table) => {
|
||
$(table).find('tr').each((rowIndex, row) => {
|
||
if (rowIndex > 0) { // 跳过表头
|
||
const cells = $(row).find('td');
|
||
if (cells.length >= 2) {
|
||
const month = $(cells[0]).text().trim();
|
||
const inventory = parseInt($(cells[1]).text().trim());
|
||
|
||
if (month && !isNaN(inventory)) {
|
||
data.push({
|
||
month,
|
||
inventory
|
||
});
|
||
}
|
||
}
|
||
}
|
||
});
|
||
});
|
||
|
||
console.log(`爬取到 ${data.length} 条数据`);
|
||
|
||
// 4. 存储数据到数据库
|
||
for (const item of data) {
|
||
try {
|
||
// 检查数据是否已存在
|
||
const existing = await prisma.econ_SowInventory.findUnique({
|
||
where: { month: item.month }
|
||
});
|
||
|
||
if (existing) {
|
||
// 更新现有数据
|
||
await prisma.econ_SowInventory.update({
|
||
where: { month: item.month },
|
||
data: { inventory: item.inventory }
|
||
});
|
||
console.log(`更新数据: ${item.month} - ${item.inventory}`);
|
||
} else {
|
||
// 创建新数据
|
||
await prisma.econ_SowInventory.create({
|
||
data: item
|
||
});
|
||
console.log(`新增数据: ${item.month} - ${item.inventory}`);
|
||
}
|
||
} catch (error) {
|
||
console.error(`处理数据 ${item.month} 时出错:`, error.message);
|
||
}
|
||
}
|
||
|
||
console.log('数据爬取和存储完成！');
|
||
|
||
} catch (error) {
|
||
console.error('爬取数据时出错:', error.message);
|
||
} finally {
|
||
// 关闭Prisma连接
|
||
await prisma.$disconnect();
|
||
}
|
||
}
|
||
|
||
// 导出函数
|
||
module.exports = { scrapeSowInventory };
|
||
|
||
// 如果直接运行此文件
|
||
if (require.main === module) {
|
||
scrapeSowInventory();
|
||
}
|