Fullstack/iboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

save
Docker Build and Push / build-image (push) Successful in 3m16s
Details

Browse Source
This commit is contained in:
Cheney
2026-08-03 18:09:08 +08:00
parent 0f74602036
commit 79341bb8cc
25 changed files with 5593 additions and 1409 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package.json
+30 -12
View File
@@ -3,31 +3,49 @@
"version": "1.0.0",
"type": "module",
"scripts": {
"dev": "next dev",
"build": "next build",
"start": "next start",
"tauri": "tauri",
"dev:web": "vite",
"dev:server": "node --experimental-strip-types --no-warnings server/index.js",
"dev": "concurrently -n web,api -c blue,green \"npm run dev:web\" \"npm run dev:server\"",
"build": "vite build",
"preview": "vite preview",
"start": "node --experimental-strip-types --no-warnings server/index.js",
"db:migrate": "prisma migrate deploy",
"db:seed": "node scripts/seed.cjs",
"icons": "node scripts/generate-icons.mjs",
"tauri:dev": "tauri dev",
"tauri:build": "tauri build",
"tauri:android:init": "tauri android init",
"tauri:android:dev": "tauri android dev",
"tauri:android:build": "tauri android build",
"tauri:android:apk": "tauri android build --apk",
"test": "echo \"Error: no test specified\" && exit 1"
},
"dependencies": {
"@prisma/adapter-mariadb": "^7.7.0",
"@prisma/adapter-mariadb": "^7.9.1",
"@prisma/client": "^7.7.0",
"cors": "^2.8.5",
"dotenv": "^16.4.5",
"express": "^4.21.0",
"gray-matter": "^4.0.3",
"mariadb": "^3.5.2",
"jsonwebtoken": "^9.0.2",
"marked": "^17.0.6",
"next": "^14.2.3",
"node-cron": "^4.2.1",
"mariadb": "^3.5.3",
"prisma": "^7.7.0",
"react": "^18.2.0",
"react-dom": "^18.2.0",
"react-router-dom": "^6.26.0",
"recharts": "^3.8.1"
},
"keywords": [],
"author": "",
"license": "ISC",
"description": "",
"devDependencies": {
"@tauri-apps/api": "^2.0.0",
"@tauri-apps/cli": "^2.0.0",
"@vitejs/plugin-react": "^4.3.1",
"axios": "^1.15.0",
"cheerio": "^1.2.0",
"csv-parser": "^3.2.0"
"concurrently": "^9.0.0",
"csv-parser": "^3.2.0",
"sharp": "^0.33.0",
"vite": "^5.4.0"
}
}
readme
-13
View File
@@ -4,16 +4,3 @@
- 数据主要是放各种经济数据和图表。
- 策略是放按照经济理论组织的相关数据和图表的对比。
## 爬虫
- 基于 bun 的定时任务
- 统一获取接口
- 统一爬取接口
```
async function fetch(startDate = null, endDate = null)
async function trigger(startDate = null, endDate = null)
```
不传递startDate和endDate，默认，默认获取最近30天的数据。
fecher 负责从目标网站获取原始数据
scraper 负责检查数据库中的数据是否完整，调用 fecher 将数据存储到数据库。
scripts/scraper/sow-inventory-scraper.js
+33 -290
View File
@@ -1,193 +1,20 @@
import axios from 'axios';
import * as cheerio from 'cheerio';
import { createPool } from 'mariadb';
import fs from 'fs';
import path from 'path';
const axios = require('axios');
const cheerio = require('cheerio');
const fs = require('fs');
const path = require('path');
const { PrismaClient } = require('../../generated/prisma');
function loadDatabaseConfig() {
const envPath = path.join(process.cwd(), '.env');
if (fs.existsSync(envPath)) {
console.log('读取 .env 文件...');
const envContent = fs.readFileSync(envPath, 'utf-8');
const lines = envContent.split('\n');
for (const line of lines) {
const trimmedLine = line.trim();
if (trimmedLine.startsWith('DATABASE_URL=')) {
const url = trimmedLine.substring('DATABASE_URL='.length).replace(/^"|"$/g, '');
console.log('从 .env 文件获取数据库配置成功!');
console.log(`DATABASE_URL: ${url}`);
// 解析URL
const parts = url.split('://')[1].split('@');
const auth = parts[0].split(':');
const hostAndDb = parts[1].split('/');
const hostAndPort = hostAndDb[0].split(':');
return {
user: auth[0],
password: auth[1],
host: hostAndPort[0],
port: parseInt(hostAndPort[1]),
database: hostAndDb[1]
};
}
}
}
console.log('未找到 .env 文件或 DATABASE_URL 配置，使用默认配置');
return {
host: '192.168.111.111',
port: 3306,
user: 'root',
password: 'fullstack',
database: 'iboard'
};
}
// 创建数据库连接池
const config = loadDatabaseConfig();
console.log('数据库配置:');
console.log(` 主机: ${config.host}`);
console.log(` 端口: ${config.port}`);
console.log(` 用户: ${config.user}`);
console.log(` 数据库: ${config.database}`);
let pool;
async function initPool() {
if (!pool) {
pool = createPool({
host: config.host,
port: config.port,
user: config.user,
password: config.password,
database: config.database,
connectionLimit: 5
});
try {
const conn = await pool.getConnection();
console.log('数据库连接成功!');
conn.release();
} catch (error) {
console.error('数据库连接失败:', error.message);
throw error;
}
}
return pool;
}
const prisma = new PrismaClient();
// 浙江数据开放网站URL（示例，实际需要根据真实网站修改）
const ZHEJIANG_OPEN_DATA_URL = 'https://data.zj.gov.cn/';
// 爬虫名称，用于缓存表中的标识
const SCRAPER_NAME = 'sow-inventory';
// 最小更新间隔（毫秒），默认30分钟
const MIN_UPDATE_INTERVAL = 30 * 60 * 1000;
// 从数据库读取缓存
async function readCache() {
// 能繁母猪数量数据爬取函数
async function scrapeSowInventory() {
try {
await initPool();
const result = await pool.query('SELECT * FROM ScraperCache WHERE scraperName = ?', [SCRAPER_NAME]);
return result.length > 0 ? result[0] : { scraperName: SCRAPER_NAME, lastDataHash: null, lastUpdateAt: null };
} catch (error) {
console.error('读取数据库缓存失败:', error.message);
return { scraperName: SCRAPER_NAME, lastDataHash: null, lastUpdateAt: null };
}
}
// 写入数据库缓存
async function writeCache(lastDataHash) {
try {
await initPool();
await pool.query(
'INSERT INTO ScraperCache (scraperName, lastDataHash, lastUpdateAt) VALUES (?, ?, ?) ON DUPLICATE KEY UPDATE lastDataHash = VALUES(lastDataHash), lastUpdateAt = VALUES(lastUpdateAt)',
[SCRAPER_NAME, lastDataHash, new Date()]
);
} catch (error) {
console.error('写入数据库缓存失败:', error.message);
}
}
// 计算数据hash，用于检测数据是否变化
function computeDataHash(data) {
const str = JSON.stringify(data);
let hash = 0;
for (let i = 0; i < str.length; i++) {
const char = str.charCodeAt(i);
hash = ((hash << 5) - hash) + char;
hash = hash & hash;
}
return hash.toString();
}
// 获取数据库中最新的月份
async function getLatestMonth() {
try {
await initPool();
const result = await pool.query('SELECT MAX(month) as latestMonth FROM Econ_SowInventory');
return result[0].latestMonth || null;
} catch (error) {
console.error('查询最新月份失败:', error.message);
return null;
}
}
// 判断月份是否需要更新（只有比数据库中更新的月份才需要处理）
function shouldUpdateMonth(month, latestDbMonth) {
if (!latestDbMonth) return true;
return month > latestDbMonth;
}
// 检查是否需要发起HTTP请求（基于更新间隔）
function shouldSkipRequest(cache) {
if (!cache.lastUpdateAt) return false;
const now = new Date();
const lastUpdate = new Date(cache.lastUpdateAt);
const interval = now - lastUpdate;
return interval < MIN_UPDATE_INTERVAL;
}
// 能繁母猪数量数据爬取函数（增量更新版本）
async function scrapeSowInventory(incremental = true) {
try {
await initPool();
console.log('='.repeat(50));
console.log('开始爬取浙江省能繁母猪数量数据...');
console.log(`增量更新模式: ${incremental ? '开启' : '关闭'}`);
console.log('='.repeat(50));
// 0. 检查是否需要发起请求
if (incremental) {
const cache = await readCache();
if (shouldSkipRequest(cache)) {
const lastUpdate = cache.lastUpdateAt ? new Date(cache.lastUpdateAt) : null;
const hoursDiff = lastUpdate ? ((new Date() - lastUpdate) / (1000 * 60 * 60)).toFixed(2) : '未知';
console.log('\n' + '='.repeat(50));
console.log('跳过请求：距离上次更新不足30分钟');
console.log(`上次更新时间: ${cache.lastUpdateAt}`);
console.log(`距离现在: ${hoursDiff} 小时`);
console.log('='.repeat(50));
return;
}
// 获取数据库中最新的月份
const latestDbMonth = await getLatestMonth();
if (latestDbMonth) {
console.log(`数据库中最新月份: ${latestDbMonth}`);
} else {
console.log('数据库为空，将获取所有数据');
}
}
// 1. 发送HTTP请求获取页面内容
console.log('\n正在请求数据源...');
const response = await axios.get(ZHEJIANG_OPEN_DATA_URL, {
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
@@ -198,7 +25,8 @@ async function scrapeSowInventory(incremental = true) {
const $ = cheerio.load(response.data);
// 3. 提取数据（这里需要根据实际网站结构修改选择器）
const allData = [];
// 示例：假设数据在表格中
const data = [];
$('table').each((tableIndex, table) => {
$(table).find('tr').each((rowIndex, row) => {
@@ -209,7 +37,7 @@ async function scrapeSowInventory(incremental = true) {
const inventory = parseInt($(cells[1]).text().trim());
if (month && !isNaN(inventory)) {
allData.push({
data.push({
month,
inventory
});
@@ -219,134 +47,49 @@ async function scrapeSowInventory(incremental = true) {
});
});
console.log(`页面总数据量: ${allData.length} 条`);
console.log(`爬取到 ${data.length} 条数据`);
// 4. 读取数据库缓存
const cache = await readCache();
// 5. 计算当前数据hash，检测数据是否变化
const currentDataHash = computeDataHash(allData);
// 6. 如果是增量更新且数据未变化，跳过更新
if (incremental && cache.lastDataHash === currentDataHash) {
console.log('\n' + '='.repeat(50));
console.log('数据未发生变化，跳过数据库更新！');
console.log(`上次更新时间: ${cache.lastUpdateAt || '未知'}`);
console.log('='.repeat(50));
return;
}
// 7. 如果是增量更新，获取数据库中最新的月份用于过滤
let latestDbMonth = null;
if (incremental) {
latestDbMonth = await getLatestMonth();
}
// 8. 过滤需要更新的数据
let dataToUpdate = allData;
let skippedCount = 0;
if (incremental && latestDbMonth) {
dataToUpdate = allData.filter(item => shouldUpdateMonth(item.month, latestDbMonth));
skippedCount = allData.length - dataToUpdate.length;
console.log(`\n增量更新: 跳过 ${skippedCount} 条已存在的数据`);
console.log(`需要处理: ${dataToUpdate.length} 条新数据`);
}
// 9. 如果没有需要更新的数据
if (dataToUpdate.length === 0) {
console.log('\n没有需要更新的数据！');
return;
}
// 10. 按月份排序（从旧到新）
dataToUpdate.sort((a, b) => a.month.localeCompare(b.month));
console.log('\n开始处理数据...');
console.log('-'.repeat(50));
let newCount = 0;
let updateCount = 0;
let errorCount = 0;
// 11. 存储数据到数据库
for (const item of dataToUpdate) {
// 4. 存储数据到数据库
for (const item of data) {
try {
// 检查数据是否已存在
const existing = await pool.query('SELECT * FROM Econ_SowInventory WHERE month = ?', [item.month]);
const existing = await prisma.econ_SowInventory.findUnique({
where: { month: item.month }
});
if (existing.length > 0) {
// 检查数据是否真的需要更新（值变化了才更新）
if (existing[0].inventory !== item.inventory) {
await pool.query(
'UPDATE Econ_SowInventory SET inventory = ? WHERE month = ?',
[item.inventory, item.month]
);
updateCount++;
console.log(`更新数据: ${item.month} - ${item.inventory} (原值: ${existing[0].inventory})`);
if (existing) {
// 更新现有数据
await prisma.econ_SowInventory.update({
where: { month: item.month },
data: { inventory: item.inventory }
});
console.log(`更新数据: ${item.month} - ${item.inventory}`);
} else {
console.log(`数据未变化，跳过: ${item.month}`);
}
} else {
await pool.query(
'INSERT INTO Econ_SowInventory (month, inventory) VALUES (?, ?)',
[item.month, item.inventory]
);
newCount++;
// 创建新数据
await prisma.econ_SowInventory.create({
data: item
});
console.log(`新增数据: ${item.month} - ${item.inventory}`);
}
} catch (error) {
errorCount++;
console.error(`处理数据 ${item.month} 时出错:`, error.message);
}
}
// 12. 更新数据库缓存
await writeCache(currentDataHash);
// 13. 输出统计信息
const now = new Date().toISOString();
console.log('\n' + '='.repeat(50));
console.log('数据爬取和存储完成！');
console.log('统计信息:');
console.log(` - 新增数据: ${newCount} 条`);
console.log(` - 更新数据: ${updateCount} 条`);
console.log(` - 跳过数据: ${skippedCount} 条`);
console.log(` - 错误数量: ${errorCount} 条`);
console.log(` - 本次更新时间: ${now}`);
console.log('='.repeat(50));
} catch (error) {
console.error('爬取数据时出错:', error.message);
throw error;
} finally {
// 关闭数据库连接池
if (pool) {
await pool.end();
console.log('数据库连接池已关闭');
// 关闭Prisma连接
await prisma.$disconnect();
}
}
}
// 强制全量更新函数
async function scrapeSowInventoryFull() {
console.log('警告: 即将执行全量更新，这将更新所有数据！');
console.log('如果只是想获取新数据，请使用 scrapeSowInventory() 函数。\n');
await scrapeSowInventory(false);
}
// 导出函数
export {
scrapeSowInventory,
scrapeSowInventoryFull
};
module.exports = { scrapeSowInventory };
// 如果直接运行此文件
if (import.meta.url === `file://${process.argv[1]}`) {
// 默认执行增量更新
// 如果需要强制全量更新，使用: scrapeSowInventoryFull()
scrapeSowInventory().catch(error => {
console.error('执行失败:', error);
process.exit(1);
});
if (require.main === module) {
scrapeSowInventory();
}
scripts/wsl/build-apk.sh
+66 -24
View File
@@ -1,7 +1,6 @@
#!/usr/bin/env bash
# iBoard Android APK 打包
# 在 WSL 内执行: bash scripts/wsl/build-apk.sh [debug|release]
# 默认 debug, release 需要先配签名 (见末尾)
# 用法: bash scripts/wsl/build-apk.sh [debug|release]
set -euo pipefail
WIN_PROJECT='/mnt/d/workbench/iboard'
@@ -10,34 +9,77 @@ cd "$WIN_PROJECT"
VARIANT="${1:-debug}"
echo "构建变体: $VARIANT"
# 环境变量 (setup.sh 已写过, 防御性再写一次)
# 环境变量
export ANDROID_HOME="${ANDROID_HOME:-$HOME/android-sdk}"
export ANDROID_NDK_HOME="${ANDROID_NDK_HOME:-$ANDROID_HOME/ndk/26.1.10909125}"
export ANDROID_NDK_HOME="${ANDROID_NDK_HOME:-$ANDROID_HOME/ndk/27.2.12479018}"
export PATH="$ANDROID_HOME/cmdline-tools/latest/bin:$ANDROID_HOME/platform-tools:$HOME/.cargo/bin:$PATH"
# 1. 先确保前端 dist/ 最新
# 0. 首次需要 android 工程
cd src-tauri
if [ ! -d "gen/android" ]; then
echo "=== 0. 初始化 Android 工程 (一次性) ==="
npx tauri android init
fi
cd "$WIN_PROJECT"
# 1. 先构建前端, 让 Tauri android 工程能拿到 dist/
echo "=== 1. 构建前端 ==="
npm run build
# 2. 进 src-tauri 跑 cargo tauri android build
echo "=== 2. 构建 APK ==="
# 2. 走不同路径
cd src-tauri
if [ "$VARIANT" = "release" ]; then
# release 需要签名, 首次请先生成 keystore:
# keytool -genkey -v -keystore ~/iboard-release.keystore -alias iboard \
# -keyalg RSA -keysize 2048 -validity 10000
# 然后在 ~/.gradle/gradle.properties 配:
# IBOARD_UPLOAD_STORE_FILE=/path/to/iboard-release.keystore
# IBOARD_UPLOAD_KEY_ALIAS=iboard
# IBOARD_UPLOAD_STORE_PASSWORD=...
# IBOARD_UPLOAD_KEY_PASSWORD=...
npx tauri android build -- --release
else
npx tauri android build -- --debug
fi
# 全部 4 个 Android 架构的 rustBuild 任务 (Tauri 2 默认会编 4 个)
RUST_BUILD_TASKS=(
":app:rustBuildArm64Release"
":app:rustBuildArmv7Release"
":app:rustBuildX86_64Release"
":app:rustBuildI686Release"
)
RUST_BUILD_TASKS_DBG=(
":app:rustBuildArm64Debug"
":app:rustBuildArmv7Debug"
":app:rustBuildX86_64Debug"
":app:rustBuildI686Debug"
)
SKIP_ARGS=()
for t in "${RUST_BUILD_TASKS[@]}"; do SKIP_ARGS+=(-x "${t#:}"); done
echo
echo "=== 产物位置 ==="
APK_DIR="$WIN_PROJECT/src-tauri/target/$( [ "$VARIANT" = "release" ] && echo release || echo debug )/bundle/android/app/build/outputs/apk/$VARIANT"
ls -lh "$APK_DIR"/*.apk 2>/dev/null || echo "(未找到 APK, 请检查上面日志)"
if [ "$VARIANT" = "release" ]; then
echo "[release] 手工 cargo 编 lib, gradle 只跑 packaging (跳过 rustBuild* 避免 --release 重复 bug)"
echo " [cargo] build lib release ..."
cargo build \
--target aarch64-linux-android \
--features custom-protocol \
--lib \
--release
echo " [gradle] assembleRelease -x rustBuild* ..."
cd gen/android
./gradlew --no-daemon assembleRelease "${SKIP_ARGS[@]}"
APK_DIR="$WIN_PROJECT/src-tauri/gen/android/app/build/outputs/apk/release"
echo
echo "=== 产物 (未签名) ==="
ls -lh "$APK_DIR"/*.apk 2>/dev/null || echo "(未找到 APK)"
echo
echo "=== Release 签名 (按需) ==="
echo "1. 生成 keystore (一次性):"
echo " keytool -genkey -v -keystore ~/iboard-release.keystore -alias iboard -keyalg RSA -keysize 2048 -validity 10000"
echo "2. 在 ~/.gradle/gradle.properties 写:"
echo " IBOARD_UPLOAD_STORE_FILE=/home/$(whoami)/iboard-release.keystore"
echo " IBOARD_UPLOAD_KEY_ALIAS=iboard"
echo " IBOARD_UPLOAD_STORE_PASSWORD=..."
echo " IBOARD_UPLOAD_KEY_PASSWORD=..."
echo "3. 重跑本脚本"
else
echo "[debug] Tauri CLI debug 路径 (不传 --release 不会触发 bug), 直接走"
npx tauri android build --apk
APK_DIR="$WIN_PROJECT/src-tauri/target/debug/bundle/android/app/build/outputs/apk/debug"
echo
echo "=== 产物 ==="
ls -lh "$APK_DIR"/*.apk 2>/dev/null || echo "(未找到 APK)"
fi
src-tauri/Cargo.lock
Generated
+4394
View File
File diff suppressed because it is too large Load Diff