Compare commits

...

7 Commits

Author SHA1 Message Date
f2824d88f7 获取沪深 300 数据
All checks were successful
Docker Build and Push / build-image (push) Successful in 4m44s
2026-04-25 20:08:03 +08:00
306af38fdc 增加数据-人民币汇率 2026-04-25 19:48:19 +08:00
d232591a38 添加汇率数据 2026-04-25 19:22:57 +08:00
b61cf9628e 添加汇率 scraper 2026-04-25 12:16:41 +08:00
75a2ec7d30 添加人民币美元汇率数据获取脚本 (fetcher) 2026-04-19 09:07:44 +08:00
f9e1471487 save 2026-04-19 08:42:20 +08:00
2820adfdd3 第一个爬虫,但是感觉是不能用的。 2026-04-17 20:37:44 +08:00
29 changed files with 4593 additions and 1190 deletions

2
.env
View File

@ -1 +1 @@
DATABASE_URL="mysql://root:fullstack@192.168.111.111:3306/iboard"
DATABASE_URL="mysql://root:fullstack@192.168.111.111:3306/iboard"

View File

@ -32,6 +32,10 @@ COPY --from=builder /app/public* ./public/
# 设置淘宝源并安装生产依赖
RUN npm config set registry https://registry.npmmirror.com && npm install --only=production
ENV DATABASE_URL="mysql://root:fullstack@baishe.vps.honor3.com:6033/iboard"
# 暴露 3000 端口
EXPOSE 3000

View File

@ -0,0 +1,145 @@
import { useState, useEffect } from 'react'
const TimeRangePicker = ({
value = '1m',
onChange,
className = ''
}) => {
const [timeRange, setTimeRange] = useState(value)
const [startDate, setStartDate] = useState('')
const [endDate, setEndDate] = useState('')
const [showDropdown, setShowDropdown] = useState(false)
// 计算日期范围
const calculateDateRange = (range) => {
const end = new Date()
const start = new Date()
if (range === '1m') {
start.setMonth(start.getMonth() - 1)
} else if (range === '3m') {
start.setMonth(start.getMonth() - 3)
} else if (range === '1y') {
start.setFullYear(start.getFullYear() - 1)
}
const formatDate = (date) => {
return date.toISOString().split('T')[0]
}
return {
start: formatDate(start),
end: formatDate(end)
}
}
// 初始化时设置日期范围
useEffect(() => {
const range = calculateDateRange(value)
setStartDate(range.start)
setEndDate(range.end)
}, [value])
// 处理时间范围变化
const handleTimeRangeChange = (range) => {
const dateRange = calculateDateRange(range)
setStartDate(dateRange.start)
setEndDate(dateRange.end)
setTimeRange(range)
if (onChange) {
onChange(range, dateRange.start, dateRange.end)
}
setShowDropdown(false)
}
// 处理自定义日期范围
const handleCustomRangeApply = () => {
if (startDate && endDate) {
setTimeRange('custom')
if (onChange) {
onChange('custom', startDate, endDate)
}
setShowDropdown(false)
}
}
// 处理开始日期变化
const handleStartDateChange = (e) => {
setStartDate(e.target.value)
}
// 处理结束日期变化
const handleEndDateChange = (e) => {
setEndDate(e.target.value)
}
return (
<div className={`time-range-picker ${className}`}>
<button
className="time-range-btn"
onClick={() => setShowDropdown(!showDropdown)}
>
{timeRange === '1m' && '近一个月'}
{timeRange === '3m' && '近三个月'}
{timeRange === '1y' && '近一年'}
{timeRange === 'custom' && startDate && endDate && `${startDate}${endDate}`}
{timeRange === 'custom' && (!startDate || !endDate) && '自定义'}
<span className="dropdown-arrow"></span>
</button>
{showDropdown && (
<div className="time-range-dropdown">
<div className="dropdown-section">
<h4 className="dropdown-title">自定义范围</h4>
<div className="custom-date-range">
<input
type="date"
value={startDate}
onChange={handleStartDateChange}
max={endDate || new Date().toISOString().split('T')[0]}
/>
<span></span>
<input
type="date"
value={endDate}
onChange={handleEndDateChange}
min={startDate}
max={new Date().toISOString().split('T')[0]}
/>
<button
className="apply-btn"
onClick={handleCustomRangeApply}
>
确定
</button>
</div>
</div>
<div className="dropdown-section">
<h4 className="dropdown-title">快捷选项</h4>
<ul className="dropdown-options">
<li
className={`dropdown-option ${timeRange === '1m' ? 'active' : ''}`}
onClick={() => handleTimeRangeChange('1m')}
>
近一个月
</li>
<li
className={`dropdown-option ${timeRange === '3m' ? 'active' : ''}`}
onClick={() => handleTimeRangeChange('3m')}
>
近三个月
</li>
<li
className={`dropdown-option ${timeRange === '1y' ? 'active' : ''}`}
onClick={() => handleTimeRangeChange('1y')}
>
近一年
</li>
</ul>
</div>
</div>
)}
</div>
)
}
export default TimeRangePicker

View File

@ -38,7 +38,12 @@ export type Strategy = Prisma.StrategyModel
*/
export type Indicator = Prisma.IndicatorModel
/**
* Model SowInventory
* Model Econ_SowInventory
*
*/
export type SowInventory = Prisma.SowInventoryModel
export type Econ_SowInventory = Prisma.Econ_SowInventoryModel
/**
* Model ScraperCache
*
*/
export type ScraperCache = Prisma.ScraperCacheModel

View File

@ -62,7 +62,12 @@ export type Strategy = Prisma.StrategyModel
*/
export type Indicator = Prisma.IndicatorModel
/**
* Model SowInventory
* Model Econ_SowInventory
*
*/
export type SowInventory = Prisma.SowInventoryModel
export type Econ_SowInventory = Prisma.Econ_SowInventoryModel
/**
* Model ScraperCache
*
*/
export type ScraperCache = Prisma.ScraperCacheModel

View File

@ -150,6 +150,95 @@ export type JsonWithAggregatesFilterBase<$PrismaModel = never> = {
_max?: Prisma.NestedJsonFilter<$PrismaModel>
}
export type StringNullableFilter<$PrismaModel = never> = {
equals?: string | Prisma.StringFieldRefInput<$PrismaModel> | null
in?: string[] | null
notIn?: string[] | null
lt?: string | Prisma.StringFieldRefInput<$PrismaModel>
lte?: string | Prisma.StringFieldRefInput<$PrismaModel>
gt?: string | Prisma.StringFieldRefInput<$PrismaModel>
gte?: string | Prisma.StringFieldRefInput<$PrismaModel>
contains?: string | Prisma.StringFieldRefInput<$PrismaModel>
startsWith?: string | Prisma.StringFieldRefInput<$PrismaModel>
endsWith?: string | Prisma.StringFieldRefInput<$PrismaModel>
search?: string
not?: Prisma.NestedStringNullableFilter<$PrismaModel> | string | null
}
export type JsonNullableFilter<$PrismaModel = never> =
| Prisma.PatchUndefined<
Prisma.Either<Required<JsonNullableFilterBase<$PrismaModel>>, Exclude<keyof Required<JsonNullableFilterBase<$PrismaModel>>, 'path'>>,
Required<JsonNullableFilterBase<$PrismaModel>>
>
| Prisma.OptionalFlat<Omit<Required<JsonNullableFilterBase<$PrismaModel>>, 'path'>>
export type JsonNullableFilterBase<$PrismaModel = never> = {
equals?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | Prisma.JsonNullValueFilter
path?: string
mode?: Prisma.QueryMode | Prisma.EnumQueryModeFieldRefInput<$PrismaModel>
string_contains?: string | Prisma.StringFieldRefInput<$PrismaModel>
string_starts_with?: string | Prisma.StringFieldRefInput<$PrismaModel>
string_ends_with?: string | Prisma.StringFieldRefInput<$PrismaModel>
array_starts_with?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | null
array_ends_with?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | null
array_contains?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | null
lt?: runtime.InputJsonValue
lte?: runtime.InputJsonValue
gt?: runtime.InputJsonValue
gte?: runtime.InputJsonValue
not?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | Prisma.JsonNullValueFilter
}
export type SortOrderInput = {
sort: Prisma.SortOrder
nulls?: Prisma.NullsOrder
}
export type StringNullableWithAggregatesFilter<$PrismaModel = never> = {
equals?: string | Prisma.StringFieldRefInput<$PrismaModel> | null
in?: string[] | null
notIn?: string[] | null
lt?: string | Prisma.StringFieldRefInput<$PrismaModel>
lte?: string | Prisma.StringFieldRefInput<$PrismaModel>
gt?: string | Prisma.StringFieldRefInput<$PrismaModel>
gte?: string | Prisma.StringFieldRefInput<$PrismaModel>
contains?: string | Prisma.StringFieldRefInput<$PrismaModel>
startsWith?: string | Prisma.StringFieldRefInput<$PrismaModel>
endsWith?: string | Prisma.StringFieldRefInput<$PrismaModel>
search?: string
not?: Prisma.NestedStringNullableWithAggregatesFilter<$PrismaModel> | string | null
_count?: Prisma.NestedIntNullableFilter<$PrismaModel>
_min?: Prisma.NestedStringNullableFilter<$PrismaModel>
_max?: Prisma.NestedStringNullableFilter<$PrismaModel>
}
export type JsonNullableWithAggregatesFilter<$PrismaModel = never> =
| Prisma.PatchUndefined<
Prisma.Either<Required<JsonNullableWithAggregatesFilterBase<$PrismaModel>>, Exclude<keyof Required<JsonNullableWithAggregatesFilterBase<$PrismaModel>>, 'path'>>,
Required<JsonNullableWithAggregatesFilterBase<$PrismaModel>>
>
| Prisma.OptionalFlat<Omit<Required<JsonNullableWithAggregatesFilterBase<$PrismaModel>>, 'path'>>
export type JsonNullableWithAggregatesFilterBase<$PrismaModel = never> = {
equals?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | Prisma.JsonNullValueFilter
path?: string
mode?: Prisma.QueryMode | Prisma.EnumQueryModeFieldRefInput<$PrismaModel>
string_contains?: string | Prisma.StringFieldRefInput<$PrismaModel>
string_starts_with?: string | Prisma.StringFieldRefInput<$PrismaModel>
string_ends_with?: string | Prisma.StringFieldRefInput<$PrismaModel>
array_starts_with?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | null
array_ends_with?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | null
array_contains?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | null
lt?: runtime.InputJsonValue
lte?: runtime.InputJsonValue
gt?: runtime.InputJsonValue
gte?: runtime.InputJsonValue
not?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | Prisma.JsonNullValueFilter
_count?: Prisma.NestedIntNullableFilter<$PrismaModel>
_min?: Prisma.NestedJsonNullableFilter<$PrismaModel>
_max?: Prisma.NestedJsonNullableFilter<$PrismaModel>
}
export type NestedIntFilter<$PrismaModel = never> = {
equals?: number | Prisma.IntFieldRefInput<$PrismaModel>
in?: number[]
@ -270,4 +359,72 @@ export type NestedJsonFilterBase<$PrismaModel = never> = {
not?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | Prisma.JsonNullValueFilter
}
export type NestedStringNullableFilter<$PrismaModel = never> = {
equals?: string | Prisma.StringFieldRefInput<$PrismaModel> | null
in?: string[] | null
notIn?: string[] | null
lt?: string | Prisma.StringFieldRefInput<$PrismaModel>
lte?: string | Prisma.StringFieldRefInput<$PrismaModel>
gt?: string | Prisma.StringFieldRefInput<$PrismaModel>
gte?: string | Prisma.StringFieldRefInput<$PrismaModel>
contains?: string | Prisma.StringFieldRefInput<$PrismaModel>
startsWith?: string | Prisma.StringFieldRefInput<$PrismaModel>
endsWith?: string | Prisma.StringFieldRefInput<$PrismaModel>
search?: string
not?: Prisma.NestedStringNullableFilter<$PrismaModel> | string | null
}
export type NestedStringNullableWithAggregatesFilter<$PrismaModel = never> = {
equals?: string | Prisma.StringFieldRefInput<$PrismaModel> | null
in?: string[] | null
notIn?: string[] | null
lt?: string | Prisma.StringFieldRefInput<$PrismaModel>
lte?: string | Prisma.StringFieldRefInput<$PrismaModel>
gt?: string | Prisma.StringFieldRefInput<$PrismaModel>
gte?: string | Prisma.StringFieldRefInput<$PrismaModel>
contains?: string | Prisma.StringFieldRefInput<$PrismaModel>
startsWith?: string | Prisma.StringFieldRefInput<$PrismaModel>
endsWith?: string | Prisma.StringFieldRefInput<$PrismaModel>
search?: string
not?: Prisma.NestedStringNullableWithAggregatesFilter<$PrismaModel> | string | null
_count?: Prisma.NestedIntNullableFilter<$PrismaModel>
_min?: Prisma.NestedStringNullableFilter<$PrismaModel>
_max?: Prisma.NestedStringNullableFilter<$PrismaModel>
}
export type NestedIntNullableFilter<$PrismaModel = never> = {
equals?: number | Prisma.IntFieldRefInput<$PrismaModel> | null
in?: number[] | null
notIn?: number[] | null
lt?: number | Prisma.IntFieldRefInput<$PrismaModel>
lte?: number | Prisma.IntFieldRefInput<$PrismaModel>
gt?: number | Prisma.IntFieldRefInput<$PrismaModel>
gte?: number | Prisma.IntFieldRefInput<$PrismaModel>
not?: Prisma.NestedIntNullableFilter<$PrismaModel> | number | null
}
export type NestedJsonNullableFilter<$PrismaModel = never> =
| Prisma.PatchUndefined<
Prisma.Either<Required<NestedJsonNullableFilterBase<$PrismaModel>>, Exclude<keyof Required<NestedJsonNullableFilterBase<$PrismaModel>>, 'path'>>,
Required<NestedJsonNullableFilterBase<$PrismaModel>>
>
| Prisma.OptionalFlat<Omit<Required<NestedJsonNullableFilterBase<$PrismaModel>>, 'path'>>
export type NestedJsonNullableFilterBase<$PrismaModel = never> = {
equals?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | Prisma.JsonNullValueFilter
path?: string
mode?: Prisma.QueryMode | Prisma.EnumQueryModeFieldRefInput<$PrismaModel>
string_contains?: string | Prisma.StringFieldRefInput<$PrismaModel>
string_starts_with?: string | Prisma.StringFieldRefInput<$PrismaModel>
string_ends_with?: string | Prisma.StringFieldRefInput<$PrismaModel>
array_starts_with?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | null
array_ends_with?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | null
array_contains?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | null
lt?: runtime.InputJsonValue
lte?: runtime.InputJsonValue
gt?: runtime.InputJsonValue
gte?: runtime.InputJsonValue
not?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | Prisma.JsonNullValueFilter
}

File diff suppressed because one or more lines are too long

View File

@ -388,7 +388,8 @@ export const ModelName = {
Article: 'Article',
Strategy: 'Strategy',
Indicator: 'Indicator',
SowInventory: 'SowInventory'
Econ_SowInventory: 'Econ_SowInventory',
ScraperCache: 'ScraperCache'
} as const
export type ModelName = (typeof ModelName)[keyof typeof ModelName]
@ -404,7 +405,7 @@ export type TypeMap<ExtArgs extends runtime.Types.Extensions.InternalArgs = runt
omit: GlobalOmitOptions
}
meta: {
modelProps: "user" | "article" | "strategy" | "indicator" | "sowInventory"
modelProps: "user" | "article" | "strategy" | "indicator" | "econ_SowInventory" | "scraperCache"
txIsolationLevel: TransactionIsolationLevel
}
model: {
@ -672,69 +673,135 @@ export type TypeMap<ExtArgs extends runtime.Types.Extensions.InternalArgs = runt
}
}
}
SowInventory: {
payload: Prisma.$SowInventoryPayload<ExtArgs>
fields: Prisma.SowInventoryFieldRefs
Econ_SowInventory: {
payload: Prisma.$Econ_SowInventoryPayload<ExtArgs>
fields: Prisma.Econ_SowInventoryFieldRefs
operations: {
findUnique: {
args: Prisma.SowInventoryFindUniqueArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$SowInventoryPayload> | null
args: Prisma.Econ_SowInventoryFindUniqueArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$Econ_SowInventoryPayload> | null
}
findUniqueOrThrow: {
args: Prisma.SowInventoryFindUniqueOrThrowArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$SowInventoryPayload>
args: Prisma.Econ_SowInventoryFindUniqueOrThrowArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$Econ_SowInventoryPayload>
}
findFirst: {
args: Prisma.SowInventoryFindFirstArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$SowInventoryPayload> | null
args: Prisma.Econ_SowInventoryFindFirstArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$Econ_SowInventoryPayload> | null
}
findFirstOrThrow: {
args: Prisma.SowInventoryFindFirstOrThrowArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$SowInventoryPayload>
args: Prisma.Econ_SowInventoryFindFirstOrThrowArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$Econ_SowInventoryPayload>
}
findMany: {
args: Prisma.SowInventoryFindManyArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$SowInventoryPayload>[]
args: Prisma.Econ_SowInventoryFindManyArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$Econ_SowInventoryPayload>[]
}
create: {
args: Prisma.SowInventoryCreateArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$SowInventoryPayload>
args: Prisma.Econ_SowInventoryCreateArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$Econ_SowInventoryPayload>
}
createMany: {
args: Prisma.SowInventoryCreateManyArgs<ExtArgs>
args: Prisma.Econ_SowInventoryCreateManyArgs<ExtArgs>
result: BatchPayload
}
delete: {
args: Prisma.SowInventoryDeleteArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$SowInventoryPayload>
args: Prisma.Econ_SowInventoryDeleteArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$Econ_SowInventoryPayload>
}
update: {
args: Prisma.SowInventoryUpdateArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$SowInventoryPayload>
args: Prisma.Econ_SowInventoryUpdateArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$Econ_SowInventoryPayload>
}
deleteMany: {
args: Prisma.SowInventoryDeleteManyArgs<ExtArgs>
args: Prisma.Econ_SowInventoryDeleteManyArgs<ExtArgs>
result: BatchPayload
}
updateMany: {
args: Prisma.SowInventoryUpdateManyArgs<ExtArgs>
args: Prisma.Econ_SowInventoryUpdateManyArgs<ExtArgs>
result: BatchPayload
}
upsert: {
args: Prisma.SowInventoryUpsertArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$SowInventoryPayload>
args: Prisma.Econ_SowInventoryUpsertArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$Econ_SowInventoryPayload>
}
aggregate: {
args: Prisma.SowInventoryAggregateArgs<ExtArgs>
result: runtime.Types.Utils.Optional<Prisma.AggregateSowInventory>
args: Prisma.Econ_SowInventoryAggregateArgs<ExtArgs>
result: runtime.Types.Utils.Optional<Prisma.AggregateEcon_SowInventory>
}
groupBy: {
args: Prisma.SowInventoryGroupByArgs<ExtArgs>
result: runtime.Types.Utils.Optional<Prisma.SowInventoryGroupByOutputType>[]
args: Prisma.Econ_SowInventoryGroupByArgs<ExtArgs>
result: runtime.Types.Utils.Optional<Prisma.Econ_SowInventoryGroupByOutputType>[]
}
count: {
args: Prisma.SowInventoryCountArgs<ExtArgs>
result: runtime.Types.Utils.Optional<Prisma.SowInventoryCountAggregateOutputType> | number
args: Prisma.Econ_SowInventoryCountArgs<ExtArgs>
result: runtime.Types.Utils.Optional<Prisma.Econ_SowInventoryCountAggregateOutputType> | number
}
}
}
ScraperCache: {
payload: Prisma.$ScraperCachePayload<ExtArgs>
fields: Prisma.ScraperCacheFieldRefs
operations: {
findUnique: {
args: Prisma.ScraperCacheFindUniqueArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$ScraperCachePayload> | null
}
findUniqueOrThrow: {
args: Prisma.ScraperCacheFindUniqueOrThrowArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$ScraperCachePayload>
}
findFirst: {
args: Prisma.ScraperCacheFindFirstArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$ScraperCachePayload> | null
}
findFirstOrThrow: {
args: Prisma.ScraperCacheFindFirstOrThrowArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$ScraperCachePayload>
}
findMany: {
args: Prisma.ScraperCacheFindManyArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$ScraperCachePayload>[]
}
create: {
args: Prisma.ScraperCacheCreateArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$ScraperCachePayload>
}
createMany: {
args: Prisma.ScraperCacheCreateManyArgs<ExtArgs>
result: BatchPayload
}
delete: {
args: Prisma.ScraperCacheDeleteArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$ScraperCachePayload>
}
update: {
args: Prisma.ScraperCacheUpdateArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$ScraperCachePayload>
}
deleteMany: {
args: Prisma.ScraperCacheDeleteManyArgs<ExtArgs>
result: BatchPayload
}
updateMany: {
args: Prisma.ScraperCacheUpdateManyArgs<ExtArgs>
result: BatchPayload
}
upsert: {
args: Prisma.ScraperCacheUpsertArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$ScraperCachePayload>
}
aggregate: {
args: Prisma.ScraperCacheAggregateArgs<ExtArgs>
result: runtime.Types.Utils.Optional<Prisma.AggregateScraperCache>
}
groupBy: {
args: Prisma.ScraperCacheGroupByArgs<ExtArgs>
result: runtime.Types.Utils.Optional<Prisma.ScraperCacheGroupByOutputType>[]
}
count: {
args: Prisma.ScraperCacheCountArgs<ExtArgs>
result: runtime.Types.Utils.Optional<Prisma.ScraperCacheCountAggregateOutputType> | number
}
}
}
@ -826,7 +893,7 @@ export const IndicatorScalarFieldEnum = {
export type IndicatorScalarFieldEnum = (typeof IndicatorScalarFieldEnum)[keyof typeof IndicatorScalarFieldEnum]
export const SowInventoryScalarFieldEnum = {
export const Econ_SowInventoryScalarFieldEnum = {
id: 'id',
month: 'month',
inventory: 'inventory',
@ -834,7 +901,20 @@ export const SowInventoryScalarFieldEnum = {
updatedAt: 'updatedAt'
} as const
export type SowInventoryScalarFieldEnum = (typeof SowInventoryScalarFieldEnum)[keyof typeof SowInventoryScalarFieldEnum]
export type Econ_SowInventoryScalarFieldEnum = (typeof Econ_SowInventoryScalarFieldEnum)[keyof typeof Econ_SowInventoryScalarFieldEnum]
export const ScraperCacheScalarFieldEnum = {
id: 'id',
scraperName: 'scraperName',
lastDataHash: 'lastDataHash',
lastUpdateAt: 'lastUpdateAt',
metadata: 'metadata',
createdAt: 'createdAt',
updatedAt: 'updatedAt'
} as const
export type ScraperCacheScalarFieldEnum = (typeof ScraperCacheScalarFieldEnum)[keyof typeof ScraperCacheScalarFieldEnum]
export const SortOrder = {
@ -852,6 +932,14 @@ export const JsonNullValueInput = {
export type JsonNullValueInput = (typeof JsonNullValueInput)[keyof typeof JsonNullValueInput]
export const NullableJsonNullValueInput = {
DbNull: DbNull,
JsonNull: JsonNull
} as const
export type NullableJsonNullValueInput = (typeof NullableJsonNullValueInput)[keyof typeof NullableJsonNullValueInput]
export const UserOrderByRelevanceFieldEnum = {
name: 'name',
email: 'email',
@ -904,11 +992,27 @@ export const IndicatorOrderByRelevanceFieldEnum = {
export type IndicatorOrderByRelevanceFieldEnum = (typeof IndicatorOrderByRelevanceFieldEnum)[keyof typeof IndicatorOrderByRelevanceFieldEnum]
export const SowInventoryOrderByRelevanceFieldEnum = {
export const Econ_SowInventoryOrderByRelevanceFieldEnum = {
month: 'month'
} as const
export type SowInventoryOrderByRelevanceFieldEnum = (typeof SowInventoryOrderByRelevanceFieldEnum)[keyof typeof SowInventoryOrderByRelevanceFieldEnum]
export type Econ_SowInventoryOrderByRelevanceFieldEnum = (typeof Econ_SowInventoryOrderByRelevanceFieldEnum)[keyof typeof Econ_SowInventoryOrderByRelevanceFieldEnum]
export const NullsOrder = {
first: 'first',
last: 'last'
} as const
export type NullsOrder = (typeof NullsOrder)[keyof typeof NullsOrder]
export const ScraperCacheOrderByRelevanceFieldEnum = {
scraperName: 'scraperName',
lastDataHash: 'lastDataHash'
} as const
export type ScraperCacheOrderByRelevanceFieldEnum = (typeof ScraperCacheOrderByRelevanceFieldEnum)[keyof typeof ScraperCacheOrderByRelevanceFieldEnum]
@ -1057,7 +1161,8 @@ export type GlobalOmitConfig = {
article?: Prisma.ArticleOmit
strategy?: Prisma.StrategyOmit
indicator?: Prisma.IndicatorOmit
sowInventory?: Prisma.SowInventoryOmit
econ_SowInventory?: Prisma.Econ_SowInventoryOmit
scraperCache?: Prisma.ScraperCacheOmit
}
/* Types for Logging */

View File

@ -55,7 +55,8 @@ export const ModelName = {
Article: 'Article',
Strategy: 'Strategy',
Indicator: 'Indicator',
SowInventory: 'SowInventory'
Econ_SowInventory: 'Econ_SowInventory',
ScraperCache: 'ScraperCache'
} as const
export type ModelName = (typeof ModelName)[keyof typeof ModelName]
@ -123,7 +124,7 @@ export const IndicatorScalarFieldEnum = {
export type IndicatorScalarFieldEnum = (typeof IndicatorScalarFieldEnum)[keyof typeof IndicatorScalarFieldEnum]
export const SowInventoryScalarFieldEnum = {
export const Econ_SowInventoryScalarFieldEnum = {
id: 'id',
month: 'month',
inventory: 'inventory',
@ -131,7 +132,20 @@ export const SowInventoryScalarFieldEnum = {
updatedAt: 'updatedAt'
} as const
export type SowInventoryScalarFieldEnum = (typeof SowInventoryScalarFieldEnum)[keyof typeof SowInventoryScalarFieldEnum]
export type Econ_SowInventoryScalarFieldEnum = (typeof Econ_SowInventoryScalarFieldEnum)[keyof typeof Econ_SowInventoryScalarFieldEnum]
export const ScraperCacheScalarFieldEnum = {
id: 'id',
scraperName: 'scraperName',
lastDataHash: 'lastDataHash',
lastUpdateAt: 'lastUpdateAt',
metadata: 'metadata',
createdAt: 'createdAt',
updatedAt: 'updatedAt'
} as const
export type ScraperCacheScalarFieldEnum = (typeof ScraperCacheScalarFieldEnum)[keyof typeof ScraperCacheScalarFieldEnum]
export const SortOrder = {
@ -149,6 +163,14 @@ export const JsonNullValueInput = {
export type JsonNullValueInput = (typeof JsonNullValueInput)[keyof typeof JsonNullValueInput]
export const NullableJsonNullValueInput = {
DbNull: DbNull,
JsonNull: JsonNull
} as const
export type NullableJsonNullValueInput = (typeof NullableJsonNullValueInput)[keyof typeof NullableJsonNullValueInput]
export const UserOrderByRelevanceFieldEnum = {
name: 'name',
email: 'email',
@ -201,9 +223,25 @@ export const IndicatorOrderByRelevanceFieldEnum = {
export type IndicatorOrderByRelevanceFieldEnum = (typeof IndicatorOrderByRelevanceFieldEnum)[keyof typeof IndicatorOrderByRelevanceFieldEnum]
export const SowInventoryOrderByRelevanceFieldEnum = {
export const Econ_SowInventoryOrderByRelevanceFieldEnum = {
month: 'month'
} as const
export type SowInventoryOrderByRelevanceFieldEnum = (typeof SowInventoryOrderByRelevanceFieldEnum)[keyof typeof SowInventoryOrderByRelevanceFieldEnum]
export type Econ_SowInventoryOrderByRelevanceFieldEnum = (typeof Econ_SowInventoryOrderByRelevanceFieldEnum)[keyof typeof Econ_SowInventoryOrderByRelevanceFieldEnum]
export const NullsOrder = {
first: 'first',
last: 'last'
} as const
export type NullsOrder = (typeof NullsOrder)[keyof typeof NullsOrder]
export const ScraperCacheOrderByRelevanceFieldEnum = {
scraperName: 'scraperName',
lastDataHash: 'lastDataHash'
} as const
export type ScraperCacheOrderByRelevanceFieldEnum = (typeof ScraperCacheOrderByRelevanceFieldEnum)[keyof typeof ScraperCacheOrderByRelevanceFieldEnum]

View File

@ -12,5 +12,6 @@ export type * from './models/User.ts'
export type * from './models/Article.ts'
export type * from './models/Strategy.ts'
export type * from './models/Indicator.ts'
export type * from './models/SowInventory.ts'
export type * from './models/Econ_SowInventory.ts'
export type * from './models/ScraperCache.ts'
export type * from './commonInputTypes.ts'

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

117
package-lock.json generated
View File

@ -9,10 +9,13 @@
"version": "1.0.0",
"license": "ISC",
"dependencies": {
"@prisma/adapter-mariadb": "^7.7.0",
"@prisma/client": "^7.7.0",
"gray-matter": "^4.0.3",
"mariadb": "^3.5.2",
"marked": "^17.0.6",
"next": "^14.2.3",
"node-cron": "^4.2.1",
"prisma": "^7.7.0",
"react": "^18.2.0",
"react-dom": "^18.2.0",
@ -219,6 +222,59 @@
"node": ">= 10"
}
},
"node_modules/@prisma/adapter-mariadb": {
"version": "7.7.0",
"resolved": "https://registry.npmmirror.com/@prisma/adapter-mariadb/-/adapter-mariadb-7.7.0.tgz",
"integrity": "sha512-BlugprCUNFGelP7t0uQEHkC5ZcNmgUxWi6xpkjWKUS9gESGBDF3mNdYFqRc2zS9w7pC83gpNxSY51A7gCY2HdA==",
"license": "Apache-2.0",
"dependencies": {
"@prisma/driver-adapter-utils": "7.7.0",
"mariadb": "3.4.5"
}
},
"node_modules/@prisma/adapter-mariadb/node_modules/@types/node": {
"version": "24.12.2",
"resolved": "https://registry.npmmirror.com/@types/node/-/node-24.12.2.tgz",
"integrity": "sha512-A1sre26ke7HDIuY/M23nd9gfB+nrmhtYyMINbjI1zHJxYteKR6qSMX56FsmjMcDb3SMcjJg5BiRRgOCC/yBD0g==",
"license": "MIT",
"dependencies": {
"undici-types": "~7.16.0"
}
},
"node_modules/@prisma/adapter-mariadb/node_modules/iconv-lite": {
"version": "0.6.3",
"resolved": "https://registry.npmmirror.com/iconv-lite/-/iconv-lite-0.6.3.tgz",
"integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
"license": "MIT",
"dependencies": {
"safer-buffer": ">= 2.1.2 < 3.0.0"
},
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/@prisma/adapter-mariadb/node_modules/mariadb": {
"version": "3.4.5",
"resolved": "https://registry.npmmirror.com/mariadb/-/mariadb-3.4.5.tgz",
"integrity": "sha512-gThTYkhIS5rRqkVr+Y0cIdzr+GRqJ9sA2Q34e0yzmyhMCwyApf3OKAC1jnF23aSlIOqJuyaUFUcj7O1qZslmmQ==",
"license": "LGPL-2.1-or-later",
"dependencies": {
"@types/geojson": "^7946.0.16",
"@types/node": "^24.0.13",
"denque": "^2.1.0",
"iconv-lite": "^0.6.3",
"lru-cache": "^10.4.3"
},
"engines": {
"node": ">= 14"
}
},
"node_modules/@prisma/adapter-mariadb/node_modules/undici-types": {
"version": "7.16.0",
"resolved": "https://registry.npmmirror.com/undici-types/-/undici-types-7.16.0.tgz",
"integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==",
"license": "MIT"
},
"node_modules/@prisma/client": {
"version": "7.7.0",
"resolved": "https://registry.npmmirror.com/@prisma/client/-/client-7.7.0.tgz",
@ -292,6 +348,15 @@
"zeptomatch": "2.1.0"
}
},
"node_modules/@prisma/driver-adapter-utils": {
"version": "7.7.0",
"resolved": "https://registry.npmmirror.com/@prisma/driver-adapter-utils/-/driver-adapter-utils-7.7.0.tgz",
"integrity": "sha512-gZXREeu6mOk7zXfGFJgh86p7Vhj0sXNKp+4Cg1tWYo7V2dfncP2qxS2BiTmbIIha8xPqItkl0WSw38RuSq1HoQ==",
"license": "Apache-2.0",
"dependencies": {
"@prisma/debug": "7.7.0"
}
},
"node_modules/@prisma/engines": {
"version": "7.7.0",
"resolved": "https://registry.npmmirror.com/@prisma/engines/-/engines-7.7.0.tgz",
@ -662,6 +727,21 @@
"integrity": "sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw==",
"license": "MIT"
},
"node_modules/@types/geojson": {
"version": "7946.0.16",
"resolved": "https://registry.npmmirror.com/@types/geojson/-/geojson-7946.0.16.tgz",
"integrity": "sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg==",
"license": "MIT"
},
"node_modules/@types/node": {
"version": "25.6.0",
"resolved": "https://registry.npmmirror.com/@types/node/-/node-25.6.0.tgz",
"integrity": "sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ==",
"license": "MIT",
"dependencies": {
"undici-types": "~7.19.0"
}
},
"node_modules/@types/react": {
"version": "19.2.14",
"resolved": "https://registry.npmmirror.com/@types/react/-/react-19.2.14.tgz",
@ -1856,6 +1936,12 @@
"loose-envify": "cli.js"
}
},
"node_modules/lru-cache": {
"version": "10.4.3",
"resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-10.4.3.tgz",
"integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==",
"license": "ISC"
},
"node_modules/lru.min": {
"version": "1.1.4",
"resolved": "https://registry.npmmirror.com/lru.min/-/lru.min-1.1.4.tgz",
@ -1871,6 +1957,22 @@
"url": "https://github.com/sponsors/wellwelwel"
}
},
"node_modules/mariadb": {
"version": "3.5.2",
"resolved": "https://registry.npmmirror.com/mariadb/-/mariadb-3.5.2.tgz",
"integrity": "sha512-9rztrI4nouxAY/82a+RlzzZ5ie2vxu2eYclkBvTy1ATXH1B9cnvZ0O71Pzsy/mlfDb5P3HhOg0JzQKkDRhctyA==",
"license": "LGPL-2.1-or-later",
"dependencies": {
"@types/geojson": "^7946.0.16",
"@types/node": ">=18",
"denque": "^2.1.0",
"iconv-lite": "^0.7.2",
"lru-cache": "^10.4.3"
},
"engines": {
"node": ">= 18"
}
},
"node_modules/marked": {
"version": "17.0.6",
"resolved": "https://registry.npmjs.org/marked/-/marked-17.0.6.tgz",
@ -2016,6 +2118,15 @@
}
}
},
"node_modules/node-cron": {
"version": "4.2.1",
"resolved": "https://registry.npmmirror.com/node-cron/-/node-cron-4.2.1.tgz",
"integrity": "sha512-lgimEHPE/QDgFlywTd8yTR61ptugX3Qer29efeyWw2rv259HtGBNn1vZVmp8lB9uo9wC0t/AT4iGqXxia+CJFg==",
"license": "ISC",
"engines": {
"node": ">=6.0.0"
}
},
"node_modules/node-fetch-native": {
"version": "1.6.7",
"resolved": "https://registry.npmmirror.com/node-fetch-native/-/node-fetch-native-1.6.7.tgz",
@ -2595,6 +2706,12 @@
"node": ">=20.18.1"
}
},
"node_modules/undici-types": {
"version": "7.19.2",
"resolved": "https://registry.npmmirror.com/undici-types/-/undici-types-7.19.2.tgz",
"integrity": "sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg==",
"license": "MIT"
},
"node_modules/use-sync-external-store": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.6.0.tgz",

View File

@ -1,6 +1,7 @@
{
"name": "iboard",
"version": "1.0.0",
"type": "module",
"scripts": {
"dev": "next dev",
"build": "next build",
@ -8,10 +9,13 @@
"test": "echo \"Error: no test specified\" && exit 1"
},
"dependencies": {
"@prisma/adapter-mariadb": "^7.7.0",
"@prisma/client": "^7.7.0",
"gray-matter": "^4.0.3",
"mariadb": "^3.5.2",
"marked": "^17.0.6",
"next": "^14.2.3",
"node-cron": "^4.2.1",
"prisma": "^7.7.0",
"react": "^18.2.0",
"react-dom": "^18.2.0",

108
pages/api/exchange-rate.js Normal file
View File

@ -0,0 +1,108 @@
import { createPool } from 'mariadb';
import fs from 'fs';
import path from 'path';
function loadDatabaseConfig() {
const envPath = path.join(process.cwd(), '.env');
if (fs.existsSync(envPath)) {
const envContent = fs.readFileSync(envPath, 'utf-8');
const lines = envContent.split('\n');
for (const line of lines) {
const trimmedLine = line.trim();
if (trimmedLine.startsWith('DATABASE_URL=')) {
const url = trimmedLine.substring('DATABASE_URL='.length).replace(/^"|"$/g, '');
// 解析URL
const parts = url.split('://')[1].split('@');
const auth = parts[0].split(':');
const hostAndDb = parts[1].split('/');
const hostAndPort = hostAndDb[0].split(':');
return {
user: auth[0],
password: auth[1],
host: hostAndPort[0],
port: parseInt(hostAndPort[1]),
database: hostAndDb[1]
};
}
}
}
return {
host: '192.168.111.111',
port: 3306,
user: 'root',
password: 'fullstack',
database: 'iboard'
};
}
let pool;
async function initPool() {
if (!pool) {
const config = loadDatabaseConfig();
pool = createPool({
host: config.host,
port: config.port,
user: config.user,
password: config.password,
database: config.database,
connectionLimit: 5
});
}
return pool;
}
export default async function handler(req, res) {
try {
await initPool();
// 获取查询参数
const { startDate, endDate, limit = 30 } = req.query;
let query = 'SELECT date, centerPrice FROM econ_ExchangeRate WHERE currency = ? ORDER BY date DESC';
const params = ['USD/CNY'];
if (startDate && endDate) {
query = 'SELECT date, centerPrice FROM econ_ExchangeRate WHERE currency = ? AND date BETWEEN ? AND ? ORDER BY date DESC';
params.push(startDate, endDate);
}
const result = await pool.query(query, params);
// 限制返回数量
const limitedResult = result.slice(0, parseInt(limit));
// 反转顺序,使日期从早到晚
const sortedResult = limitedResult.reverse();
// 格式化数据
const formattedData = sortedResult.map(row => ({
date: row.date instanceof Date ? row.date.toISOString().split('T')[0] : row.date,
rate: parseFloat(row.centerPrice)
}));
res.status(200).json({
success: true,
data: formattedData,
total: result.length
});
} catch (error) {
console.error('获取汇率数据失败:', error.message);
res.status(500).json({
success: false,
message: '获取汇率数据失败',
error: error.message
});
} finally {
// 不关闭连接池,保持连接池打开以提高性能
// 如果需要关闭连接池,应该在关闭后将 pool 变量设置为 null
// if (pool) {
// await pool.end();
// pool = null;
// }
}
}

View File

@ -1,7 +1,8 @@
import { useState } from 'react'
import { useState, useEffect } from 'react'
import Head from 'next/head'
import Navbar from '../components/Navbar'
import Footer from '../components/Footer'
import TimeRangePicker from '../components/TimeRangePicker'
import { LineChart, Line, XAxis, YAxis, CartesianGrid, Tooltip, Legend, ResponsiveContainer } from 'recharts'
const chinaBondData = [
@ -159,6 +160,55 @@ const sowInventoryData = [
export default function Data() {
const [selectedIndicator, setSelectedIndicator] = useState('bond-china-treasury')
const [exchangeRateData, setExchangeRateData] = useState([])
const [loading, setLoading] = useState(false)
const [error, setError] = useState(null)
const [timeRange, setTimeRange] = useState('1m') // 1m: 近一个月, 3m: 近三个月, 1y: 近一年, custom: 自定义
const [startDate, setStartDate] = useState('')
const [endDate, setEndDate] = useState('')
useEffect(() => {
if (selectedIndicator === 'central-bank-usd-cny-rate') {
fetchExchangeRateData()
}
}, [selectedIndicator, timeRange, startDate, endDate])
const handleTimeRangeChange = (range, start, end) => {
setTimeRange(range)
setStartDate(start)
setEndDate(end)
}
const fetchExchangeRateData = async () => {
setLoading(true)
setError(null)
try {
let url = '/api/exchange-rate'
if (timeRange === 'custom' && startDate && endDate) {
url += `?startDate=${startDate}&endDate=${endDate}`
} else if (timeRange === '1m') {
url += '?limit=30'
} else if (timeRange === '3m') {
url += '?limit=90'
} else if (timeRange === '1y') {
url += '?limit=365'
}
const response = await fetch(url)
const data = await response.json()
if (data.success) {
setExchangeRateData(data.data)
} else {
setError(data.message)
}
} catch (err) {
setError('获取汇率数据失败')
console.error('获取汇率数据失败:', err)
} finally {
setLoading(false)
}
}
const indicators = [
{
@ -180,7 +230,8 @@ export default function Data() {
{ id: 'central-bank-china-gold-monthly', name: '中国黄金储备(月)' },
{ id: 'central-bank-us-gold-monthly', name: '美国黄金储备(月)' },
{ id: 'central-bank-china-gold-yearly', name: '中国黄金储备(年)' },
{ id: 'central-bank-us-gold-yearly', name: '美国黄金储备(年)' }
{ id: 'central-bank-us-gold-yearly', name: '美国黄金储备(年)' },
{ id: 'central-bank-usd-cny-rate', name: '人民币兑美元汇率' }
]
},
{
@ -526,6 +577,61 @@ export default function Data() {
</div>
)}
{selectedIndicator === 'central-bank-usd-cny-rate' && (
<div className="indicator-detail">
<h2 className="indicator-title">人民币兑美元汇率</h2>
<p className="indicator-description">展示人民币兑美元汇率的变化趋势</p>
{loading ? (
<div className="loading">加载中...</div>
) : error ? (
<div className="error">{error}</div>
) : (
<>
<div className="chart-header">
<TimeRangePicker
value={timeRange}
onChange={handleTimeRangeChange}
/>
</div>
<div className="chart-container">
<ResponsiveContainer width="100%" height={500}>
<LineChart data={exchangeRateData} margin={{ top: 20, right: 30, left: 20, bottom: 5 }}>
<CartesianGrid strokeDasharray="3 3" />
<XAxis dataKey="date" />
<YAxis domain={['dataMin - 0.1', 'dataMax + 0.1']} tickFormatter={(value) => value.toFixed(4)} />
<Tooltip formatter={(value) => [value.toFixed(4), '汇率']} />
<Legend />
<Line type="monotone" dataKey="rate" name="汇率" stroke="#1e88e5" strokeWidth={2} activeDot={{ r: 8 }} />
</LineChart>
</ResponsiveContainer>
</div>
<div className="data-table-container">
<h3 className="table-title">详细数据</h3>
<table className="data-table">
<thead>
<tr>
<th>日期</th>
<th>汇率</th>
</tr>
</thead>
<tbody>
{exchangeRateData.map((row, index) => (
<tr key={index}>
<td>{row.date}</td>
<td>{row.rate.toFixed(4)}</td>
</tr>
))}
</tbody>
</table>
</div>
</>
)}
</div>
)}
{selectedIndicator === 'commodity-pork-price' && (
<div className="indicator-detail">
<h2 className="indicator-title">猪肉价格</h2>

View File

@ -0,0 +1,13 @@
-- CreateTable
CREATE TABLE `ScraperCache` (
`id` INTEGER NOT NULL AUTO_INCREMENT,
`scraperName` VARCHAR(191) NOT NULL,
`lastDataHash` VARCHAR(191) NULL,
`lastUpdateAt` DATETIME(3) NOT NULL,
`metadata` JSON NULL,
`createdAt` DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
`updatedAt` DATETIME(3) NOT NULL,
UNIQUE INDEX `ScraperCache_scraperName_key`(`scraperName`),
PRIMARY KEY (`id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;

View File

@ -56,3 +56,13 @@ model Econ_SowInventory {
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
}
model ScraperCache {
id Int @id @default(autoincrement())
scraperName String @unique // 爬虫名称sow-inventory
lastDataHash String? // 上次数据指纹
lastUpdateAt DateTime // 上次更新时间
metadata Json? // 其他元数据
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
}

13
readme
View File

@ -4,3 +4,16 @@
- 数据主要是放各种经济数据和图表。
- 策略是放按照经济理论组织的相关数据和图表的对比。
## 爬虫
- 基于 bun 的定时任务
- 统一获取接口
- 统一爬取接口
```
async function fetch(startDate = null, endDate = null)
async function trigger(startDate = null, endDate = null)
```
不传递startDate和endDate默认默认获取最近30天的数据。
fecher 负责从目标网站获取原始数据
scraper 负责检查数据库中的数据是否完整,调用 fecher 将数据存储到数据库。

View File

@ -0,0 +1,168 @@
import axios from 'axios';
import { fileURLToPath } from 'url';
async function fetch(startDate = null, endDate = null, retryCount = 3) {
try {
const currentDate = new Date();
if (!startDate) {
startDate = new Date(currentDate);
startDate.setDate(startDate.getDate() - 30);
startDate = startDate.toISOString().split('T')[0];
}
if (!endDate) {
endDate = currentDate.toISOString().split('T')[0];
}
console.log('正在获取沪深300指数数据...');
console.log(`时间范围: ${startDate}${endDate}`);
console.log(`数据源: 腾讯财经API`);
// 沪深300指数代码
const symbol = '000300';
// 使用腾讯财经的API端点获取历史数据
const response = await axios.get('https://web.ifzq.gtimg.cn/appstock/app/kline/kline?param=sh' + symbol + ',day,' + startDate + ',' + endDate + ',640', {
headers: {
'Accept': 'application/json',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
},
timeout: 10000 // 设置10秒超时
});
console.log(`\nAPI状态: ${response.status}`);
const data = response.data;
console.log(`响应数据:`, JSON.stringify(data, null, 2));
const allData = [];
if (data && data.data && data.data['sh' + symbol] && data.data['sh' + symbol].day) {
const klineData = data.data['sh' + symbol].day;
console.log(`数据条数: ${klineData.length}`);
for (const item of klineData) {
allData.push({
date: item[0],
open: parseFloat(item[1]),
close: parseFloat(item[2]),
high: parseFloat(item[3]),
low: parseFloat(item[4]),
volume: parseFloat(item[5]),
amount: 0, // 腾讯财经API没有返回成交额数据设置为0
source: '腾讯财经API',
fetchTime: new Date().toISOString()
});
}
} else {
console.log('数据条数: 0');
}
allData.sort((a, b) => a.date.localeCompare(b.date));
console.log(`\n共获取到 ${allData.length} 条沪深300指数数据`);
return {
symbol: 'sh000300',
name: '沪深300指数',
startDate,
endDate,
count: allData.length,
data: allData,
fetchTime: new Date().toISOString(),
note: '数据来源: 腾讯财经API'
};
} catch (error) {
console.error('获取沪深300指数数据失败:', error.message);
if (error.response) {
console.error(`HTTP状态码: ${error.response.status}`);
console.error(`响应数据:`, error.response.data);
}
// 重试机制
if (retryCount > 0) {
console.log(`正在重试... (剩余重试次数: ${retryCount - 1})`);
// 等待1秒后重试
await new Promise(resolve => setTimeout(resolve, 1000));
return fetch(startDate, endDate, retryCount - 1);
}
throw error;
}
}
export {
fetch
};
const isMainModule = () => {
const currentFile = fileURLToPath(import.meta.url);
const mainFile = process.argv[1];
return currentFile === mainFile;
};
function parseCommandLineArgs() {
const args = process.argv.slice(2);
let startDate = null;
let endDate = null;
for (let i = 0; i < args.length; i++) {
if (args[i] === '--start' && i + 1 < args.length) {
startDate = args[i + 1];
i++;
} else if (args[i] === '--end' && i + 1 < args.length) {
endDate = args[i + 1];
i++;
} else if (args[i] === '--help') {
console.log('使用方法:');
console.log(' bun sh300.fetcher.js [选项]');
console.log('');
console.log('选项:');
console.log(' --start YYYY-MM-DD 指定开始日期默认30天前');
console.log(' --end YYYY-MM-DD 指定结束日期(默认:今天)');
console.log(' --help 显示帮助信息');
console.log('');
console.log('示例:');
console.log(' bun sh300.fetcher.js');
console.log(' bun sh300.fetcher.js --start 2024-01-01 --end 2024-01-31');
process.exit(0);
}
}
return { startDate, endDate };
}
if (isMainModule()) {
const { startDate, endDate } = parseCommandLineArgs();
console.log('='.repeat(50));
console.log('沪深300指数数据获取脚本');
console.log('='.repeat(50));
fetch(startDate, endDate)
.then(result => {
console.log('\n' + '='.repeat(50));
console.log('获取结果:');
console.log(` 指数代码: ${result.symbol}`);
console.log(` 指数名称: ${result.name}`);
console.log(` 开始日期: ${result.startDate}`);
console.log(` 结束日期: ${result.endDate}`);
console.log(` 数据条数: ${result.count}`);
console.log(` 获取时间: ${result.fetchTime}`);
console.log(` 提示: ${result.note}`);
if (result.count > 0) {
console.log('-'.repeat(50));
console.log('指数数据:');
result.data.forEach(item => {
console.log(` ${item.date} 开盘: ${item.open} 最高: ${item.high} 最低: ${item.low} 收盘: ${item.close}`);
});
}
console.log('='.repeat(50));
})
.catch(error => {
console.error('执行失败:', error);
process.exit(1);
});
}

View File

@ -0,0 +1,150 @@
import axios from 'axios';
import { fileURLToPath } from 'url';
const FRANKFURTER_API = 'https://api.frankfurter.app';
async function fetch(startDate = null, endDate = null) {
try {
const currentDate = new Date();
if (!startDate) {
startDate = new Date(currentDate);
startDate.setDate(startDate.getDate() - 30);
startDate = startDate.toISOString().split('T')[0];
}
if (!endDate) {
endDate = currentDate.toISOString().split('T')[0];
}
console.log('正在获取人民币美元汇率数据...');
console.log(`时间范围: ${startDate}${endDate}`);
console.log(`数据源: Frankfurter API (欧洲央行汇率数据)`);
const response = await axios.get(`${FRANKFURTER_API}/${startDate}..${endDate}`, {
params: {
from: 'USD',
to: 'CNY'
},
headers: {
'Accept': 'application/json'
}
});
const data = response.data;
console.log(`\nAPI状态: ${response.status}`);
console.log(`数据条数: ${Object.keys(data.rates || {}).length}`);
const allRates = [];
if (data.rates) {
for (const [date, rateObj] of Object.entries(data.rates)) {
const rateValue = rateObj.CNY;
allRates.push({
date: date,
currency: 'USD/CNY',
centerPrice: parseFloat(rateValue.toFixed(4)),
sellingRate: parseFloat(rateValue.toFixed(4)),
buyingRate: parseFloat(rateValue.toFixed(4)),
source: '欧洲央行 (Frankfurter API)',
fetchTime: new Date().toISOString()
});
}
}
allRates.sort((a, b) => a.date.localeCompare(b.date));
console.log(`\n共获取到 ${allRates.length} 条汇率数据`);
return {
currency: 'USD/CNY',
startDate,
endDate,
count: allRates.length,
data: allRates,
fetchTime: new Date().toISOString(),
note: '数据来源: 欧洲央行, 通过 Frankfurter API 提供'
};
} catch (error) {
console.error('获取汇率数据失败:', error.message);
if (error.response) {
console.error(`HTTP状态码: ${error.response.status}`);
console.error(`响应数据:`, error.response.data);
}
throw error;
}
}
export {
fetch
};
const isMainModule = () => {
const currentFile = fileURLToPath(import.meta.url);
const mainFile = process.argv[1];
return currentFile === mainFile;
};
function parseCommandLineArgs() {
const args = process.argv.slice(2);
let startDate = null;
let endDate = null;
for (let i = 0; i < args.length; i++) {
if (args[i] === '--start' && i + 1 < args.length) {
startDate = args[i + 1];
i++;
} else if (args[i] === '--end' && i + 1 < args.length) {
endDate = args[i + 1];
i++;
} else if (args[i] === '--help') {
console.log('使用方法:');
console.log(' bun usd-cny-rate.fetcher.js [选项]');
console.log('');
console.log('选项:');
console.log(' --start YYYY-MM-DD 指定开始日期默认30天前');
console.log(' --end YYYY-MM-DD 指定结束日期(默认:今天)');
console.log(' --help 显示帮助信息');
console.log('');
console.log('示例:');
console.log(' bun usd-cny-rate.fetcher.js');
console.log(' bun usd-cny-rate.fetcher.js --start 2024-01-01 --end 2024-01-31');
process.exit(0);
}
}
return { startDate, endDate };
}
if (isMainModule()) {
const { startDate, endDate } = parseCommandLineArgs();
console.log('='.repeat(50));
console.log('人民币美元汇率获取脚本');
console.log('='.repeat(50));
fetch(startDate, endDate)
.then(result => {
console.log('\n' + '='.repeat(50));
console.log('获取结果:');
console.log(` 货币对: ${result.currency}`);
console.log(` 开始日期: ${result.startDate}`);
console.log(` 结束日期: ${result.endDate}`);
console.log(` 数据条数: ${result.count}`);
console.log(` 获取时间: ${result.fetchTime}`);
console.log(` 提示: ${result.note}`);
if (result.count > 0) {
console.log('-'.repeat(50));
console.log('汇率数据:');
result.data.forEach(item => {
console.log(` ${item.date} 中间价: ${item.centerPrice}`);
});
}
console.log('='.repeat(50));
})
.catch(error => {
console.error('执行失败:', error);
process.exit(1);
});
}

View File

@ -0,0 +1,49 @@
import cron from 'node-cron';
import { trigger as triggerUSDCNYRate } from '../scraper/usd-cny-rate.scraper.js';
console.log('='.repeat(70));
console.log('🚀 启动定时任务调度器');
console.log('='.repeat(70));
// 注册定时任务
const job = cron.schedule('0 0 * * *', async () => {
console.log('\n' + '='.repeat(70));
console.log('⏰ 执行定时任务: USD-CNY 汇率数据更新');
console.log('📅 执行时间:', new Date().toISOString());
console.log('='.repeat(70));
try {
await triggerUSDCNYRate();
console.log('\n✅ 任务执行成功!');
} catch (error) {
console.error('\n❌ 任务执行失败:', error.message);
} finally {
console.log('='.repeat(70));
}
}, {
scheduled: true,
timezone: 'Asia/Shanghai'
});
console.log('✅ 定时任务已启动: USD-CNY 汇率数据更新');
console.log('📅 执行表达式: 0 0 * * * (每天 0 点)');
console.log('📅 下次执行时间: 明天 00:00');
console.log('');
console.log('按 Ctrl+C 停止...');
console.log('='.repeat(70));
// 优雅处理退出
process.on('SIGINT', () => {
console.log('\n' + '='.repeat(70));
console.log('🛑 停止定时任务调度器');
console.log('='.repeat(70));
job.stop();
console.log('✅ 定时任务已停止');
console.log('\n' + '='.repeat(70));
console.log('📊 定时任务调度器已停止');
console.log('='.repeat(70));
process.exit(0);
});

49
scripts/scheduler/test.js Normal file
View File

@ -0,0 +1,49 @@
import cron from 'node-cron';
import { trigger as triggerUSDCNYRate } from '../scraper/usd-cny-rate.scraper.js';
console.log('='.repeat(70));
console.log('🚀 启动定时任务调度器');
console.log('='.repeat(70));
// 注册定时任务
const job = cron.schedule('0 0 * * *', async () => {
console.log('\n' + '='.repeat(70));
console.log('⏰ 执行定时任务: USD-CNY 汇率数据更新');
console.log('📅 执行时间:', new Date().toISOString());
console.log('='.repeat(70));
try {
await triggerUSDCNYRate();
console.log('\n✅ 任务执行成功!');
} catch (error) {
console.error('\n❌ 任务执行失败:', error.message);
} finally {
console.log('='.repeat(70));
}
}, {
scheduled: true,
timezone: 'Asia/Shanghai'
});
console.log('✅ 定时任务已启动: USD-CNY 汇率数据更新');
console.log('📅 执行表达式: 0 0 * * * (每天 0 点)');
console.log('📅 下次执行时间: 明天 00:00');
console.log('');
console.log('按 Ctrl+C 停止...');
console.log('='.repeat(70));
// 优雅处理退出
process.on('SIGINT', () => {
console.log('\n' + '='.repeat(70));
console.log('🛑 停止定时任务调度器');
console.log('='.repeat(70));
job.stop();
console.log('✅ 定时任务已停止');
console.log('\n' + '='.repeat(70));
console.log('📊 定时任务调度器已停止');
console.log('='.repeat(70));
process.exit(0);
});

3
scripts/scraper/code.md Normal file
View File

@ -0,0 +1,3 @@
git:<http://git.honor3.com/Fullstack/iboard>
dragon:D:\workbench\iboard

View File

@ -0,0 +1,302 @@
import { createPool } from 'mariadb';
import { fetch as fetchSH300Data } from '../fetchers/sh300.fetcher.js';
import fs from 'fs';
import path from 'path';
let pool;
function loadDatabaseConfig() {
const envPath = path.join(process.cwd(), '.env');
if (fs.existsSync(envPath)) {
console.log('读取 .env 文件...');
const envContent = fs.readFileSync(envPath, 'utf-8');
const lines = envContent.split('\n');
for (const line of lines) {
const trimmedLine = line.trim();
if (trimmedLine.startsWith('DATABASE_URL=')) {
const url = trimmedLine.substring('DATABASE_URL='.length).replace(/^"|"$/g, '');
console.log('从 .env 文件获取数据库配置成功!');
console.log(`DATABASE_URL: ${url}`);
// 解析URL
const parts = url.split('://')[1].split('@');
const auth = parts[0].split(':');
const hostAndDb = parts[1].split('/');
const hostAndPort = hostAndDb[0].split(':');
return {
user: auth[0],
password: auth[1],
host: hostAndPort[0],
port: parseInt(hostAndPort[1]),
database: hostAndDb[1]
};
}
}
}
console.log('未找到 .env 文件或 DATABASE_URL 配置,使用默认配置');
return {
host: '192.168.111.111',
port: 3306,
user: 'root',
password: 'fullstack',
database: 'iboard'
};
}
async function initPool() {
if (!pool) {
console.log('初始化数据库连接池...');
const config = loadDatabaseConfig();
console.log('数据库配置:');
console.log(` 主机: ${config.host}`);
console.log(` 端口: ${config.port}`);
console.log(` 用户: ${config.user}`);
console.log(` 数据库: ${config.database}`);
pool = createPool({
host: config.host,
port: config.port,
user: config.user,
password: config.password,
database: config.database,
connectionLimit: 5
});
try {
const conn = await pool.getConnection();
console.log('数据库连接成功!');
conn.release();
} catch (error) {
console.error('数据库连接失败:', error.message);
throw error;
}
}
return pool;
}
async function checkAndFillSH300Data(startDate = null, endDate = null) {
try {
await initPool();
console.log('='.repeat(60));
console.log('开始检查和补充沪深300指数数据...');
console.log('='.repeat(60));
const currentDate = new Date();
if (!startDate) {
startDate = new Date(currentDate);
startDate.setDate(startDate.getDate() - 30);
startDate = startDate.toISOString().split('T')[0];
}
if (!endDate) {
endDate = currentDate.toISOString().split('T')[0];
}
console.log(`检查时间范围: ${startDate}${endDate}`);
// 1. 检查数据库中是否存在沪深300指数数据表
const hasTable = await checkTableExists();
if (!hasTable) {
console.log('创建沪深300指数数据表...');
await createSH300Table();
}
// 2. 检查数据库中已有的数据
const existingData = await getExistingSH300Data(startDate, endDate);
console.log(`数据库中已有 ${existingData.length} 条数据`);
// 3. 计算缺失的日期
const missingDates = calculateMissingDates(startDate, endDate, existingData);
console.log(`缺失 ${missingDates.length} 天的数据`);
// 4. 如果有缺失,调用 fetcher 补足
if (missingDates.length > 0) {
console.log('\n开始获取缺失的沪深300指数数据...');
const firstMissingDate = missingDates[0];
const lastMissingDate = missingDates[missingDates.length - 1];
console.log(`获取范围: ${firstMissingDate}${lastMissingDate}`);
const fetchedData = await fetchSH300Data(firstMissingDate, lastMissingDate);
console.log(`\n获取到 ${fetchedData.count} 条数据`);
// 5. 存储新数据到数据库
const storedCount = await storeSH300Data(fetchedData.data);
console.log(`成功存储 ${storedCount} 条数据`);
} else {
console.log('\n数据库数据完整无需补充');
}
// 6. 验证最终数据完整性
const finalData = await getExistingSH300Data(startDate, endDate);
console.log(`\n验证后的数据总量: ${finalData.length}`);
return {
startDate,
endDate,
existingCount: existingData.length,
missingCount: missingDates.length,
finalCount: finalData.length,
success: true
};
} catch (error) {
console.error('检查和补充沪深300指数数据失败:', error.message);
throw error;
} finally {
if (pool) {
await pool.end();
console.log('数据库连接池已关闭');
}
}
}
async function checkTableExists() {
try {
console.log('检查沪深300指数数据表是否存在...');
const result = await pool.query('SHOW TABLES LIKE ?', ['econ_SH300Index']);
return result.length > 0;
} catch (error) {
console.error('检查表存在失败:', error.message);
return false;
}
}
async function createSH300Table() {
try {
console.log('执行创建沪深300指数数据表SQL...');
await pool.query(`
CREATE TABLE IF NOT EXISTS econ_SH300Index (
id INT AUTO_INCREMENT PRIMARY KEY,
date DATE UNIQUE NOT NULL,
symbol VARCHAR(10) NOT NULL,
open DECIMAL(10,2) NOT NULL,
close DECIMAL(10,2) NOT NULL,
high DECIMAL(10,2) NOT NULL,
low DECIMAL(10,2) NOT NULL,
volume DECIMAL(20,2) NOT NULL,
amount DECIMAL(20,2) NOT NULL,
source VARCHAR(100) NOT NULL,
createdAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updatedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
`);
console.log('沪深300指数数据表创建成功');
} catch (error) {
console.error('创建沪深300指数数据表失败:', error.message);
throw error;
}
}
async function getExistingSH300Data(startDate, endDate) {
try {
console.log('查询已有沪深300指数数据...');
const result = await pool.query(
'SELECT date FROM econ_SH300Index WHERE date BETWEEN ? AND ? ORDER BY date',
[startDate, endDate]
);
const dates = result.map(row => {
if (row.date instanceof Date) {
return row.date.toISOString().split('T')[0];
} else if (typeof row.date === 'string') {
return row.date;
}
return null;
}).filter(Boolean);
console.log(`查询到 ${dates.length} 条数据`);
return dates;
} catch (error) {
console.error('查询已有沪深300指数数据失败:', error.message);
return [];
}
}
function calculateMissingDates(startDate, endDate, existingDates) {
const existingSet = new Set(existingDates);
const missingDates = [];
const currentDate = new Date(startDate);
const finalDate = new Date(endDate);
while (currentDate <= finalDate) {
const dateStr = currentDate.toISOString().split('T')[0];
const dayOfWeek = currentDate.getDay();
// 跳过周末(周末没有股市数据)
if (dayOfWeek !== 0 && dayOfWeek !== 6) {
if (!existingSet.has(dateStr)) {
missingDates.push(dateStr);
}
}
currentDate.setDate(currentDate.getDate() + 1);
}
return missingDates;
}
async function storeSH300Data(data) {
let storedCount = 0;
for (const item of data) {
try {
await pool.query(
'INSERT INTO econ_SH300Index (date, symbol, open, close, high, low, volume, amount, source) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE open = VALUES(open), close = VALUES(close), high = VALUES(high), low = VALUES(low), volume = VALUES(volume), amount = VALUES(amount), source = VALUES(source)',
[item.date, 'sh000300', item.open, item.close, item.high, item.low, item.volume, item.amount, item.source]
);
storedCount++;
} catch (error) {
console.error(`存储沪深300指数数据 ${item.date} 失败:`, error.message);
}
}
return storedCount;
}
export async function trigger(startDate = null, endDate = null) {
return await checkAndFillSH300Data(startDate, endDate);
}
// 直接运行测试
async function runTest() {
console.log('='.repeat(60));
console.log('测试 sh300.scraper.js');
console.log('='.repeat(60));
// 测试配置读取
console.log('\n测试数据库配置读取...');
const config = loadDatabaseConfig();
console.log('配置读取结果:');
console.log(` 主机: ${config.host}`);
console.log(` 端口: ${config.port}`);
console.log(` 用户: ${config.user}`);
console.log(` 数据库: ${config.database}`);
console.log('配置读取测试完成!');
try {
const result = await trigger();
console.log('\n' + '='.repeat(60));
console.log('测试结果:');
console.log(` 开始日期: ${result.startDate}`);
console.log(` 结束日期: ${result.endDate}`);
console.log(` 原有数据: ${result.existingCount}`);
console.log(` 缺失数据: ${result.missingCount}`);
console.log(` 最终数据: ${result.finalCount}`);
console.log(` 操作成功: ${result.success}`);
console.log('='.repeat(60));
} catch (error) {
console.error('测试失败:', error);
process.exit(1);
}
}
// 运行测试
runTest();

View File

@ -1,33 +1,205 @@
const axios = require('axios');
const cheerio = require('cheerio');
const fs = require('fs');
const path = require('path');
const { PrismaClient } = require('../../generated/prisma');
import axios from 'axios';
import * as cheerio from 'cheerio';
import { createPool } from 'mariadb';
import fs from 'fs';
import path from 'path';
const prisma = new PrismaClient();
function loadDatabaseConfig() {
const envPath = path.join(process.cwd(), '.env');
if (fs.existsSync(envPath)) {
console.log('读取 .env 文件...');
const envContent = fs.readFileSync(envPath, 'utf-8');
const lines = envContent.split('\n');
for (const line of lines) {
const trimmedLine = line.trim();
if (trimmedLine.startsWith('DATABASE_URL=')) {
const url = trimmedLine.substring('DATABASE_URL='.length).replace(/^"|"$/g, '');
console.log('从 .env 文件获取数据库配置成功!');
console.log(`DATABASE_URL: ${url}`);
// 解析URL
const parts = url.split('://')[1].split('@');
const auth = parts[0].split(':');
const hostAndDb = parts[1].split('/');
const hostAndPort = hostAndDb[0].split(':');
return {
user: auth[0],
password: auth[1],
host: hostAndPort[0],
port: parseInt(hostAndPort[1]),
database: hostAndDb[1]
};
}
}
}
console.log('未找到 .env 文件或 DATABASE_URL 配置,使用默认配置');
return {
host: '192.168.111.111',
port: 3306,
user: 'root',
password: 'fullstack',
database: 'iboard'
};
}
// 创建数据库连接池
const config = loadDatabaseConfig();
console.log('数据库配置:');
console.log(` 主机: ${config.host}`);
console.log(` 端口: ${config.port}`);
console.log(` 用户: ${config.user}`);
console.log(` 数据库: ${config.database}`);
let pool;
async function initPool() {
if (!pool) {
pool = createPool({
host: config.host,
port: config.port,
user: config.user,
password: config.password,
database: config.database,
connectionLimit: 5
});
try {
const conn = await pool.getConnection();
console.log('数据库连接成功!');
conn.release();
} catch (error) {
console.error('数据库连接失败:', error.message);
throw error;
}
}
return pool;
}
// 浙江数据开放网站URL示例实际需要根据真实网站修改
const ZHEJIANG_OPEN_DATA_URL = 'https://data.zj.gov.cn/';
// 能繁母猪数量数据爬取函数
async function scrapeSowInventory() {
// 爬虫名称,用于缓存表中的标识
const SCRAPER_NAME = 'sow-inventory';
// 最小更新间隔毫秒默认30分钟
const MIN_UPDATE_INTERVAL = 30 * 60 * 1000;
// 从数据库读取缓存
async function readCache() {
try {
await initPool();
const result = await pool.query('SELECT * FROM ScraperCache WHERE scraperName = ?', [SCRAPER_NAME]);
return result.length > 0 ? result[0] : { scraperName: SCRAPER_NAME, lastDataHash: null, lastUpdateAt: null };
} catch (error) {
console.error('读取数据库缓存失败:', error.message);
return { scraperName: SCRAPER_NAME, lastDataHash: null, lastUpdateAt: null };
}
}
// 写入数据库缓存
async function writeCache(lastDataHash) {
try {
await initPool();
await pool.query(
'INSERT INTO ScraperCache (scraperName, lastDataHash, lastUpdateAt) VALUES (?, ?, ?) ON DUPLICATE KEY UPDATE lastDataHash = VALUES(lastDataHash), lastUpdateAt = VALUES(lastUpdateAt)',
[SCRAPER_NAME, lastDataHash, new Date()]
);
} catch (error) {
console.error('写入数据库缓存失败:', error.message);
}
}
// 计算数据hash用于检测数据是否变化
function computeDataHash(data) {
const str = JSON.stringify(data);
let hash = 0;
for (let i = 0; i < str.length; i++) {
const char = str.charCodeAt(i);
hash = ((hash << 5) - hash) + char;
hash = hash & hash;
}
return hash.toString();
}
// 获取数据库中最新的月份
async function getLatestMonth() {
try {
await initPool();
const result = await pool.query('SELECT MAX(month) as latestMonth FROM Econ_SowInventory');
return result[0].latestMonth || null;
} catch (error) {
console.error('查询最新月份失败:', error.message);
return null;
}
}
// 判断月份是否需要更新(只有比数据库中更新的月份才需要处理)
function shouldUpdateMonth(month, latestDbMonth) {
if (!latestDbMonth) return true;
return month > latestDbMonth;
}
// 检查是否需要发起HTTP请求基于更新间隔
function shouldSkipRequest(cache) {
if (!cache.lastUpdateAt) return false;
const now = new Date();
const lastUpdate = new Date(cache.lastUpdateAt);
const interval = now - lastUpdate;
return interval < MIN_UPDATE_INTERVAL;
}
// 能繁母猪数量数据爬取函数(增量更新版本)
async function scrapeSowInventory(incremental = true) {
try {
await initPool();
console.log('='.repeat(50));
console.log('开始爬取浙江省能繁母猪数量数据...');
console.log(`增量更新模式: ${incremental ? '开启' : '关闭'}`);
console.log('='.repeat(50));
// 0. 检查是否需要发起请求
if (incremental) {
const cache = await readCache();
if (shouldSkipRequest(cache)) {
const lastUpdate = cache.lastUpdateAt ? new Date(cache.lastUpdateAt) : null;
const hoursDiff = lastUpdate ? ((new Date() - lastUpdate) / (1000 * 60 * 60)).toFixed(2) : '未知';
console.log('\n' + '='.repeat(50));
console.log('跳过请求距离上次更新不足30分钟');
console.log(`上次更新时间: ${cache.lastUpdateAt}`);
console.log(`距离现在: ${hoursDiff} 小时`);
console.log('='.repeat(50));
return;
}
// 获取数据库中最新的月份
const latestDbMonth = await getLatestMonth();
if (latestDbMonth) {
console.log(`数据库中最新月份: ${latestDbMonth}`);
} else {
console.log('数据库为空,将获取所有数据');
}
}
// 1. 发送HTTP请求获取页面内容
console.log('\n正在请求数据源...');
const response = await axios.get(ZHEJIANG_OPEN_DATA_URL, {
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
}
});
// 2. 解析HTML内容
const $ = cheerio.load(response.data);
// 3. 提取数据(这里需要根据实际网站结构修改选择器)
// 示例:假设数据在表格中
const data = [];
const allData = [];
$('table').each((tableIndex, table) => {
$(table).find('tr').each((rowIndex, row) => {
if (rowIndex > 0) { // 跳过表头
@ -35,9 +207,9 @@ async function scrapeSowInventory() {
if (cells.length >= 2) {
const month = $(cells[0]).text().trim();
const inventory = parseInt($(cells[1]).text().trim());
if (month && !isNaN(inventory)) {
data.push({
allData.push({
month,
inventory
});
@ -46,50 +218,135 @@ async function scrapeSowInventory() {
}
});
});
console.log(`爬取到 ${data.length} 条数据`);
// 4. 存储数据到数据库
for (const item of data) {
console.log(`页面总数据量: ${allData.length}`);
// 4. 读取数据库缓存
const cache = await readCache();
// 5. 计算当前数据hash检测数据是否变化
const currentDataHash = computeDataHash(allData);
// 6. 如果是增量更新且数据未变化,跳过更新
if (incremental && cache.lastDataHash === currentDataHash) {
console.log('\n' + '='.repeat(50));
console.log('数据未发生变化,跳过数据库更新!');
console.log(`上次更新时间: ${cache.lastUpdateAt || '未知'}`);
console.log('='.repeat(50));
return;
}
// 7. 如果是增量更新,获取数据库中最新的月份用于过滤
let latestDbMonth = null;
if (incremental) {
latestDbMonth = await getLatestMonth();
}
// 8. 过滤需要更新的数据
let dataToUpdate = allData;
let skippedCount = 0;
if (incremental && latestDbMonth) {
dataToUpdate = allData.filter(item => shouldUpdateMonth(item.month, latestDbMonth));
skippedCount = allData.length - dataToUpdate.length;
console.log(`\n增量更新: 跳过 ${skippedCount} 条已存在的数据`);
console.log(`需要处理: ${dataToUpdate.length} 条新数据`);
}
// 9. 如果没有需要更新的数据
if (dataToUpdate.length === 0) {
console.log('\n没有需要更新的数据');
return;
}
// 10. 按月份排序(从旧到新)
dataToUpdate.sort((a, b) => a.month.localeCompare(b.month));
console.log('\n开始处理数据...');
console.log('-'.repeat(50));
let newCount = 0;
let updateCount = 0;
let errorCount = 0;
// 11. 存储数据到数据库
for (const item of dataToUpdate) {
try {
// 检查数据是否已存在
const existing = await prisma.econ_SowInventory.findUnique({
where: { month: item.month }
});
const existing = await pool.query('SELECT * FROM Econ_SowInventory WHERE month = ?', [item.month]);
if (existing) {
// 更新现有数据
await prisma.econ_SowInventory.update({
where: { month: item.month },
data: { inventory: item.inventory }
});
console.log(`更新数据: ${item.month} - ${item.inventory}`);
if (existing.length > 0) {
// 检查数据是否真的需要更新(值变化了才更新)
if (existing[0].inventory !== item.inventory) {
await pool.query(
'UPDATE Econ_SowInventory SET inventory = ? WHERE month = ?',
[item.inventory, item.month]
);
updateCount++;
console.log(`更新数据: ${item.month} - ${item.inventory} (原值: ${existing[0].inventory})`);
} else {
console.log(`数据未变化,跳过: ${item.month}`);
}
} else {
// 创建新数据
await prisma.econ_SowInventory.create({
data: item
});
await pool.query(
'INSERT INTO Econ_SowInventory (month, inventory) VALUES (?, ?)',
[item.month, item.inventory]
);
newCount++;
console.log(`新增数据: ${item.month} - ${item.inventory}`);
}
} catch (error) {
errorCount++;
console.error(`处理数据 ${item.month} 时出错:`, error.message);
}
}
// 12. 更新数据库缓存
await writeCache(currentDataHash);
// 13. 输出统计信息
const now = new Date().toISOString();
console.log('\n' + '='.repeat(50));
console.log('数据爬取和存储完成!');
console.log('统计信息:');
console.log(` - 新增数据: ${newCount}`);
console.log(` - 更新数据: ${updateCount}`);
console.log(` - 跳过数据: ${skippedCount}`);
console.log(` - 错误数量: ${errorCount}`);
console.log(` - 本次更新时间: ${now}`);
console.log('='.repeat(50));
} catch (error) {
console.error('爬取数据时出错:', error.message);
throw error;
} finally {
// 关闭Prisma连接
await prisma.$disconnect();
// 关闭数据库连接池
if (pool) {
await pool.end();
console.log('数据库连接池已关闭');
}
}
}
// 强制全量更新函数
async function scrapeSowInventoryFull() {
console.log('警告: 即将执行全量更新,这将更新所有数据!');
console.log('如果只是想获取新数据,请使用 scrapeSowInventory() 函数。\n');
await scrapeSowInventory(false);
}
// 导出函数
module.exports = { scrapeSowInventory };
export {
scrapeSowInventory,
scrapeSowInventoryFull
};
// 如果直接运行此文件
if (require.main === module) {
scrapeSowInventory();
if (import.meta.url === `file://${process.argv[1]}`) {
// 默认执行增量更新
// 如果需要强制全量更新,使用: scrapeSowInventoryFull()
scrapeSowInventory().catch(error => {
console.error('执行失败:', error);
process.exit(1);
});
}

View File

@ -0,0 +1,299 @@
import { createPool } from 'mariadb';
import { fetch as fetchUSDToCNYRate } from '../fetchers/usd-cny-rate.fetcher.js';
import fs from 'fs';
import path from 'path';
let pool;
function loadDatabaseConfig() {
const envPath = path.join(process.cwd(), '.env');
if (fs.existsSync(envPath)) {
console.log('读取 .env 文件...');
const envContent = fs.readFileSync(envPath, 'utf-8');
const lines = envContent.split('\n');
for (const line of lines) {
const trimmedLine = line.trim();
if (trimmedLine.startsWith('DATABASE_URL=')) {
const url = trimmedLine.substring('DATABASE_URL='.length).replace(/^"|"$/g, '');
console.log('从 .env 文件获取数据库配置成功!');
console.log(`DATABASE_URL: ${url}`);
// 解析URL
const parts = url.split('://')[1].split('@');
const auth = parts[0].split(':');
const hostAndDb = parts[1].split('/');
const hostAndPort = hostAndDb[0].split(':');
return {
user: auth[0],
password: auth[1],
host: hostAndPort[0],
port: parseInt(hostAndPort[1]),
database: hostAndDb[1]
};
}
}
}
console.log('未找到 .env 文件或 DATABASE_URL 配置,使用默认配置');
return {
host: '192.168.111.111',
port: 3306,
user: 'root',
password: 'fullstack',
database: 'iboard'
};
}
async function initPool() {
if (!pool) {
console.log('初始化数据库连接池...');
const config = loadDatabaseConfig();
console.log('数据库配置:');
console.log(` 主机: ${config.host}`);
console.log(` 端口: ${config.port}`);
console.log(` 用户: ${config.user}`);
console.log(` 数据库: ${config.database}`);
pool = createPool({
host: config.host,
port: config.port,
user: config.user,
password: config.password,
database: config.database,
connectionLimit: 5
});
try {
const conn = await pool.getConnection();
console.log('数据库连接成功!');
conn.release();
} catch (error) {
console.error('数据库连接失败:', error.message);
throw error;
}
}
return pool;
}
async function checkAndFillUSDCNYRate(startDate = null, endDate = null) {
try {
await initPool();
console.log('='.repeat(60));
console.log('开始检查和补充人民币美元汇率数据...');
console.log('='.repeat(60));
const currentDate = new Date();
if (!startDate) {
startDate = new Date(currentDate);
startDate.setDate(startDate.getDate() - 30);
startDate = startDate.toISOString().split('T')[0];
}
if (!endDate) {
endDate = currentDate.toISOString().split('T')[0];
}
console.log(`检查时间范围: ${startDate}${endDate}`);
// 1. 检查数据库中是否存在汇率数据表
const hasTable = await checkTableExists();
if (!hasTable) {
console.log('创建汇率数据表...');
await createExchangeRateTable();
}
// 2. 检查数据库中已有的数据
const existingData = await getExistingExchangeRates(startDate, endDate);
console.log(`数据库中已有 ${existingData.length} 条数据`);
// 3. 计算缺失的日期
const missingDates = calculateMissingDates(startDate, endDate, existingData);
console.log(`缺失 ${missingDates.length} 天的数据`);
// 4. 如果有缺失,调用 fetcher 补足
if (missingDates.length > 0) {
console.log('\n开始获取缺失的汇率数据...');
const firstMissingDate = missingDates[0];
const lastMissingDate = missingDates[missingDates.length - 1];
console.log(`获取范围: ${firstMissingDate}${lastMissingDate}`);
const fetchedData = await fetchUSDToCNYRate(firstMissingDate, lastMissingDate);
console.log(`\n获取到 ${fetchedData.count} 条数据`);
// 5. 存储新数据到数据库
const storedCount = await storeExchangeRates(fetchedData.data);
console.log(`成功存储 ${storedCount} 条数据`);
} else {
console.log('\n数据库数据完整无需补充');
}
// 6. 验证最终数据完整性
const finalData = await getExistingExchangeRates(startDate, endDate);
console.log(`\n验证后的数据总量: ${finalData.length}`);
return {
startDate,
endDate,
existingCount: existingData.length,
missingCount: missingDates.length,
finalCount: finalData.length,
success: true
};
} catch (error) {
console.error('检查和补充汇率数据失败:', error.message);
throw error;
} finally {
if (pool) {
await pool.end();
console.log('数据库连接池已关闭');
}
}
}
async function checkTableExists() {
try {
console.log('检查汇率数据表是否存在...');
const result = await pool.query('SHOW TABLES LIKE ?', ['econ_ExchangeRate']);
return result.length > 0;
} catch (error) {
console.error('检查表存在失败:', error.message);
return false;
}
}
async function createExchangeRateTable() {
try {
console.log('执行创建汇率数据表SQL...');
await pool.query(`
CREATE TABLE IF NOT EXISTS econ_ExchangeRate (
id INT AUTO_INCREMENT PRIMARY KEY,
date DATE UNIQUE NOT NULL,
currency VARCHAR(10) NOT NULL,
centerPrice DECIMAL(10,4) NOT NULL,
sellingRate DECIMAL(10,4) NOT NULL,
buyingRate DECIMAL(10,4) NOT NULL,
source VARCHAR(100) NOT NULL,
createdAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updatedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
`);
console.log('汇率数据表创建成功');
} catch (error) {
console.error('创建汇率数据表失败:', error.message);
throw error;
}
}
async function getExistingExchangeRates(startDate, endDate) {
try {
console.log('查询已有汇率数据...');
const result = await pool.query(
'SELECT date FROM econ_ExchangeRate WHERE date BETWEEN ? AND ? ORDER BY date',
[startDate, endDate]
);
const dates = result.map(row => {
if (row.date instanceof Date) {
return row.date.toISOString().split('T')[0];
} else if (typeof row.date === 'string') {
return row.date;
}
return null;
}).filter(Boolean);
console.log(`查询到 ${dates.length} 条数据`);
return dates;
} catch (error) {
console.error('查询已有汇率数据失败:', error.message);
return [];
}
}
function calculateMissingDates(startDate, endDate, existingDates) {
const existingSet = new Set(existingDates);
const missingDates = [];
const currentDate = new Date(startDate);
const finalDate = new Date(endDate);
while (currentDate <= finalDate) {
const dateStr = currentDate.toISOString().split('T')[0];
const dayOfWeek = currentDate.getDay();
// 跳过周末(周末没有汇率数据)
if (dayOfWeek !== 0 && dayOfWeek !== 6) {
if (!existingSet.has(dateStr)) {
missingDates.push(dateStr);
}
}
currentDate.setDate(currentDate.getDate() + 1);
}
return missingDates;
}
async function storeExchangeRates(rates) {
let storedCount = 0;
for (const rate of rates) {
try {
await pool.query(
'INSERT INTO econ_ExchangeRate (date, currency, centerPrice, sellingRate, buyingRate, source) VALUES (?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE centerPrice = VALUES(centerPrice), sellingRate = VALUES(sellingRate), buyingRate = VALUES(buyingRate), source = VALUES(source)',
[rate.date, rate.currency, rate.centerPrice, rate.sellingRate, rate.buyingRate, rate.source]
);
storedCount++;
} catch (error) {
console.error(`存储汇率数据 ${rate.date} 失败:`, error.message);
}
}
return storedCount;
}
export async function trigger(startDate = null, endDate = null) {
return await checkAndFillUSDCNYRate(startDate, endDate);
}
// 直接运行测试
async function runTest() {
console.log('='.repeat(60));
console.log('测试 usd-cny-rate.scraper.js');
console.log('='.repeat(60));
// 测试配置读取
console.log('\n测试数据库配置读取...');
const config = loadDatabaseConfig();
console.log('配置读取结果:');
console.log(` 主机: ${config.host}`);
console.log(` 端口: ${config.port}`);
console.log(` 用户: ${config.user}`);
console.log(` 数据库: ${config.database}`);
console.log('配置读取测试完成!');
try {
const result = await trigger();
console.log('\n' + '='.repeat(60));
console.log('测试结果:');
console.log(` 开始日期: ${result.startDate}`);
console.log(` 结束日期: ${result.endDate}`);
console.log(` 原有数据: ${result.existingCount}`);
console.log(` 缺失数据: ${result.missingCount}`);
console.log(` 最终数据: ${result.finalCount}`);
console.log(` 操作成功: ${result.success}`);
console.log('='.repeat(60));
} catch (error) {
console.error('测试失败:', error);
process.exit(1);
}
}
// 运行测试
runTest();

View File

@ -493,3 +493,154 @@ main {
margin-bottom: 1rem;
}
/* Time range selector */
.chart-header {
display: flex;
justify-content: flex-end;
margin-bottom: 10px;
}
.time-range-picker {
position: relative;
display: inline-block;
}
.time-range-btn {
padding: 8px 16px;
border: 1px solid #ddd;
border-radius: 4px;
background-color: #fff;
cursor: pointer;
font-size: 14px;
display: flex;
align-items: center;
gap: 8px;
min-width: 150px;
justify-content: space-between;
transition: all 0.3s ease;
box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05);
}
.time-range-btn:hover {
border-color: #333;
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
}
.dropdown-arrow {
font-size: 12px;
color: #666;
transition: transform 0.3s ease;
}
.time-range-dropdown {
position: absolute;
top: 100%;
right: 0;
margin-top: 8px;
background-color: #fff;
border: 1px solid #ddd;
border-radius: 4px;
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15);
padding: 16px;
min-width: 300px;
z-index: 1000;
}
.dropdown-section {
margin-bottom: 16px;
}
.dropdown-section:last-child {
margin-bottom: 0;
}
.dropdown-title {
font-size: 12px;
font-weight: 600;
color: #999;
margin-bottom: 8px;
text-transform: uppercase;
}
.dropdown-options {
list-style: none;
padding: 0;
margin: 0;
}
.dropdown-option {
padding: 8px 12px;
cursor: pointer;
border-radius: 4px;
transition: all 0.2s ease;
font-size: 14px;
}
.dropdown-option:hover {
background-color: #f0f0f0;
}
.dropdown-option.active {
background-color: #333;
color: #fff;
}
.custom-date-range {
display: flex;
align-items: center;
gap: 8px;
flex-wrap: wrap;
}
.custom-date-range input[type="date"] {
padding: 6px 8px;
border: 1px solid #ddd;
border-radius: 4px;
font-size: 14px;
min-width: 120px;
}
.custom-date-range span {
font-size: 14px;
color: #666;
white-space: nowrap;
}
.apply-btn {
padding: 6px 12px;
border: 1px solid #333;
border-radius: 4px;
background-color: #333;
color: #fff;
cursor: pointer;
font-size: 14px;
transition: all 0.3s ease;
}
.apply-btn:hover {
background-color: #555;
border-color: #555;
}
.apply-btn:disabled {
background-color: #f0f0f0;
color: #999;
border-color: #ddd;
cursor: not-allowed;
}
/* Loading and error states */
.loading {
text-align: center;
padding: 40px;
font-size: 16px;
color: #666;
}
.error {
text-align: center;
padding: 40px;
font-size: 16px;
color: #e53935;
}