Fullstack/iboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

第一个爬虫，但是感觉是不能用的。

Browse Source
This commit is contained in:
cheney 2026-04-17 20:37:44 +08:00
parent c61610db20
commit 2820adfdd3
14 changed files with 2846 additions and 1174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
generated/prisma/browser.ts
View File

@ -38,7 +38,12 @@ export type Strategy = Prisma.StrategyModel
*/
export type Indicator = Prisma.IndicatorModel
/**
* Model SowInventory
* Model Econ_SowInventory
*
*/
export type SowInventory = Prisma.SowInventoryModel
export type Econ_SowInventory = Prisma.Econ_SowInventoryModel
/**
* Model ScraperCache
*
*/
export type ScraperCache = Prisma.ScraperCacheModel

9
generated/prisma/client.ts
View File

@ -62,7 +62,12 @@ export type Strategy = Prisma.StrategyModel
*/
export type Indicator = Prisma.IndicatorModel
/**
* Model SowInventory
* Model Econ_SowInventory
*
*/
export type SowInventory = Prisma.SowInventoryModel
export type Econ_SowInventory = Prisma.Econ_SowInventoryModel
/**
* Model ScraperCache
*
*/
export type ScraperCache = Prisma.ScraperCacheModel

157
generated/prisma/commonInputTypes.ts
View File

@ -150,6 +150,95 @@ export type JsonWithAggregatesFilterBase<$PrismaModel = never> = {
_max?: Prisma.NestedJsonFilter<$PrismaModel>
}
export type StringNullableFilter<$PrismaModel = never> = {
equals?: string | Prisma.StringFieldRefInput<$PrismaModel> | null
in?: string[] | null
notIn?: string[] | null
lt?: string | Prisma.StringFieldRefInput<$PrismaModel>
lte?: string | Prisma.StringFieldRefInput<$PrismaModel>
gt?: string | Prisma.StringFieldRefInput<$PrismaModel>
gte?: string | Prisma.StringFieldRefInput<$PrismaModel>
contains?: string | Prisma.StringFieldRefInput<$PrismaModel>
startsWith?: string | Prisma.StringFieldRefInput<$PrismaModel>
endsWith?: string | Prisma.StringFieldRefInput<$PrismaModel>
search?: string
not?: Prisma.NestedStringNullableFilter<$PrismaModel> | string | null
}
export type JsonNullableFilter<$PrismaModel = never> =
| Prisma.PatchUndefined<
Prisma.Either<Required<JsonNullableFilterBase<$PrismaModel>>, Exclude<keyof Required<JsonNullableFilterBase<$PrismaModel>>, 'path'>>,
Required<JsonNullableFilterBase<$PrismaModel>>
>
| Prisma.OptionalFlat<Omit<Required<JsonNullableFilterBase<$PrismaModel>>, 'path'>>
export type JsonNullableFilterBase<$PrismaModel = never> = {
equals?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | Prisma.JsonNullValueFilter
path?: string
mode?: Prisma.QueryMode | Prisma.EnumQueryModeFieldRefInput<$PrismaModel>
string_contains?: string | Prisma.StringFieldRefInput<$PrismaModel>
string_starts_with?: string | Prisma.StringFieldRefInput<$PrismaModel>
string_ends_with?: string | Prisma.StringFieldRefInput<$PrismaModel>
array_starts_with?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | null
array_ends_with?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | null
array_contains?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | null
lt?: runtime.InputJsonValue
lte?: runtime.InputJsonValue
gt?: runtime.InputJsonValue
gte?: runtime.InputJsonValue
not?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | Prisma.JsonNullValueFilter
}
export type SortOrderInput = {
sort: Prisma.SortOrder
nulls?: Prisma.NullsOrder
}
export type StringNullableWithAggregatesFilter<$PrismaModel = never> = {
equals?: string | Prisma.StringFieldRefInput<$PrismaModel> | null
in?: string[] | null
notIn?: string[] | null
lt?: string | Prisma.StringFieldRefInput<$PrismaModel>
lte?: string | Prisma.StringFieldRefInput<$PrismaModel>
gt?: string | Prisma.StringFieldRefInput<$PrismaModel>
gte?: string | Prisma.StringFieldRefInput<$PrismaModel>
contains?: string | Prisma.StringFieldRefInput<$PrismaModel>
startsWith?: string | Prisma.StringFieldRefInput<$PrismaModel>
endsWith?: string | Prisma.StringFieldRefInput<$PrismaModel>
search?: string
not?: Prisma.NestedStringNullableWithAggregatesFilter<$PrismaModel> | string | null
_count?: Prisma.NestedIntNullableFilter<$PrismaModel>
_min?: Prisma.NestedStringNullableFilter<$PrismaModel>
_max?: Prisma.NestedStringNullableFilter<$PrismaModel>
}
export type JsonNullableWithAggregatesFilter<$PrismaModel = never> =
| Prisma.PatchUndefined<
Prisma.Either<Required<JsonNullableWithAggregatesFilterBase<$PrismaModel>>, Exclude<keyof Required<JsonNullableWithAggregatesFilterBase<$PrismaModel>>, 'path'>>,
Required<JsonNullableWithAggregatesFilterBase<$PrismaModel>>
>
| Prisma.OptionalFlat<Omit<Required<JsonNullableWithAggregatesFilterBase<$PrismaModel>>, 'path'>>
export type JsonNullableWithAggregatesFilterBase<$PrismaModel = never> = {
equals?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | Prisma.JsonNullValueFilter
path?: string
mode?: Prisma.QueryMode | Prisma.EnumQueryModeFieldRefInput<$PrismaModel>
string_contains?: string | Prisma.StringFieldRefInput<$PrismaModel>
string_starts_with?: string | Prisma.StringFieldRefInput<$PrismaModel>
string_ends_with?: string | Prisma.StringFieldRefInput<$PrismaModel>
array_starts_with?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | null
array_ends_with?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | null
array_contains?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | null
lt?: runtime.InputJsonValue
lte?: runtime.InputJsonValue
gt?: runtime.InputJsonValue
gte?: runtime.InputJsonValue
not?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | Prisma.JsonNullValueFilter
_count?: Prisma.NestedIntNullableFilter<$PrismaModel>
_min?: Prisma.NestedJsonNullableFilter<$PrismaModel>
_max?: Prisma.NestedJsonNullableFilter<$PrismaModel>
}
export type NestedIntFilter<$PrismaModel = never> = {
equals?: number | Prisma.IntFieldRefInput<$PrismaModel>
in?: number[]
@ -270,4 +359,72 @@ export type NestedJsonFilterBase<$PrismaModel = never> = {
not?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | Prisma.JsonNullValueFilter
}
export type NestedStringNullableFilter<$PrismaModel = never> = {
equals?: string | Prisma.StringFieldRefInput<$PrismaModel> | null
in?: string[] | null
notIn?: string[] | null
lt?: string | Prisma.StringFieldRefInput<$PrismaModel>
lte?: string | Prisma.StringFieldRefInput<$PrismaModel>
gt?: string | Prisma.StringFieldRefInput<$PrismaModel>
gte?: string | Prisma.StringFieldRefInput<$PrismaModel>
contains?: string | Prisma.StringFieldRefInput<$PrismaModel>
startsWith?: string | Prisma.StringFieldRefInput<$PrismaModel>
endsWith?: string | Prisma.StringFieldRefInput<$PrismaModel>
search?: string
not?: Prisma.NestedStringNullableFilter<$PrismaModel> | string | null
}
export type NestedStringNullableWithAggregatesFilter<$PrismaModel = never> = {
equals?: string | Prisma.StringFieldRefInput<$PrismaModel> | null
in?: string[] | null
notIn?: string[] | null
lt?: string | Prisma.StringFieldRefInput<$PrismaModel>
lte?: string | Prisma.StringFieldRefInput<$PrismaModel>
gt?: string | Prisma.StringFieldRefInput<$PrismaModel>
gte?: string | Prisma.StringFieldRefInput<$PrismaModel>
contains?: string | Prisma.StringFieldRefInput<$PrismaModel>
startsWith?: string | Prisma.StringFieldRefInput<$PrismaModel>
endsWith?: string | Prisma.StringFieldRefInput<$PrismaModel>
search?: string
not?: Prisma.NestedStringNullableWithAggregatesFilter<$PrismaModel> | string | null
_count?: Prisma.NestedIntNullableFilter<$PrismaModel>
_min?: Prisma.NestedStringNullableFilter<$PrismaModel>
_max?: Prisma.NestedStringNullableFilter<$PrismaModel>
}
export type NestedIntNullableFilter<$PrismaModel = never> = {
equals?: number | Prisma.IntFieldRefInput<$PrismaModel> | null
in?: number[] | null
notIn?: number[] | null
lt?: number | Prisma.IntFieldRefInput<$PrismaModel>
lte?: number | Prisma.IntFieldRefInput<$PrismaModel>
gt?: number | Prisma.IntFieldRefInput<$PrismaModel>
gte?: number | Prisma.IntFieldRefInput<$PrismaModel>
not?: Prisma.NestedIntNullableFilter<$PrismaModel> | number | null
}
export type NestedJsonNullableFilter<$PrismaModel = never> =
| Prisma.PatchUndefined<
Prisma.Either<Required<NestedJsonNullableFilterBase<$PrismaModel>>, Exclude<keyof Required<NestedJsonNullableFilterBase<$PrismaModel>>, 'path'>>,
Required<NestedJsonNullableFilterBase<$PrismaModel>>
>
| Prisma.OptionalFlat<Omit<Required<NestedJsonNullableFilterBase<$PrismaModel>>, 'path'>>
export type NestedJsonNullableFilterBase<$PrismaModel = never> = {
equals?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | Prisma.JsonNullValueFilter
path?: string
mode?: Prisma.QueryMode | Prisma.EnumQueryModeFieldRefInput<$PrismaModel>
string_contains?: string | Prisma.StringFieldRefInput<$PrismaModel>
string_starts_with?: string | Prisma.StringFieldRefInput<$PrismaModel>
string_ends_with?: string | Prisma.StringFieldRefInput<$PrismaModel>
array_starts_with?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | null
array_ends_with?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | null
array_contains?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | null
lt?: runtime.InputJsonValue
lte?: runtime.InputJsonValue
gt?: runtime.InputJsonValue
gte?: runtime.InputJsonValue
not?: runtime.InputJsonValue | Prisma.JsonFieldRefInput<$PrismaModel> | Prisma.JsonNullValueFilter
}

26
generated/prisma/internal/class.ts
View File

File diff suppressed because one or more lines are too long

179
generated/prisma/internal/prismaNamespace.ts
View File

@ -388,7 +388,8 @@ export const ModelName = {
Article: 'Article',
Strategy: 'Strategy',
Indicator: 'Indicator',
SowInventory: 'SowInventory'
Econ_SowInventory: 'Econ_SowInventory',
ScraperCache: 'ScraperCache'
} as const
export type ModelName = (typeof ModelName)[keyof typeof ModelName]
@ -404,7 +405,7 @@ export type TypeMap<ExtArgs extends runtime.Types.Extensions.InternalArgs = runt
omit: GlobalOmitOptions
}
meta: {
modelProps: "user" | "article" | "strategy" | "indicator" | "sowInventory"
modelProps: "user" | "article" | "strategy" | "indicator" | "econ_SowInventory" | "scraperCache"
txIsolationLevel: TransactionIsolationLevel
}
model: {
@ -672,69 +673,135 @@ export type TypeMap<ExtArgs extends runtime.Types.Extensions.InternalArgs = runt
}
}
}
SowInventory: {
payload: Prisma.$SowInventoryPayload<ExtArgs>
fields: Prisma.SowInventoryFieldRefs
Econ_SowInventory: {
payload: Prisma.$Econ_SowInventoryPayload<ExtArgs>
fields: Prisma.Econ_SowInventoryFieldRefs
operations: {
findUnique: {
args: Prisma.SowInventoryFindUniqueArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$SowInventoryPayload> | null
args: Prisma.Econ_SowInventoryFindUniqueArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$Econ_SowInventoryPayload> | null
}
findUniqueOrThrow: {
args: Prisma.SowInventoryFindUniqueOrThrowArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$SowInventoryPayload>
args: Prisma.Econ_SowInventoryFindUniqueOrThrowArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$Econ_SowInventoryPayload>
}
findFirst: {
args: Prisma.SowInventoryFindFirstArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$SowInventoryPayload> | null
args: Prisma.Econ_SowInventoryFindFirstArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$Econ_SowInventoryPayload> | null
}
findFirstOrThrow: {
args: Prisma.SowInventoryFindFirstOrThrowArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$SowInventoryPayload>
args: Prisma.Econ_SowInventoryFindFirstOrThrowArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$Econ_SowInventoryPayload>
}
findMany: {
args: Prisma.SowInventoryFindManyArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$SowInventoryPayload>[]
args: Prisma.Econ_SowInventoryFindManyArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$Econ_SowInventoryPayload>[]
}
create: {
args: Prisma.SowInventoryCreateArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$SowInventoryPayload>
args: Prisma.Econ_SowInventoryCreateArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$Econ_SowInventoryPayload>
}
createMany: {
args: Prisma.SowInventoryCreateManyArgs<ExtArgs>
args: Prisma.Econ_SowInventoryCreateManyArgs<ExtArgs>
result: BatchPayload
}
delete: {
args: Prisma.SowInventoryDeleteArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$SowInventoryPayload>
args: Prisma.Econ_SowInventoryDeleteArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$Econ_SowInventoryPayload>
}
update: {
args: Prisma.SowInventoryUpdateArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$SowInventoryPayload>
args: Prisma.Econ_SowInventoryUpdateArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$Econ_SowInventoryPayload>
}
deleteMany: {
args: Prisma.SowInventoryDeleteManyArgs<ExtArgs>
args: Prisma.Econ_SowInventoryDeleteManyArgs<ExtArgs>
result: BatchPayload
}
updateMany: {
args: Prisma.SowInventoryUpdateManyArgs<ExtArgs>
args: Prisma.Econ_SowInventoryUpdateManyArgs<ExtArgs>
result: BatchPayload
}
upsert: {
args: Prisma.SowInventoryUpsertArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$SowInventoryPayload>
args: Prisma.Econ_SowInventoryUpsertArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$Econ_SowInventoryPayload>
}
aggregate: {
args: Prisma.SowInventoryAggregateArgs<ExtArgs>
result: runtime.Types.Utils.Optional<Prisma.AggregateSowInventory>
args: Prisma.Econ_SowInventoryAggregateArgs<ExtArgs>
result: runtime.Types.Utils.Optional<Prisma.AggregateEcon_SowInventory>
}
groupBy: {
args: Prisma.SowInventoryGroupByArgs<ExtArgs>
result: runtime.Types.Utils.Optional<Prisma.SowInventoryGroupByOutputType>[]
args: Prisma.Econ_SowInventoryGroupByArgs<ExtArgs>
result: runtime.Types.Utils.Optional<Prisma.Econ_SowInventoryGroupByOutputType>[]
}
count: {
args: Prisma.SowInventoryCountArgs<ExtArgs>
result: runtime.Types.Utils.Optional<Prisma.SowInventoryCountAggregateOutputType> | number
args: Prisma.Econ_SowInventoryCountArgs<ExtArgs>
result: runtime.Types.Utils.Optional<Prisma.Econ_SowInventoryCountAggregateOutputType> | number
}
}
}
ScraperCache: {
payload: Prisma.$ScraperCachePayload<ExtArgs>
fields: Prisma.ScraperCacheFieldRefs
operations: {
findUnique: {
args: Prisma.ScraperCacheFindUniqueArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$ScraperCachePayload> | null
}
findUniqueOrThrow: {
args: Prisma.ScraperCacheFindUniqueOrThrowArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$ScraperCachePayload>
}
findFirst: {
args: Prisma.ScraperCacheFindFirstArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$ScraperCachePayload> | null
}
findFirstOrThrow: {
args: Prisma.ScraperCacheFindFirstOrThrowArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$ScraperCachePayload>
}
findMany: {
args: Prisma.ScraperCacheFindManyArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$ScraperCachePayload>[]
}
create: {
args: Prisma.ScraperCacheCreateArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$ScraperCachePayload>
}
createMany: {
args: Prisma.ScraperCacheCreateManyArgs<ExtArgs>
result: BatchPayload
}
delete: {
args: Prisma.ScraperCacheDeleteArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$ScraperCachePayload>
}
update: {
args: Prisma.ScraperCacheUpdateArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$ScraperCachePayload>
}
deleteMany: {
args: Prisma.ScraperCacheDeleteManyArgs<ExtArgs>
result: BatchPayload
}
updateMany: {
args: Prisma.ScraperCacheUpdateManyArgs<ExtArgs>
result: BatchPayload
}
upsert: {
args: Prisma.ScraperCacheUpsertArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$ScraperCachePayload>
}
aggregate: {
args: Prisma.ScraperCacheAggregateArgs<ExtArgs>
result: runtime.Types.Utils.Optional<Prisma.AggregateScraperCache>
}
groupBy: {
args: Prisma.ScraperCacheGroupByArgs<ExtArgs>
result: runtime.Types.Utils.Optional<Prisma.ScraperCacheGroupByOutputType>[]
}
count: {
args: Prisma.ScraperCacheCountArgs<ExtArgs>
result: runtime.Types.Utils.Optional<Prisma.ScraperCacheCountAggregateOutputType> | number
}
}
}
@ -826,7 +893,7 @@ export const IndicatorScalarFieldEnum = {
export type IndicatorScalarFieldEnum = (typeof IndicatorScalarFieldEnum)[keyof typeof IndicatorScalarFieldEnum]
export const SowInventoryScalarFieldEnum = {
export const Econ_SowInventoryScalarFieldEnum = {
id: 'id',
month: 'month',
inventory: 'inventory',
@ -834,7 +901,20 @@ export const SowInventoryScalarFieldEnum = {
updatedAt: 'updatedAt'
} as const
export type SowInventoryScalarFieldEnum = (typeof SowInventoryScalarFieldEnum)[keyof typeof SowInventoryScalarFieldEnum]
export type Econ_SowInventoryScalarFieldEnum = (typeof Econ_SowInventoryScalarFieldEnum)[keyof typeof Econ_SowInventoryScalarFieldEnum]
export const ScraperCacheScalarFieldEnum = {
id: 'id',
scraperName: 'scraperName',
lastDataHash: 'lastDataHash',
lastUpdateAt: 'lastUpdateAt',
metadata: 'metadata',
createdAt: 'createdAt',
updatedAt: 'updatedAt'
} as const
export type ScraperCacheScalarFieldEnum = (typeof ScraperCacheScalarFieldEnum)[keyof typeof ScraperCacheScalarFieldEnum]
export const SortOrder = {
@ -852,6 +932,14 @@ export const JsonNullValueInput = {
export type JsonNullValueInput = (typeof JsonNullValueInput)[keyof typeof JsonNullValueInput]
export const NullableJsonNullValueInput = {
DbNull: DbNull,
JsonNull: JsonNull
} as const
export type NullableJsonNullValueInput = (typeof NullableJsonNullValueInput)[keyof typeof NullableJsonNullValueInput]
export const UserOrderByRelevanceFieldEnum = {
name: 'name',
email: 'email',
@ -904,11 +992,27 @@ export const IndicatorOrderByRelevanceFieldEnum = {
export type IndicatorOrderByRelevanceFieldEnum = (typeof IndicatorOrderByRelevanceFieldEnum)[keyof typeof IndicatorOrderByRelevanceFieldEnum]
export const SowInventoryOrderByRelevanceFieldEnum = {
export const Econ_SowInventoryOrderByRelevanceFieldEnum = {
month: 'month'
} as const
export type SowInventoryOrderByRelevanceFieldEnum = (typeof SowInventoryOrderByRelevanceFieldEnum)[keyof typeof SowInventoryOrderByRelevanceFieldEnum]
export type Econ_SowInventoryOrderByRelevanceFieldEnum = (typeof Econ_SowInventoryOrderByRelevanceFieldEnum)[keyof typeof Econ_SowInventoryOrderByRelevanceFieldEnum]
export const NullsOrder = {
first: 'first',
last: 'last'
} as const
export type NullsOrder = (typeof NullsOrder)[keyof typeof NullsOrder]
export const ScraperCacheOrderByRelevanceFieldEnum = {
scraperName: 'scraperName',
lastDataHash: 'lastDataHash'
} as const
export type ScraperCacheOrderByRelevanceFieldEnum = (typeof ScraperCacheOrderByRelevanceFieldEnum)[keyof typeof ScraperCacheOrderByRelevanceFieldEnum]
@ -1057,7 +1161,8 @@ export type GlobalOmitConfig = {
article?: Prisma.ArticleOmit
strategy?: Prisma.StrategyOmit
indicator?: Prisma.IndicatorOmit
sowInventory?: Prisma.SowInventoryOmit
econ_SowInventory?: Prisma.Econ_SowInventoryOmit
scraperCache?: Prisma.ScraperCacheOmit
}
/* Types for Logging */

48
generated/prisma/internal/prismaNamespaceBrowser.ts
View File

@ -55,7 +55,8 @@ export const ModelName = {
Article: 'Article',
Strategy: 'Strategy',
Indicator: 'Indicator',
SowInventory: 'SowInventory'
Econ_SowInventory: 'Econ_SowInventory',
ScraperCache: 'ScraperCache'
} as const
export type ModelName = (typeof ModelName)[keyof typeof ModelName]
@ -123,7 +124,7 @@ export const IndicatorScalarFieldEnum = {
export type IndicatorScalarFieldEnum = (typeof IndicatorScalarFieldEnum)[keyof typeof IndicatorScalarFieldEnum]
export const SowInventoryScalarFieldEnum = {
export const Econ_SowInventoryScalarFieldEnum = {
id: 'id',
month: 'month',
inventory: 'inventory',
@ -131,7 +132,20 @@ export const SowInventoryScalarFieldEnum = {
updatedAt: 'updatedAt'
} as const
export type SowInventoryScalarFieldEnum = (typeof SowInventoryScalarFieldEnum)[keyof typeof SowInventoryScalarFieldEnum]
export type Econ_SowInventoryScalarFieldEnum = (typeof Econ_SowInventoryScalarFieldEnum)[keyof typeof Econ_SowInventoryScalarFieldEnum]
export const ScraperCacheScalarFieldEnum = {
id: 'id',
scraperName: 'scraperName',
lastDataHash: 'lastDataHash',
lastUpdateAt: 'lastUpdateAt',
metadata: 'metadata',
createdAt: 'createdAt',
updatedAt: 'updatedAt'
} as const
export type ScraperCacheScalarFieldEnum = (typeof ScraperCacheScalarFieldEnum)[keyof typeof ScraperCacheScalarFieldEnum]
export const SortOrder = {
@ -149,6 +163,14 @@ export const JsonNullValueInput = {
export type JsonNullValueInput = (typeof JsonNullValueInput)[keyof typeof JsonNullValueInput]
export const NullableJsonNullValueInput = {
DbNull: DbNull,
JsonNull: JsonNull
} as const
export type NullableJsonNullValueInput = (typeof NullableJsonNullValueInput)[keyof typeof NullableJsonNullValueInput]
export const UserOrderByRelevanceFieldEnum = {
name: 'name',
email: 'email',
@ -201,9 +223,25 @@ export const IndicatorOrderByRelevanceFieldEnum = {
export type IndicatorOrderByRelevanceFieldEnum = (typeof IndicatorOrderByRelevanceFieldEnum)[keyof typeof IndicatorOrderByRelevanceFieldEnum]
export const SowInventoryOrderByRelevanceFieldEnum = {
export const Econ_SowInventoryOrderByRelevanceFieldEnum = {
month: 'month'
} as const
export type SowInventoryOrderByRelevanceFieldEnum = (typeof SowInventoryOrderByRelevanceFieldEnum)[keyof typeof SowInventoryOrderByRelevanceFieldEnum]
export type Econ_SowInventoryOrderByRelevanceFieldEnum = (typeof Econ_SowInventoryOrderByRelevanceFieldEnum)[keyof typeof Econ_SowInventoryOrderByRelevanceFieldEnum]
export const NullsOrder = {
first: 'first',
last: 'last'
} as const
export type NullsOrder = (typeof NullsOrder)[keyof typeof NullsOrder]
export const ScraperCacheOrderByRelevanceFieldEnum = {
scraperName: 'scraperName',
lastDataHash: 'lastDataHash'
} as const
export type ScraperCacheOrderByRelevanceFieldEnum = (typeof ScraperCacheOrderByRelevanceFieldEnum)[keyof typeof ScraperCacheOrderByRelevanceFieldEnum]

3
generated/prisma/models.ts
View File

@ -12,5 +12,6 @@ export type * from './models/User.ts'
export type * from './models/Article.ts'
export type * from './models/Strategy.ts'
export type * from './models/Indicator.ts'
export type * from './models/SowInventory.ts'
export type * from './models/Econ_SowInventory.ts'
export type * from './models/ScraperCache.ts'
export type * from './commonInputTypes.ts'

1090
generated/prisma/models/Econ_SowInventory.ts Normal file
View File

File diff suppressed because it is too large Load Diff

1134
generated/prisma/models/ScraperCache.ts Normal file
View File

File diff suppressed because it is too large Load Diff

1090
generated/prisma/models/SowInventory.ts
View File

File diff suppressed because it is too large Load Diff

13
prisma/migrations/20260417122901_add_scraper_cache/migration.sql Normal file
View File

@ -0,0 +1,13 @@
-- CreateTable
CREATE TABLE `ScraperCache` (
`id` INTEGER NOT NULL AUTO_INCREMENT,
`scraperName` VARCHAR(191) NOT NULL,
`lastDataHash` VARCHAR(191) NULL,
`lastUpdateAt` DATETIME(3) NOT NULL,
`metadata` JSON NULL,
`createdAt` DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
`updatedAt` DATETIME(3) NOT NULL,
UNIQUE INDEX `ScraperCache_scraperName_key`(`scraperName`),
PRIMARY KEY (`id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;

10
prisma/schema.prisma
View File

@ -56,3 +56,13 @@ model Econ_SowInventory {
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
}
model ScraperCache {
id Int @id @default(autoincrement())
scraperName String @unique // 爬虫名称，如：sow-inventory
lastDataHash String? // 上次数据指纹
lastUpdateAt DateTime // 上次更新时间
metadata Json? // 其他元数据
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
}

3
scripts/scraper/code.md Normal file
View File

@ -0,0 +1,3 @@
git:<http://git.honor3.com/Fullstack/iboard>
dragon:D:\workbench\iboard

221
scripts/scraper/sow-inventory-scraper.js
View File

@ -1,7 +1,5 @@
const axios = require('axios');
const cheerio = require('cheerio');
const fs = require('fs');
const path = require('path');
const { PrismaClient } = require('../../generated/prisma');
const prisma = new PrismaClient();
@ -9,12 +7,123 @@ const prisma = new PrismaClient();
// 浙江数据开放网站URL（示例，实际需要根据真实网站修改）
const ZHEJIANG_OPEN_DATA_URL = 'https://data.zj.gov.cn/';
// 能繁母猪数量数据爬取函数
async function scrapeSowInventory() {
// 爬虫名称，用于缓存表中的标识
const SCRAPER_NAME = 'sow-inventory';
// 最小更新间隔（毫秒），默认30分钟
const MIN_UPDATE_INTERVAL = 30 * 60 * 1000;
// 从数据库读取缓存
async function readCache() {
try {
const cache = await prisma.scraperCache.findUnique({
where: { scraperName: SCRAPER_NAME }
});
return cache || { scraperName: SCRAPER_NAME, lastDataHash: null, lastUpdateAt: null };
} catch (error) {
console.error('读取数据库缓存失败:', error.message);
return { scraperName: SCRAPER_NAME, lastDataHash: null, lastUpdateAt: null };
}
}
// 写入数据库缓存
async function writeCache(lastDataHash) {
try {
await prisma.scraperCache.upsert({
where: { scraperName: SCRAPER_NAME },
update: {
lastDataHash,
lastUpdateAt: new Date()
},
create: {
scraperName: SCRAPER_NAME,
lastDataHash,
lastUpdateAt: new Date()
}
});
} catch (error) {
console.error('写入数据库缓存失败:', error.message);
}
}
// 计算数据hash，用于检测数据是否变化
function computeDataHash(data) {
const str = JSON.stringify(data);
let hash = 0;
for (let i = 0; i < str.length; i++) {
const char = str.charCodeAt(i);
hash = ((hash << 5) - hash) + char;
hash = hash & hash;
}
return hash.toString();
}
// 获取数据库中最新的月份
async function getLatestMonth() {
try {
const latest = await prisma.econ_SowInventory.findFirst({
orderBy: {
month: 'desc'
}
});
return latest ? latest.month : null;
} catch (error) {
console.error('查询最新月份失败:', error.message);
return null;
}
}
// 判断月份是否需要更新（只有比数据库中更新的月份才需要处理）
function shouldUpdateMonth(month, latestDbMonth) {
if (!latestDbMonth) return true;
return month > latestDbMonth;
}
// 检查是否需要发起HTTP请求（基于更新间隔）
function shouldSkipRequest(cache) {
if (!cache.lastUpdateAt) return false;
const now = new Date();
const lastUpdate = new Date(cache.lastUpdateAt);
const interval = now - lastUpdate;
return interval < MIN_UPDATE_INTERVAL;
}
// 能繁母猪数量数据爬取函数（增量更新版本）
async function scrapeSowInventory(incremental = true) {
try {
console.log('='.repeat(50));
console.log('开始爬取浙江省能繁母猪数量数据...');
console.log(`增量更新模式: ${incremental ? '开启' : '关闭'}`);
console.log('='.repeat(50));
// 0. 检查是否需要发起请求
if (incremental) {
const cache = await readCache();
if (shouldSkipRequest(cache)) {
const lastUpdate = cache.lastUpdateAt ? new Date(cache.lastUpdateAt) : null;
const hoursDiff = lastUpdate ? ((new Date() - lastUpdate) / (1000 * 60 * 60)).toFixed(2) : '未知';
console.log('\n' + '='.repeat(50));
console.log('跳过请求：距离上次更新不足30分钟');
console.log(`上次更新时间: ${cache.lastUpdateAt}`);
console.log(`距离现在: ${hoursDiff} 小时`);
console.log('='.repeat(50));
return;
}
// 获取数据库中最新的月份
const latestDbMonth = await getLatestMonth();
if (latestDbMonth) {
console.log(`数据库中最新月份: ${latestDbMonth}`);
} else {
console.log('数据库为空，将获取所有数据');
}
}
// 1. 发送HTTP请求获取页面内容
console.log('\n正在请求数据源...');
const response = await axios.get(ZHEJIANG_OPEN_DATA_URL, {
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
@ -25,8 +134,7 @@ async function scrapeSowInventory() {
const $ = cheerio.load(response.data);
// 3. 提取数据（这里需要根据实际网站结构修改选择器）
// 示例：假设数据在表格中
const data = [];
const allData = [];
$('table').each((tableIndex, table) => {
$(table).find('tr').each((rowIndex, row) => {
@ -37,7 +145,7 @@ async function scrapeSowInventory() {
const inventory = parseInt($(cells[1]).text().trim());
if (month && !isNaN(inventory)) {
data.push({
allData.push({
month,
inventory
});
@ -47,10 +155,58 @@ async function scrapeSowInventory() {
});
});
console.log(`爬取到 ${data.length} 条数据`);
console.log(`页面总数据量: ${allData.length} 条`);
// 4. 存储数据到数据库
for (const item of data) {
// 4. 读取数据库缓存
const cache = await readCache();
// 5. 计算当前数据hash，检测数据是否变化
const currentDataHash = computeDataHash(allData);
// 6. 如果是增量更新且数据未变化，跳过更新
if (incremental && cache.lastDataHash === currentDataHash) {
console.log('\n' + '='.repeat(50));
console.log('数据未发生变化，跳过数据库更新！');
console.log(`上次更新时间: ${cache.lastUpdateAt || '未知'}`);
console.log('='.repeat(50));
return;
}
// 7. 如果是增量更新，获取数据库中最新的月份用于过滤
let latestDbMonth = null;
if (incremental) {
latestDbMonth = await getLatestMonth();
}
// 8. 过滤需要更新的数据
let dataToUpdate = allData;
let skippedCount = 0;
if (incremental && latestDbMonth) {
dataToUpdate = allData.filter(item => shouldUpdateMonth(item.month, latestDbMonth));
skippedCount = allData.length - dataToUpdate.length;
console.log(`\n增量更新: 跳过 ${skippedCount} 条已存在的数据`);
console.log(`需要处理: ${dataToUpdate.length} 条新数据`);
}
// 9. 如果没有需要更新的数据
if (dataToUpdate.length === 0) {
console.log('\n没有需要更新的数据！');
return;
}
// 10. 按月份排序（从旧到新）
dataToUpdate.sort((a, b) => a.month.localeCompare(b.month));
console.log('\n开始处理数据...');
console.log('-'.repeat(50));
let newCount = 0;
let updateCount = 0;
let errorCount = 0;
// 11. 存储数据到数据库
for (const item of dataToUpdate) {
try {
// 检查数据是否已存在
const existing = await prisma.econ_SowInventory.findUnique({
@ -58,38 +214,73 @@ async function scrapeSowInventory() {
});
if (existing) {
// 更新现有数据
// 检查数据是否真的需要更新（值变化了才更新）
if (existing.inventory !== item.inventory) {
await prisma.econ_SowInventory.update({
where: { month: item.month },
data: { inventory: item.inventory }
});
console.log(`更新数据: ${item.month} - ${item.inventory}`);
updateCount++;
console.log(`更新数据: ${item.month} - ${item.inventory} (原值: ${existing.inventory})`);
} else {
console.log(`数据未变化，跳过: ${item.month}`);
}
} else {
// 创建新数据
await prisma.econ_SowInventory.create({
data: item
});
newCount++;
console.log(`新增数据: ${item.month} - ${item.inventory}`);
}
} catch (error) {
errorCount++;
console.error(`处理数据 ${item.month} 时出错:`, error.message);
}
}
// 12. 更新数据库缓存
await writeCache(currentDataHash);
// 13. 输出统计信息
const now = new Date().toISOString();
console.log('\n' + '='.repeat(50));
console.log('数据爬取和存储完成！');
console.log('统计信息:');
console.log(` - 新增数据: ${newCount} 条`);
console.log(` - 更新数据: ${updateCount} 条`);
console.log(` - 跳过数据: ${skippedCount} 条`);
console.log(` - 错误数量: ${errorCount} 条`);
console.log(` - 本次更新时间: ${now}`);
console.log('='.repeat(50));
} catch (error) {
console.error('爬取数据时出错:', error.message);
throw error;
} finally {
// 关闭Prisma连接
await prisma.$disconnect();
}
}
// 强制全量更新函数
async function scrapeSowInventoryFull() {
console.log('警告: 即将执行全量更新，这将更新所有数据！');
console.log('如果只是想获取新数据，请使用 scrapeSowInventory() 函数。\n');
await scrapeSowInventory(false);
}
// 导出函数
module.exports = { scrapeSowInventory };
module.exports = {
scrapeSowInventory,
scrapeSowInventoryFull
};
// 如果直接运行此文件
if (require.main === module) {
scrapeSowInventory();
// 默认执行增量更新
// 如果需要强制全量更新，使用: scrapeSowInventoryFull()
scrapeSowInventory().catch(error => {
console.error('执行失败:', error);
process.exit(1);
});
}