feat: 引入 so-novel submodule 并提供非交互下载工具
- vendor/so-novel: 上游 v1.12.0 (规则驱动的书源抓取/下载引擎) - tools/src: ToolMain 封装 search/download/toc/sources，stdout 只输出 JSON - tools/build.ps1: 自动定位/下载 JDK21，按平台显式激活 javet 原生库 profile - bin/novel.ps1|.cmd: 命令行入口（人读输出或 -Json） - SKILL.md/README.md: 使用说明与下载原理分析
This commit is contained in:
cheney
+114
@@ -0,0 +1,114 @@
|
||||
# Builds the so-novel engine jar (from the submodule) and the tool wrapper classes.
|
||||
# Run once after clone / submodule update: pwsh tools\build.ps1
|
||||
param(
|
||||
[switch]$Force, # rebuild so-novel even if runtime\app.jar exists
|
||||
[switch]$RefreshRules # re-copy bundle\rules over runtime\rules
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
. (Join-Path $PSScriptRoot 'lib.ps1')
|
||||
|
||||
$p = Get-ToolPaths
|
||||
Write-Host "[build] project root : $($p.Root)"
|
||||
|
||||
# ---------------------------------------------------------------- 1. JDK 21
|
||||
$jdk = Find-Jdk21 -AllowDownload
|
||||
if (-not $jdk) { throw 'No JDK 21 available and download failed. Set EBOOK_NOVEL_JAVA_HOME.' }
|
||||
Write-Host "[build] JDK 21 : $jdk"
|
||||
$javac = Join-Path $jdk 'bin\javac.exe'
|
||||
if (-not (Test-Path $javac)) { throw "javac not found in $jdk (a JRE is not enough)" }
|
||||
|
||||
# ---------------------------------------------------------------- 2. so-novel app.jar
|
||||
New-Item -ItemType Directory -Force $p.Runtime | Out-Null
|
||||
if ($Force -or -not (Test-Path $p.AppJar)) {
|
||||
if (-not (Test-Path (Join-Path $p.Vendor 'pom.xml'))) {
|
||||
throw 'vendor/so-novel is empty. Run: git submodule update --init --recursive'
|
||||
}
|
||||
$mvn = Get-Command mvn -ErrorAction SilentlyContinue
|
||||
if (-not $mvn) { throw 'Maven (mvn) is required to build so-novel. Install Maven or use a prebuilt app.jar.' }
|
||||
|
||||
Write-Host '[build] mvn package so-novel ...' -ForegroundColor Cyan
|
||||
# so-novel 用 <os><arch>x86_64</arch> 激活 javet 原生库 profile，而 Windows JDK 报的是 amd64，
|
||||
# 自动激活会失效（导致 JS 规则的书源全部 "Javet library ... not found"），这里显式指定。
|
||||
$platformProfile = switch -Regex ("$([System.Runtime.InteropServices.RuntimeInformation]::OSDescription)|$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture)") {
|
||||
'Windows.*X64' { 'windows-x64' }
|
||||
'Linux.*X64' { 'linux-x64' }
|
||||
'Linux.*Arm64' { 'linux-arm64' }
|
||||
'OSX|Darwin.*X64' { 'macos-x64' }
|
||||
'OSX|Darwin.*Arm64' { 'macos-arm64' }
|
||||
default { $null }
|
||||
}
|
||||
if ($platformProfile) {
|
||||
Write-Host "[build] javet profile: $platformProfile" -ForegroundColor Cyan
|
||||
} else {
|
||||
Write-Warning 'Unknown platform, javet native library may be missing (JS-based rules will fail).'
|
||||
}
|
||||
$previousJavaHome = $env:JAVA_HOME
|
||||
$previousPath = $env:PATH
|
||||
try {
|
||||
$env:JAVA_HOME = $jdk
|
||||
$env:PATH = (Join-Path $jdk 'bin') + ';' + $previousPath
|
||||
Push-Location $p.Vendor
|
||||
try {
|
||||
$mvnArgs = @('-B', '-DskipTests', '-Dlaunch4j.skip=true')
|
||||
if ($platformProfile) { $mvnArgs += @('-P', $platformProfile) }
|
||||
$mvnArgs += @('clean', 'package')
|
||||
& $mvn.Source @mvnArgs
|
||||
if ($LASTEXITCODE -ne 0) { throw "mvn package failed with exit code $LASTEXITCODE" }
|
||||
} finally {
|
||||
Pop-Location
|
||||
}
|
||||
} finally {
|
||||
$env:JAVA_HOME = $previousJavaHome
|
||||
$env:PATH = $previousPath
|
||||
}
|
||||
|
||||
$built = Join-Path $p.Vendor 'target\app-jar-with-dependencies.jar'
|
||||
if (-not (Test-Path $built)) { $built = Join-Path $p.Vendor 'target\app.jar' }
|
||||
if (-not (Test-Path $built)) { throw 'so-novel build produced no jar in target/' }
|
||||
Copy-Item $built $p.AppJar -Force
|
||||
Write-Host "[build] app.jar : $($p.AppJar) ($([math]::Round((Get-Item $p.AppJar).Length / 1MB, 1)) MB)"
|
||||
} else {
|
||||
Write-Host "[build] app.jar : up to date ($([math]::Round((Get-Item $p.AppJar).Length / 1MB, 1)) MB)"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------- 3. rule bundles
|
||||
New-Item -ItemType Directory -Force $p.Rules | Out-Null
|
||||
$sourceRules = Join-Path $p.Vendor 'bundle\rules'
|
||||
foreach ($file in (Get-ChildItem $sourceRules -Filter '*.json*' -File)) {
|
||||
$target = Join-Path $p.Rules $file.Name
|
||||
if ($RefreshRules -or -not (Test-Path $target)) {
|
||||
Copy-Item $file.FullName $target -Force
|
||||
}
|
||||
}
|
||||
Write-Host "[build] rules : $((Get-ChildItem $p.Rules -File).Count) files in $($p.Rules)"
|
||||
|
||||
# ---------------------------------------------------------------- 4. tool classes
|
||||
if (Test-Path $p.Classes) { Remove-Item $p.Classes -Recurse -Force }
|
||||
New-Item -ItemType Directory -Force $p.Classes | Out-Null
|
||||
$sources = @(Get-ChildItem (Join-Path $p.Tools 'src') -Filter '*.java' -Recurse | ForEach-Object { $_.FullName })
|
||||
Write-Host '[build] javac : tool wrapper ...' -ForegroundColor Cyan
|
||||
& $javac '-encoding' 'UTF-8' '--release' '21' '-cp' $p.AppJar '-d' $p.Classes @sources
|
||||
if ($LASTEXITCODE -ne 0) { throw "javac failed with exit code $LASTEXITCODE" }
|
||||
|
||||
# Optional single-file tool jar (a plain classes dir works too; JBR has no jar.exe).
|
||||
$jarTool = Find-JarTool
|
||||
if ($jarTool) {
|
||||
& $jarTool '--create' '--file' $p.ToolJar '-C' $p.Classes '.'
|
||||
if ($LASTEXITCODE -ne 0) { Write-Warning 'jar packaging failed; the classes directory will be used instead.' }
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------- 5. runtime config
|
||||
New-Item -ItemType Directory -Force $p.Books | Out-Null
|
||||
if ($Force -or -not (Test-Path $p.Config)) {
|
||||
$template = Get-Content (Join-Path $p.Tools 'config.ini.template') -Raw
|
||||
$ini = $template.Replace('@BOOKS@', ($p.Books -replace '\\', '/'))
|
||||
$ini = $ini.Replace('@RULES@', ((Join-Path $p.Rules 'main.json') -replace '\\', '/'))
|
||||
[System.IO.File]::WriteAllText($p.Config, $ini, (New-Object System.Text.UTF8Encoding($false)))
|
||||
Write-Host "[build] config.ini : $($p.Config)"
|
||||
} else {
|
||||
Write-Host "[build] config.ini : kept existing ($($p.Config))"
|
||||
}
|
||||
|
||||
Write-Host ''
|
||||
Write-Host 'Build finished. Try: pwsh bin\novel.ps1 sources' -ForegroundColor Green
|
||||
@@ -0,0 +1,43 @@
|
||||
[global]
|
||||
# 关闭 GitHub 自动更新检查（内网/无外网环境必须为 0）
|
||||
auto-update = 0
|
||||
gh-proxy =
|
||||
cf-bypass =
|
||||
|
||||
[download]
|
||||
# 默认落盘目录，由 tools/build.ps1 按仓库位置生成
|
||||
download-path = @BOOKS@
|
||||
extname = txt
|
||||
txt-encoding = UTF-8
|
||||
preserve-chapter-cache = 0
|
||||
# 工具模式不需要终端进度条，日志统一走 stdout 捕获
|
||||
enable-progressbar = 0
|
||||
|
||||
[source]
|
||||
language = zh-CN
|
||||
# 书源规则绝对路径，避免依赖运行目录
|
||||
active-rules = @RULES@
|
||||
source-id =
|
||||
search-limit = 30
|
||||
search-filter = 1
|
||||
|
||||
[crawl]
|
||||
concurrency =
|
||||
min-interval = 200
|
||||
max-interval = 400
|
||||
enable-retry = 1
|
||||
max-retries = 3
|
||||
retry-min-interval = 2000
|
||||
retry-max-interval = 4000
|
||||
|
||||
[web]
|
||||
enabled = 0
|
||||
port = 7765
|
||||
|
||||
[cookie]
|
||||
qidian =
|
||||
|
||||
[proxy]
|
||||
enabled = 0
|
||||
host = 127.0.0.1
|
||||
port = 7890
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
# Shared helpers for the ebook-skill command line (Windows PowerShell 5.1+ / pwsh 7+).
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
function Get-ProjectRoot {
|
||||
param([string]$Start = $PSScriptRoot)
|
||||
$dir = (Resolve-Path $Start).Path
|
||||
while ($dir) {
|
||||
if (Test-Path (Join-Path $dir 'vendor\so-novel\pom.xml')) { return $dir }
|
||||
$parent = Split-Path $dir -Parent
|
||||
if ($parent -eq $dir -or -not $parent) { break }
|
||||
$dir = $parent
|
||||
}
|
||||
throw "Cannot locate project root (vendor/so-novel/pom.xml not found above $Start). Run: git submodule update --init"
|
||||
}
|
||||
|
||||
function Get-ToolPaths {
|
||||
param([string]$Root = (Get-ProjectRoot))
|
||||
[pscustomobject]@{
|
||||
Root = $Root
|
||||
Vendor = Join-Path $Root 'vendor\so-novel'
|
||||
Tools = Join-Path $Root 'tools'
|
||||
Runtime = Join-Path $Root 'tools\runtime'
|
||||
Build = Join-Path $Root 'tools\build'
|
||||
Classes = Join-Path $Root 'tools\build\classes'
|
||||
AppJar = Join-Path $Root 'tools\runtime\app.jar'
|
||||
ToolJar = Join-Path $Root 'tools\runtime\sonovel-tool.jar'
|
||||
Config = Join-Path $Root 'tools\runtime\config.ini'
|
||||
Rules = Join-Path $Root 'tools\runtime\rules'
|
||||
Books = Join-Path $Root 'books'
|
||||
}
|
||||
}
|
||||
|
||||
function Test-Jdk21 {
|
||||
param([string]$JdkHome)
|
||||
if (-not $JdkHome) { return $false }
|
||||
$java = Join-Path $JdkHome 'bin\java.exe'
|
||||
if (-not (Test-Path $java)) { return $false }
|
||||
try {
|
||||
$out = (& $java -version 2>&1 | Out-String)
|
||||
} catch {
|
||||
return $false
|
||||
}
|
||||
if ($out -match 'version "(?:1\.)?(\d+)') {
|
||||
return ([int]$Matches[1] -ge 21)
|
||||
}
|
||||
return $false
|
||||
}
|
||||
|
||||
function Find-Jdk21 {
|
||||
param([switch]$AllowDownload)
|
||||
|
||||
$candidates = New-Object System.Collections.Generic.List[string]
|
||||
if ($env:EBOOK_NOVEL_JAVA_HOME) { $candidates.Add($env:EBOOK_NOVEL_JAVA_HOME) }
|
||||
if ($env:JAVA_HOME) { $candidates.Add($env:JAVA_HOME) }
|
||||
|
||||
$patterns = @(
|
||||
(Join-Path $PSScriptRoot '.jdk\*\*'),
|
||||
(Join-Path $PSScriptRoot '.jdk'),
|
||||
'C:\Program Files\Java\jdk-21*',
|
||||
'C:\Program Files\Eclipse Adoptium\jdk-21*',
|
||||
'C:\Program Files\JetBrains\*\jbr',
|
||||
(Join-Path $env:LOCALAPPDATA 'mise\installs\java\21*'),
|
||||
(Join-Path $env:LOCALAPPDATA 'mise\installs\java\2*')
|
||||
)
|
||||
foreach ($pattern in $patterns) {
|
||||
foreach ($dir in (Get-ChildItem -Path $pattern -Directory -ErrorAction SilentlyContinue)) {
|
||||
$candidates.Add($dir.FullName)
|
||||
}
|
||||
}
|
||||
|
||||
foreach ($candidate in $candidates) {
|
||||
if (Test-Jdk21 -JdkHome $candidate) { return (Resolve-Path $candidate).Path }
|
||||
}
|
||||
|
||||
if ($AllowDownload) { return (Install-Jdk21) }
|
||||
return $null
|
||||
}
|
||||
|
||||
# Downloads a portable Temurin JDK 21 (used only when no local JDK 21 exists).
|
||||
function Install-Jdk21 {
|
||||
$dest = Join-Path $PSScriptRoot '.jdk'
|
||||
New-Item -ItemType Directory -Force $dest | Out-Null
|
||||
|
||||
Write-Host '[setup] no JDK 21 found, downloading Temurin 21 from the Tsinghua mirror ...' -ForegroundColor Yellow
|
||||
$mirror = 'https://mirrors.tuna.tsinghua.edu.cn/Adoptium/21/jdk/x64/windows/'
|
||||
$listing = (Invoke-WebRequest -Uri $mirror -UseBasicParsing -TimeoutSec 60).Content
|
||||
$names = [regex]::Matches($listing, 'OpenJDK21U-jdk_x64_windows_hotspot_[0-9._]+\.zip') |
|
||||
ForEach-Object { $_.Value } | Sort-Object -Unique
|
||||
if (-not $names) { throw "Could not find a JDK 21 archive at $mirror" }
|
||||
$zipName = $names[$names.Count - 1]
|
||||
|
||||
$zip = Join-Path $env:TEMP $zipName
|
||||
if (-not (Test-Path $zip)) {
|
||||
Invoke-WebRequest -Uri ($mirror + $zipName) -OutFile $zip -UseBasicParsing -TimeoutSec 1800
|
||||
}
|
||||
Expand-Archive -Path $zip -DestinationPath $dest -Force
|
||||
|
||||
$java = Get-ChildItem -Path (Join-Path $dest '*\bin\java.exe') -ErrorAction SilentlyContinue | Select-Object -First 1
|
||||
if (-not $java) { throw "JDK extraction failed under $dest" }
|
||||
$JdkHome = Split-Path (Split-Path $java.FullName -Parent) -Parent
|
||||
Write-Host "[setup] JDK 21 installed at $JdkHome" -ForegroundColor Green
|
||||
return $JdkHome
|
||||
}
|
||||
|
||||
function Get-JavaCommand {
|
||||
param([string]$Root = (Get-ProjectRoot))
|
||||
$jdkHome = Find-Jdk21
|
||||
if (-not $jdkHome) {
|
||||
throw 'JDK 21 is required. Run tools\build.ps1 (it can download one), or set EBOOK_NOVEL_JAVA_HOME.'
|
||||
}
|
||||
return (Join-Path $jdkHome 'bin\java.exe')
|
||||
}
|
||||
|
||||
function Find-JarTool {
|
||||
# JBR ships without jar.exe, so fall back to any other JDK on the machine.
|
||||
foreach ($pattern in @('C:\Program Files\Java\*\bin\jar.exe',
|
||||
(Join-Path $env:LOCALAPPDATA 'mise\installs\java\*\bin\jar.exe'),
|
||||
'C:\Program Files\Eclipse Adoptium\*\bin\jar.exe')) {
|
||||
$exe = Get-ChildItem -Path $pattern -ErrorAction SilentlyContinue | Select-Object -First 1
|
||||
if ($exe) { return $exe.FullName }
|
||||
}
|
||||
$cmd = Get-Command jar.exe -ErrorAction SilentlyContinue
|
||||
if ($cmd) { return $cmd.Source }
|
||||
return $null
|
||||
}
|
||||
|
||||
@@ -0,0 +1,482 @@
|
||||
package skills.novel;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import cn.hutool.json.JSONArray;
|
||||
import cn.hutool.json.JSONObject;
|
||||
import cn.hutool.json.JSONUtil;
|
||||
import com.pcdd.sonovel.context.HttpClientContext;
|
||||
import com.pcdd.sonovel.context.OkHttpClientFactory;
|
||||
import com.pcdd.sonovel.core.AppConfigLoader;
|
||||
import com.pcdd.sonovel.core.Crawler;
|
||||
import com.pcdd.sonovel.core.Source;
|
||||
import com.pcdd.sonovel.handler.SearchResultsHandler;
|
||||
import com.pcdd.sonovel.model.AppConfig;
|
||||
import com.pcdd.sonovel.model.Chapter;
|
||||
import com.pcdd.sonovel.model.Rule;
|
||||
import com.pcdd.sonovel.model.Rule.Book;
|
||||
import com.pcdd.sonovel.model.SearchResult;
|
||||
import com.pcdd.sonovel.parser.BookParser;
|
||||
import com.pcdd.sonovel.parser.SearchParser;
|
||||
import com.pcdd.sonovel.parser.TocParser;
|
||||
import com.pcdd.sonovel.utils.SourceUtils;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileDescriptor;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.io.PrintStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
/**
|
||||
* Non-interactive wrapper around so-novel's crawler library.
|
||||
*
|
||||
* <p>so-novel ships as TUI / Web / single-URL CLI. This entry point adds the missing
|
||||
* piece for tool usage: aggregated keyword search with machine-readable output, plus a
|
||||
* download command that reports the produced file. Everything else (rule engine,
|
||||
* HTTP layer, chapter concurrency, epub/txt merging) is delegated to so-novel itself.
|
||||
*
|
||||
* <p>stdout carries exactly one JSON document; all library console output is captured
|
||||
* and returned in {@code logTail} when {@code --log} is given.
|
||||
*/
|
||||
public final class ToolMain {
|
||||
|
||||
private static final PrintStream OUT =
|
||||
new PrintStream(new FileOutputStream(FileDescriptor.out), true, StandardCharsets.UTF_8);
|
||||
|
||||
private static final int LOG_TAIL_BYTES = 64 * 1024;
|
||||
|
||||
public static void main(String[] args) {
|
||||
TailStream tail = new TailStream(LOG_TAIL_BYTES);
|
||||
PrintStream sink = new PrintStream(tail, true, StandardCharsets.UTF_8);
|
||||
PrintStream originalOut = System.out;
|
||||
System.setOut(sink);
|
||||
System.setErr(sink);
|
||||
|
||||
JSONObject result;
|
||||
try {
|
||||
result = dispatch(args);
|
||||
} catch (Throwable t) {
|
||||
result = new JSONObject();
|
||||
result.set("ok", false);
|
||||
result.set("error", t.getClass().getSimpleName() + ": " + String.valueOf(t.getMessage()));
|
||||
result.set("detail", StrUtil.sub(stackTrace(t), 0, 2000));
|
||||
} finally {
|
||||
System.setOut(originalOut);
|
||||
System.setErr(originalOut);
|
||||
}
|
||||
|
||||
if (hasFlag(args, "--log")) {
|
||||
result.set("logTail", tail.tailLines(40));
|
||||
}
|
||||
OUT.println(escapeNonAscii(JSONUtil.toJsonPrettyStr(result)));
|
||||
OUT.flush();
|
||||
|
||||
// so-novel opens pooled V8 runtimes and okhttp dispatchers; exit explicitly.
|
||||
System.exit(Boolean.TRUE.equals(result.getBool("ok")) ? 0 : 1);
|
||||
}
|
||||
|
||||
private static JSONObject dispatch(String[] args) {
|
||||
if (args.length == 0 || hasFlag(args, "-h") || hasFlag(args, "--help")) {
|
||||
JSONObject o = new JSONObject();
|
||||
o.set("ok", false);
|
||||
o.set("error", "missing command");
|
||||
o.set("usage", new JSONArray(List.of(
|
||||
"search <keyword> [--limit N] [--source ID] [--timeout SEC] [--log]",
|
||||
"download <detail-url> [--ext txt|epub|html|pdf] [--out DIR] [--log]",
|
||||
"toc <detail-url> [--log]",
|
||||
"sources")));
|
||||
return o;
|
||||
}
|
||||
|
||||
String command = args[0];
|
||||
return switch (command) {
|
||||
case "search" -> search(positional(args, 1), intOpt(args, "--limit", -1),
|
||||
intOpt(args, "--source", -1), intOpt(args, "--timeout", 180));
|
||||
case "download" -> download(positional(args, 1).isEmpty() ? opt(args, "--url") : positional(args, 1).get(0),
|
||||
opt(args, "--ext", "txt"), opt(args, "--out", null));
|
||||
case "toc" -> toc(positional(args, 1).isEmpty() ? opt(args, "--url") : positional(args, 1).get(0),
|
||||
intOpt(args, "--limit", 10));
|
||||
case "sources" -> sources();
|
||||
default -> {
|
||||
JSONObject o = new JSONObject();
|
||||
o.set("ok", false);
|
||||
o.set("error", "unknown command: " + command);
|
||||
yield o;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- commands
|
||||
|
||||
private static JSONObject search(List<String> words, int limit, int sourceId, int timeoutSec) {
|
||||
String keyword = String.join(" ", words).trim();
|
||||
JSONObject out = new JSONObject();
|
||||
out.set("ok", false);
|
||||
out.set("command", "search");
|
||||
if (keyword.isEmpty()) {
|
||||
out.set("error", "keyword is required");
|
||||
return out;
|
||||
}
|
||||
out.set("keyword", keyword);
|
||||
|
||||
AppConfig cfg = AppConfigLoader.APP_CONFIG;
|
||||
cfg.setEnableProgressbar(0);
|
||||
cfg.setAutoUpdate(0);
|
||||
if (limit > 0) cfg.setSearchLimit(limit);
|
||||
HttpClientContext.set(OkHttpClientFactory.create(cfg));
|
||||
warmUpJsRuntime(cfg);
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
List<Source> sources = new ArrayList<>(SourceUtils.getSearchableSources());
|
||||
if (sourceId > 0) {
|
||||
sources.removeIf(s -> s.rule.getId() != sourceId);
|
||||
}
|
||||
|
||||
ConcurrentLinkedQueue<SearchResult> collected = new ConcurrentLinkedQueue<>();
|
||||
ConcurrentLinkedQueue<JSONObject> failed = new ConcurrentLinkedQueue<>();
|
||||
AtomicInteger empty = new AtomicInteger();
|
||||
CountDownLatch latch = new CountDownLatch(sources.size());
|
||||
|
||||
try (var pool = Executors.newVirtualThreadPerTaskExecutor()) {
|
||||
for (Source source : sources) {
|
||||
int effectiveLimit = limit > 0 ? limit : cfg.getSearchLimit();
|
||||
if (effectiveLimit > 0) source.config.setSearchLimit(effectiveLimit);
|
||||
source.config.setEnableProgressbar(0);
|
||||
|
||||
pool.execute(() -> {
|
||||
long t0 = System.currentTimeMillis();
|
||||
try {
|
||||
List<SearchResult> hits = new SearchParser(source.config).parse(keyword);
|
||||
if (hits == null || hits.isEmpty()) {
|
||||
empty.incrementAndGet();
|
||||
} else {
|
||||
collected.addAll(hits);
|
||||
}
|
||||
} catch (Throwable e) {
|
||||
JSONObject f = new JSONObject();
|
||||
f.set("sourceId", source.rule.getId());
|
||||
f.set("sourceName", source.rule.getName());
|
||||
f.set("error", String.valueOf(e.getMessage()));
|
||||
f.set("ms", System.currentTimeMillis() - t0);
|
||||
failed.add(f);
|
||||
} finally {
|
||||
latch.countDown();
|
||||
}
|
||||
});
|
||||
}
|
||||
if (!latch.await(timeoutSec, TimeUnit.SECONDS)) {
|
||||
out.set("timedOut", true);
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
out.set("error", "interrupted: " + e.getMessage());
|
||||
return out;
|
||||
}
|
||||
|
||||
List<SearchResult> ranked = SearchResultsHandler.filterAndSort(new ArrayList<>(collected), keyword);
|
||||
|
||||
JSONArray rows = new JSONArray();
|
||||
int index = 1;
|
||||
for (SearchResult sr : ranked) {
|
||||
JSONObject row = new JSONObject();
|
||||
row.set("index", index++);
|
||||
row.set("sourceId", sr.getSourceId());
|
||||
row.set("sourceName", sr.getSourceName());
|
||||
row.set("bookName", sr.getBookName());
|
||||
row.set("author", sr.getAuthor());
|
||||
row.set("latestChapter", sr.getLatestChapter());
|
||||
row.set("lastUpdateTime", sr.getLastUpdateTime());
|
||||
row.set("status", sr.getStatus());
|
||||
row.set("category", sr.getCategory());
|
||||
row.set("wordCount", sr.getWordCount());
|
||||
row.set("url", sr.getUrl());
|
||||
rows.add(row);
|
||||
}
|
||||
|
||||
JSONObject stats = new JSONObject();
|
||||
stats.set("total", sources.size());
|
||||
stats.set("returned", collected.size());
|
||||
stats.set("empty", empty.get());
|
||||
stats.set("failed", new JSONArray(failed));
|
||||
|
||||
out.set("ok", true);
|
||||
out.set("count", rows.size());
|
||||
out.set("results", rows);
|
||||
out.set("sources", stats);
|
||||
out.set("tookMs", System.currentTimeMillis() - start);
|
||||
return out;
|
||||
}
|
||||
|
||||
private static JSONObject download(String url, String ext, String outDir) {
|
||||
JSONObject out = new JSONObject();
|
||||
out.set("ok", false);
|
||||
out.set("command", "download");
|
||||
if (StrUtil.isBlank(url)) {
|
||||
out.set("error", "book detail url is required (--url or first positional argument)");
|
||||
return out;
|
||||
}
|
||||
String format = ext == null ? "txt" : ext.toLowerCase();
|
||||
if (!format.matches("txt|epub|html|pdf")) {
|
||||
out.set("error", "unsupported ext: " + ext + " (txt|epub|html|pdf)");
|
||||
return out;
|
||||
}
|
||||
out.set("url", url);
|
||||
out.set("ext", format);
|
||||
|
||||
AppConfig cfg = AppConfigLoader.APP_CONFIG;
|
||||
cfg.setEnableProgressbar(0);
|
||||
cfg.setAutoUpdate(0);
|
||||
cfg.setExtName(format);
|
||||
if (StrUtil.isNotBlank(outDir)) cfg.setDownloadPath(new File(outDir).getAbsolutePath());
|
||||
|
||||
// bookUrl -> rule: same lookup the TUI/CLI uses, so any active source is accepted
|
||||
Rule rule = SourceUtils.getRule(url);
|
||||
cfg.setSourceId(rule.getId());
|
||||
out.set("sourceId", rule.getId());
|
||||
out.set("sourceName", rule.getName());
|
||||
|
||||
HttpClientContext.set(OkHttpClientFactory.create(cfg));
|
||||
warmUpJsRuntime(cfg);
|
||||
|
||||
Book book = new BookParser(cfg).parse(url);
|
||||
out.set("bookName", book.getBookName());
|
||||
out.set("author", book.getAuthor());
|
||||
|
||||
List<Chapter> toc = new TocParser(cfg).parseAll(url);
|
||||
out.set("chapters", toc.size());
|
||||
if (toc.isEmpty()) {
|
||||
out.set("error", "source returned an empty table of contents (rule broken or page blocked)");
|
||||
return out;
|
||||
}
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
// reuse the parsed toc: avoids a second directory fetch and lets us report the chapter count
|
||||
double seconds = new Crawler(cfg).crawl(url, toc);
|
||||
long tookMs = System.currentTimeMillis() - start;
|
||||
|
||||
File produced = locateOutput(cfg.getDownloadPath(), book, format);
|
||||
boolean ok = produced != null && produced.length() > 0;
|
||||
out.set("ok", ok);
|
||||
out.set("seconds", Math.round(seconds * 100.0) / 100.0);
|
||||
out.set("tookMs", tookMs);
|
||||
out.set("downloadPath", new File(cfg.getDownloadPath()).getAbsolutePath());
|
||||
if (ok) {
|
||||
out.set("file", produced.getAbsolutePath());
|
||||
out.set("bytes", produced.length());
|
||||
} else {
|
||||
out.set("error", "download finished but no output file was produced "
|
||||
+ "(source probably returned an empty table of contents)");
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private static JSONObject toc(String url, int preview) {
|
||||
JSONObject out = new JSONObject();
|
||||
out.set("ok", false);
|
||||
out.set("command", "toc");
|
||||
if (StrUtil.isBlank(url)) {
|
||||
out.set("error", "book detail url is required");
|
||||
return out;
|
||||
}
|
||||
|
||||
AppConfig cfg = AppConfigLoader.APP_CONFIG;
|
||||
cfg.setEnableProgressbar(0);
|
||||
cfg.setAutoUpdate(0);
|
||||
Rule rule = SourceUtils.getRule(url);
|
||||
cfg.setSourceId(rule.getId());
|
||||
|
||||
HttpClientContext.set(OkHttpClientFactory.create(cfg));
|
||||
warmUpJsRuntime(cfg);
|
||||
|
||||
List<Chapter> chapters = new TocParser(cfg).parseAll(url);
|
||||
JSONArray head = new JSONArray();
|
||||
JSONArray tail = new JSONArray();
|
||||
for (int i = 0; i < chapters.size(); i++) {
|
||||
if (i < preview) head.add(chapters.get(i).getTitle());
|
||||
if (i >= chapters.size() - preview) tail.add(chapters.get(i).getTitle());
|
||||
}
|
||||
out.set("ok", !chapters.isEmpty());
|
||||
out.set("url", url);
|
||||
out.set("sourceId", rule.getId());
|
||||
out.set("sourceName", rule.getName());
|
||||
out.set("chapters", chapters.size());
|
||||
out.set("firstChapters", head);
|
||||
out.set("lastChapters", tail);
|
||||
if (chapters.isEmpty()) out.set("error", "empty table of contents");
|
||||
return out;
|
||||
}
|
||||
|
||||
private static JSONObject sources() {
|
||||
AppConfig cfg = AppConfigLoader.APP_CONFIG;
|
||||
cfg.setAutoUpdate(0);
|
||||
JSONArray rows = new JSONArray();
|
||||
for (Rule r : SourceUtils.getActivatedRules()) {
|
||||
JSONObject row = new JSONObject();
|
||||
row.set("id", r.getId());
|
||||
row.set("name", r.getName());
|
||||
row.set("url", r.getUrl());
|
||||
row.set("searchable", r.getSearch() != null && !r.getSearch().isDisabled() && !r.isDisabled());
|
||||
row.set("language", r.getLanguage());
|
||||
row.set("needProxy", r.isNeedProxy());
|
||||
rows.add(row);
|
||||
}
|
||||
JSONObject out = new JSONObject();
|
||||
out.set("ok", true);
|
||||
out.set("command", "sources");
|
||||
out.set("activeRules", cfg.getActiveRules());
|
||||
out.set("count", rows.size());
|
||||
out.set("sources", rows);
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- helpers
|
||||
|
||||
/** Creates and closes one V8 runtime, mirroring com.pcdd.sonovel.Main's Windows workaround. */
|
||||
private static void warmUpJsRuntime(AppConfig cfg) {
|
||||
try (var v8 = com.caoccao.javet.interop.V8Host.getV8Instance().createV8Runtime()) {
|
||||
// no-op
|
||||
} catch (Throwable ignored) {
|
||||
// JS-based rules are optional; CSS/XPath rules still work without V8
|
||||
}
|
||||
}
|
||||
|
||||
private static File locateOutput(String downloadPath, Book book, String ext) {
|
||||
File dir = new File(downloadPath);
|
||||
File direct = new File(dir, "%s(%s).%s".formatted(book.getBookName(), book.getAuthor(), ext));
|
||||
if (direct.exists()) return direct;
|
||||
|
||||
File[] candidates = dir.listFiles((d, name) -> name.toLowerCase().endsWith("." + ext));
|
||||
if (candidates == null || candidates.length == 0) return null;
|
||||
return Arrays.stream(candidates)
|
||||
.filter(f -> f.getName().contains(book.getBookName()))
|
||||
.max(Comparator.comparingLong(File::lastModified))
|
||||
.orElseGet(() -> Arrays.stream(candidates)
|
||||
.max(Comparator.comparingLong(File::lastModified))
|
||||
.orElse(null));
|
||||
}
|
||||
|
||||
private static List<String> positional(String[] args, int from) {
|
||||
List<String> values = new ArrayList<>();
|
||||
for (int i = from; i < args.length; i++) {
|
||||
String a = args[i];
|
||||
if (a.startsWith("--")) {
|
||||
// skip option values that are not positional
|
||||
if (needsValue(a) && i + 1 < args.length) i++;
|
||||
continue;
|
||||
}
|
||||
values.add(a);
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
private static String opt(String[] args, String name, String fallback) {
|
||||
String value = opt(args, name);
|
||||
return value == null ? fallback : value;
|
||||
}
|
||||
|
||||
private static String opt(String[] args, String name) {
|
||||
for (int i = 0; i < args.length; i++) {
|
||||
String a = args[i];
|
||||
if (a.equals(name) && i + 1 < args.length) return args[i + 1];
|
||||
if (a.startsWith(name + "=")) return a.substring(name.length() + 1);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static int intOpt(String[] args, String name, int fallback) {
|
||||
String v = opt(args, name);
|
||||
if (StrUtil.isBlank(v)) return fallback;
|
||||
try {
|
||||
return Integer.parseInt(v.trim());
|
||||
} catch (NumberFormatException e) {
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean hasFlag(String[] args, String name) {
|
||||
return Arrays.stream(args).anyMatch(a -> a.equalsIgnoreCase(name));
|
||||
}
|
||||
|
||||
private static boolean needsValue(String option) {
|
||||
String bare = option.contains("=") ? option.substring(0, option.indexOf('=')) : option;
|
||||
return switch (bare) {
|
||||
case "--limit", "--source", "--timeout", "--ext", "--out", "--url" -> true;
|
||||
default -> false;
|
||||
};
|
||||
}
|
||||
|
||||
private static String stackTrace(Throwable t) {
|
||||
java.io.StringWriter sw = new java.io.StringWriter();
|
||||
t.printStackTrace(new java.io.PrintWriter(sw));
|
||||
return sw.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* JSON only needs ASCII: escaping non-ASCII keeps the output byte-identical under any
|
||||
* console code page, which avoids mojibake in Windows shells and pipelines.
|
||||
*/
|
||||
private static String escapeNonAscii(String json) {
|
||||
StringBuilder sb = new StringBuilder(json.length() + 64);
|
||||
for (int i = 0; i < json.length(); i++) {
|
||||
char c = json.charAt(i);
|
||||
if (c < 0x80) {
|
||||
sb.append(c);
|
||||
} else {
|
||||
sb.append(String.format("\\u%04x", (int) c));
|
||||
}
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
/** Bounded byte buffer that keeps the last N bytes written by the library. */
|
||||
private static final class TailStream extends OutputStream {
|
||||
private final int max;
|
||||
private final ByteArrayOutputStream buffer = new ByteArrayOutputStream();
|
||||
|
||||
TailStream(int max) {
|
||||
this.max = max;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void write(int b) {
|
||||
trimIfNeeded(1);
|
||||
buffer.write(b);
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void write(byte[] b, int off, int len) {
|
||||
trimIfNeeded(len);
|
||||
buffer.write(b, off, len);
|
||||
}
|
||||
|
||||
private void trimIfNeeded(int incoming) {
|
||||
if (buffer.size() + incoming <= max) return;
|
||||
byte[] keep = Arrays.copyOfRange(buffer.toByteArray(), max / 2, buffer.size());
|
||||
buffer.reset();
|
||||
buffer.write(keep, 0, keep.length);
|
||||
}
|
||||
|
||||
synchronized List<String> tailLines(int count) {
|
||||
String text = buffer.toString(StandardCharsets.UTF_8);
|
||||
List<String> lines = Arrays.stream(text.split("\\R"))
|
||||
.filter(StrUtil::isNotBlank)
|
||||
.toList();
|
||||
int from = Math.max(0, lines.size() - count);
|
||||
return new ArrayList<>(lines.subList(from, lines.size()));
|
||||
}
|
||||
}
|
||||
|
||||
private ToolMain() {
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user