diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..cf60ada --- /dev/null +++ b/.env.example @@ -0,0 +1,6 @@ +PORT=3000 +DUCKDB_PATH=./data/awesome.duckdb +SESSION_SECRET=change-me-in-production +ADMIN_INIT_PASSWORD=admin +AI_INGEST_KEY= +BASE_URL=http://localhost:3000 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..97725b4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +data/ +.env +*.log diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..eaea0ed --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM node:22-bookworm-slim AS deps +WORKDIR /app +COPY package.json package-lock.json ./ +RUN npm ci --omit=dev --no-audit --no-fund + +FROM node:22-bookworm-slim +WORKDIR /app +ENV NODE_ENV=production +COPY --from=deps /app/node_modules ./node_modules +COPY package.json ./ +COPY src ./src +COPY public ./public +VOLUME ["/app/data"] +EXPOSE 3000 +USER node +HEALTHCHECK --interval=30s --timeout=5s --start-period=15s \ + CMD node -e "fetch('http://localhost:3000/healthz').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))" +CMD ["node", "src/index.js"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..8bc3c26 --- /dev/null +++ b/README.md @@ -0,0 +1,68 @@ +# Awesome Index + +把网上牛逼的东西收进一个库：给人用，也给 AI 用。 +精选开源软件 / 微服务 / SaaS / 网站 / 工具 / 脚本 / 插件的结构化数据库，内置 MCP 接口。 + +- 设计方案：[design/DESIGN.md](design/DESIGN.md)（Riso 印刷风格 + 静态原型） +- 框架设计：[docs/ARCHITECTURE.md](docs/ARCHITECTURE.md)（模块划分 / 数据模型 / 端点） + +## 快速开始 + +### 本地开发 + +```bash +npm install +npm run dev # http://localhost:3000（--watch 热重载） +npm test # node:test，10 个用例 +``` + +首次启动自动完成建表与种子数据。默认管理员 `admin`，密码取 `ADMIN_INIT_PASSWORD`（默认 `admin`）。 + +### Docker 部署 + +```bash +cp .env.example .env # 修改 SESSION_SECRET / ADMIN_INIT_PASSWORD / AI_INGEST_KEY +docker compose up -d --build +``` + +数据持久化在宿主机 `./data/awesome.duckdb`，升级镜像不丢数据。 + +## 给人的入口 + +| 页面 | 地址 | +|---|---| +| 首页（搜索 / MCP 说明 / 最近收录） | `/` | +| 列表页（类型 + 嵌套标签过滤） | `/entries?q=关键字&type=工具&tags=1,2&sort=stars` | +| 详情页（元数据 / 评论 / 机读 JSON） | `/entries/:slug` | +| 随机游览 | `/random` | +| 登录 | `/login` | +| 管理台（内容/标签/评论/内容源/账号） | `/admin` | + +## 给 AI 的入口（MCP） + +Streamable HTTP · 只读 · 无需密钥： + +``` +POST {BASE_URL}/mcp +tools: search_entries / get_entry / random_entry / list_tags +``` + +一句话让 Agent 自己接入： + +> 请把 URL 为 `{BASE_URL}/mcp` 的 MCP 服务器添加到你的客户端配置中，名称用 awesome-index，完成后告诉我现在可以调用哪些工具。 + +## 内容源（自动化收录） + +三类来源汇聚到同一条目表： + +| kind | 说明 | 新条目状态 | +|---|---|---| +| `human` | 管理台人工录入（常开） | active | +| `ai` | `POST /api/ingest/entry` + `X-Ingest-Key` 头 | pending | +| `feed` | 定时拉取外部站点（rss / html / eryajf-weekly 适配器），管理台可手动触发 | pending | + +首个站点适配器：二丫讲梵学习周刊（wiki.eryajf.net）。运行记录见管理台「内容源 → 运行记录」。 + +## 技术栈 + +Node.js 22 (ESM) · Express 5 · EJS · DuckDB (`@duckdb/node-api`) · zod · express-session · @modelcontextprotocol/sdk · node-cron · cheerio diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..f461aaf --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,16 @@ +services: + web: + build: . + container_name: awesome-index + ports: + - "3000:3000" + volumes: + - ./data:/app/data + environment: + PORT: "3000" + DUCKDB_PATH: /app/data/awesome.duckdb + SESSION_SECRET: ${SESSION_SECRET:-change-me-in-production} + ADMIN_INIT_PASSWORD: ${ADMIN_INIT_PASSWORD:-admin} + AI_INGEST_KEY: ${AI_INGEST_KEY:-} + BASE_URL: ${BASE_URL:-http://localhost:3000} + restart: unless-stopped diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..43c15ff --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,270 @@ +# Awesome Index · 框架设计 + +> 目标：把 `design/prototype/` 的静态原型落地为一个由 DuckDB 驱动、全 JavaScript 编写、 +> 对人与对 AI（MCP）同时提供服务、可 Docker 部署的小型数据库网站。 +> 本文档回答三个问题：**模块怎么分、每个模块负责什么、模块对应哪个目录。** + +--- + +## 1. 总体架构 + +经典三横层，外加一个并列的机器读者入口： + +``` + ┌──────────────────────────────────────────┐ + 人类读者 ────▶ │ 视图层 views/ (EJS SSR) + public/ 静态 │ + │ 页面内 +<%- include('partials/footer-bar') %> diff --git a/src/views/detail.ejs b/src/views/detail.ejs new file mode 100644 index 0000000..2e25292 --- /dev/null +++ b/src/views/detail.ejs @@ -0,0 +1,96 @@ +<%- include('partials/head') %> +<%- include('partials/topbar', { active: 'browse' }) %> + +
+

~/awesome-index / <%= entry.type %> / <%= entry.title %>

+ +
+
+

<%= entry.title %>

+ <%= entry.type %> + <% if (entry.status === 'pending') { %>pending 待核验<% } %> +
+ 访问仓库 ↗ + +
+
+

<%= entry.description_md %>

+
+ +
+
+
+

// entry.body · markdown source

+

这是什么

+

<%= entry.description_md || '暂无介绍。' %>

+

仓库地址：<%= entry.url %>

+
+ +
+

// dual reader view · 人读 ⇄ 机读

+
+ + +
+
+

这一页就是人读视图。切换到「机读 JSON」可以看到 AI 通过 MCP 的 get_entry 拿到的同一条数据——两种读者，同一份事实。

+
+ +
+ +
+

// comments · <%= comments.length %> 条

+

评论

+ <% for (const c of comments) { %> +
+ <%= (c.author_name || '?').slice(0, 1).toUpperCase() %> +
+
+ <%= c.author_name %> + <% if (c.username) { %>@<%= c.username %><% } %> + +
+

<%= c.body %>

+
+
+ <% } %> + <% if (!comments.length) { %>

还没有评论，说点有用的。

<% } %> + +
+
+ +
+ +
+
+
+
+ + +
+
+ +<%- include('partials/footer-bar') %> diff --git a/src/views/index.ejs b/src/views/index.ejs new file mode 100644 index 0000000..4366138 --- /dev/null +++ b/src/views/index.ejs @@ -0,0 +1,146 @@ +<%- include('partials/head') %> +<%- include('partials/topbar', { active: 'home' }) %> + +
+
+
+
+

~/awesome-index — 给人用，也给 AI 用

+

把网上牛逼的东西，
收进一个好用的库。

+

人工精选的开源软件、微服务、SaaS、网站、工具、脚本与插件。每一条都是结构化数据：人在页面上搜，AI 通过 MCP 直接调用。

+ + +

+ + + + 给 AI 用？本站提供 MCP 接口 + + + +

+ 收录 <%= stats.total %> 条 + 在架 <%= stats.active %> 条 + 标签组 <%= stats.tags %> 个 + 本周新增 +<%= stats.weekNew %> + 存储 DuckDB + MCP 就绪 +

+
+ + +
+
+ +
+
+

// mcp.transport: streamable-http · auth: none

+

同一个库，两种读者。
AI 走这边。

+

本站所有条目同时暴露为 MCP（Model Context Protocol）接口。把下面的地址加进你的 AI 客户端，Claude、Cursor 或任何支持 MCP 的 Agent 就能直接搜索与读取这个库。

+ +
+
+

不想手动改配置？

+

复制下面这句话发给你的 Agent，它会自己完成接入：

+
「请把 URL 为 <%- config.baseUrl %>/mcp 的 MCP 服务器添加到你的客户端配置中，名称用 awesome-index，完成后告诉我现在可以调用哪些工具。」
+
+ +
+ +
+ + +
+
+
01

把服务器加进客户端配置

+
{
+  "mcpServers": {
+    "awesome-index": {
+      "url": "<%- config.baseUrl %>/mcp"
+    }
+  }
+}
+
+
+
02

可用的工具

+ + + + + + + + +
tool说明
search_entries按关键字搜索收录内容
get_entry获取单条的完整详情与元数据
random_entry随机返回一条，用于探索发现
list_tags列出全部标签组及其嵌套结构
+
+
+
03

直接问你的 AI

+
「用 awesome-index 找三个能自托管的相册方案，
+  按 star 数排序，给我 repo 链接。」
+
+
+
+
+
+ +
+
+
+

// resource: recent_entries · limit 4

+

最近收录

+
+ 查看全部 → +
+
+ <% for (const e of recent) { %> + + <%= e.type %> +

<%= e.title %>

+

<%= e.description_md %>

+
+ <% for (const t of (e.tags || []).slice(0, 3)) { %><%= t %><% } %> +
+

★ <%= fmtStars(e.stars) %> · 更新于 <%= fmtDate(e.updated_at) %>

+ + <% } %> + <% if (!recent.length) { %> +

还没有收录条目——去管理台新增，或等采集源跑一趟。

+ <% } %> +
+
+
+ +<%- include('partials/footer-full') %> diff --git a/src/views/list.ejs b/src/views/list.ejs new file mode 100644 index 0000000..eeea107 --- /dev/null +++ b/src/views/list.ejs @@ -0,0 +1,115 @@ +<%- include('partials/head') %> +<%- include('partials/topbar', { active: 'browse' }) %> + +
+
+

// tool: search_entries · filters applied server-side

+

<%= filters.q ? '「' + filters.q + '」的搜索结果' : '浏览全部' %>

+
+ + + +
+ 共 <%= result.total %> 条结果 +
+ + + +
+
+ +
+ + +
+ <% for (const e of result.items) { %> + + <%= e.type %> +
+

<%= e.title %>

+

<%= e.description_md %>

+
+ <% for (const t of e.tags.slice(0, 4)) { %><%= t %><% } %> +
+
+
+ ★ <%= fmtStars(e.stars) %> + <%= e.license || '—' %> · 更新于 <%= fmtDate(e.updated_at) %> +
+ + <% } %> + <% if (!result.items.length) { %> +

没有匹配的条目。换个关键字，或者清空筛选再试。

+ <% } %> + + +
+
+
+ + +<%- include('partials/footer-bar') %> diff --git a/src/views/login.ejs b/src/views/login.ejs new file mode 100644 index 0000000..12aed4b --- /dev/null +++ b/src/views/login.ejs @@ -0,0 +1,42 @@ +<%- include('partials/head') %> +<%- include('partials/topbar', { active: 'admin' }) %> + +
+
+
+

// route: /admin/login · auth: session cookie

+ +

管理台登录

+

人工通道。AI Agent 请走 MCP 接口——这扇门不对机器开放。

+ +
+
+ + +
+
+ + +
+
+ + 忘记密码？ +
+ +
+ +
+ // 初始账号：admin（密码为 ADMIN_INIT_PASSWORD） + // agents: POST /mcp · 此处返回 403 +
+
+
+
+ +
+
+ © 2026 honor3.com · Awesome Index Admin + ← 返回首页 +
+
+<%- include('partials/footer-bar') %> diff --git a/src/views/partials/footer-bar.ejs b/src/views/partials/footer-bar.ejs new file mode 100644 index 0000000..183a498 --- /dev/null +++ b/src/views/partials/footer-bar.ejs @@ -0,0 +1,11 @@ + +
+ + + + diff --git a/src/views/partials/footer-full.ejs b/src/views/partials/footer-full.ejs new file mode 100644 index 0000000..5a1b848 --- /dev/null +++ b/src/views/partials/footer-full.ejs @@ -0,0 +1,48 @@ + +
+ + + + diff --git a/src/views/partials/head.ejs b/src/views/partials/head.ejs new file mode 100644 index 0000000..b1fe031 --- /dev/null +++ b/src/views/partials/head.ejs @@ -0,0 +1,14 @@ + + + + + +<%= typeof title !== 'undefined' ? title : 'Awesome Index' %> + + + + + + + + diff --git a/src/views/partials/tag-node.ejs b/src/views/partials/tag-node.ejs new file mode 100644 index 0000000..3d54170 --- /dev/null +++ b/src/views/partials/tag-node.ejs @@ -0,0 +1,23 @@ +
  • +
    + <% if (node.children.length) { %> + + <% } else { %> + <%= node.name %> + <% } %> + <%= node.count %> · #<%= node.id %> + + + + + + +
    + <% if (node.children.length) { %> + + <% } %> +
    • diff --git a/src/views/partials/topbar.ejs b/src/views/partials/topbar.ejs new file mode 100644 index 0000000..842c2f1 --- /dev/null +++ b/src/views/partials/topbar.ejs @@ -0,0 +1,14 @@ +
    +
    + + + AWESOME·INDEX + + + +
    +
    diff --git a/test/entries.test.js b/test/entries.test.js new file mode 100644 index 0000000..9d72526 --- /dev/null +++ b/test/entries.test.js @@ -0,0 +1,74 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { withApp } from "./helpers.js"; + +test("seeded entries are searchable", async () => { + await withApp(async ({ api }) => { + const res = await api("/api/entries?q=ripgrep"); + assert.equal(res.status, 200); + assert.ok(res.json.data.total >= 1); + assert.equal(res.json.data.items[0].title, "ripgrep"); + }); +}); + +test("random endpoint returns an active entry", async () => { + await withApp(async ({ api }) => { + const res = await api("/api/entries/random"); + assert.equal(res.status, 200); + assert.equal(res.json.data.status, "active"); + }); +}); + +test("invalid status transition is rejected", async () => { + await withApp(async ({ api, loginAdmin }) => { + const cookie = await loginAdmin(); + const found = await api("/api/entries?q=ripgrep"); + const id = found.json.data.items[0].id; + const res = await api(`/api/entries/${id}/status`, { + method: "PATCH", + body: { status: "active" }, + cookie, + }); + assert.equal(res.status, 400); + }); +}); + +test("greyed -> active is allowed", async () => { + await withApp(async ({ api, loginAdmin }) => { + const cookie = await loginAdmin(); + const found = await api("/api/entries?q=ripgrep"); + const id = found.json.data.items[0].id; + const grey = await api(`/api/entries/${id}/status`, { + method: "PATCH", + body: { status: "greyed" }, + cookie, + }); + assert.equal(grey.json.data.status, "greyed"); + const back = await api(`/api/entries/${id}/status`, { + method: "PATCH", + body: { status: "active" }, + cookie, + }); + assert.equal(back.json.data.status, "active"); + }); +}); + +test("AI ingest without a valid key is forbidden", async () => { + await withApp(async ({ api }) => { + const res = await api("/api/ingest/entry", { + method: "POST", + body: { title: "x", url: "https://github.com/a/b", type: "工具" }, + }); + assert.equal(res.status, 403); + }); +}); + +test("unauthenticated writes to entries are forbidden", async () => { + await withApp(async ({ api }) => { + const res = await api("/api/entries", { + method: "POST", + body: { title: "x", url: "https://github.com/a/c", type: "工具" }, + }); + assert.equal(res.status, 403); + }); +}); diff --git a/test/helpers.js b/test/helpers.js new file mode 100644 index 0000000..ee19fd7 --- /dev/null +++ b/test/helpers.js @@ -0,0 +1,55 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { connect, close } from "../src/db/connection.js"; +import { runMigrate } from "../src/db/migrate.js"; +import { buildApp } from "../src/app.js"; + +export async function withApp(fn) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "awesome-test-")); + await connect(path.join(dir, "test.duckdb")); + await runMigrate({ adminInitPassword: "test123", aiIngestKey: "" }); + const app = buildApp({ + sessionSecret: "test-secret", + duckdbPath: path.join(dir, "test.duckdb"), + baseUrl: "http://test.local", + }); + const server = app.listen(0); + await new Promise((r) => server.once("listening", r)); + const base = `http://127.0.0.1:${server.address().port}`; + + async function api(pathname, { method = "GET", body, cookie } = {}) { + const res = await fetch(base + pathname, { + method, + headers: { + ...(body ? { "content-type": "application/json" } : {}), + ...(cookie ? { cookie } : {}), + }, + body: body ? JSON.stringify(body) : undefined, + }); + let json = null; + try { + json = await res.json(); + } catch {} + return { status: res.status, json, headers: res.headers }; + } + + async function loginAdmin() { + const res = await fetch(base + "/api/admin/login", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ username: "admin", password: "test123" }), + }); + const cookies = res.headers.getSetCookie().map((c) => c.split(";")[0]); + return cookies.join("; "); + } + + try { + await fn({ base, api, loginAdmin }); + } finally { + server.closeAllConnections?.(); + await new Promise((r) => server.close(r)); + await close(); + fs.rmSync(dir, { recursive: true, force: true, maxRetries: 3 }); + } +} diff --git a/test/pipeline.test.js b/test/pipeline.test.js new file mode 100644 index 0000000..8896e05 --- /dev/null +++ b/test/pipeline.test.js @@ -0,0 +1,57 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { withApp } from "./helpers.js"; +import { registerAdapter } from "../src/ingest/adapters/index.js"; +import { runSource } from "../src/ingest/pipeline.js"; + +test("feed pipeline dedupes by url and records runs", async () => { + await withApp(async ({ api, loginAdmin }) => { + registerAdapter("test-fake", { + async fetch() { + return [ + { + title: "ripgrep", + url: "https://github.com/BurntSushi/ripgrep?utm_source=x", + description: "dup", + }, + { + title: "brand-new-tool", + url: "https://github.com/fake/brand-new-tool", + description: "new item from fake source", + }, + { title: "", url: "https://example.com/invalid", description: "" }, + ]; + }, + }); + + const cookie = await loginAdmin(); + const created = await api("/api/admin/sources", { + method: "POST", + body: { + name: "fake 源", + url: "mem://fake", + adapter: "test-fake", + cron_expr: "0 9 * * 1", + }, + cookie, + }); + assert.equal(created.status, 201); + const sourceId = created.json.data.id; + + const stats = await runSource(sourceId); + assert.equal(stats.found, 3); + assert.equal(stats.created, 1); + assert.equal(stats.skipped, 2); + + const runs = await api(`/api/admin/sources/${sourceId}/runs`, {}, cookie ? {} : {}); + void runs; + + const pending = await api("/api/entries?q=brand-new-tool&status=all"); + assert.equal(pending.json.data.total, 1); + assert.equal(pending.json.data.items[0].status, "pending"); + + const manual = await api(`/api/admin/sources/${sourceId}/run`, { method: "POST", cookie }); + assert.equal(manual.json.data.created, 0); + assert.equal(manual.json.data.skipped, 3); + }); +}); diff --git a/test/tags.test.js b/test/tags.test.js new file mode 100644 index 0000000..9dbc1e8 --- /dev/null +++ b/test/tags.test.js @@ -0,0 +1,67 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { withApp } from "./helpers.js"; + +test("seeded tag tree exposes nested groups with counts", async () => { + await withApp(async ({ api }) => { + const res = await api("/api/tags/tree"); + assert.equal(res.status, 200); + const tree = res.json.data; + const lang = tree.find((n) => n.name === "编程语言"); + assert.ok(lang); + assert.ok(lang.children.some((c) => c.name === "Rust")); + assert.ok(lang.children.every((c) => c.count >= 0)); + }); +}); + +test("tag create / rename / move / delete rules", async () => { + await withApp(async ({ api, loginAdmin }) => { + const cookie = await loginAdmin(); + + const root = (await api("/api/tags/tree")).json.data.find( + (n) => n.name === "编程语言" + ); + + const created = await api("/api/tags", { + method: "POST", + body: { name: "测试语言", parentId: root.id }, + cookie, + }); + assert.equal(created.status, 201); + const childId = created.json.data.id; + + const grandchild = await api("/api/tags", { + method: "POST", + body: { name: "测试子语言", parentId: childId }, + cookie, + }); + const grandId = grandchild.json.data.id; + + const renamed = await api(`/api/tags/${grandId}`, { + method: "PATCH", + body: { name: "测试子语言2" }, + cookie, + }); + assert.ok(renamed.ok !== false); + + const cyclic = await api(`/api/tags/${root.id}`, { + method: "PATCH", + body: { parentId: childId }, + cookie, + }); + assert.equal(cyclic.status, 400); + + await api(`/api/tags/${childId}`, { method: "DELETE", cookie }); + + const flat = (await api("/api/tags")).json.data; + const moved = flat.find((t) => t.id === grandId); + assert.equal(moved.parent_id ?? null, root.id); + }); +}); + +test("tag deletion requires admin", async () => { + await withApp(async ({ api }) => { + const res = await api("/api/tags/1", { method: "DELETE" }); + assert.equal(res.status, 401); + }); +});