From 82879a36cc23f242fbaec3586f33c44b1fbb9ed7 Mon Sep 17 00:00:00 2001 From: tangsongdayo Date: Sun, 19 Jul 2026 12:52:44 +0800 Subject: [PATCH] feat: index public search with FTS5 --- CHANGELOG.md | 1 + app/page.tsx | 5 +++-- lib/db.ts | 18 ++++++++++++++++++ tests/sync.test.ts | 3 ++- 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8ddc8f..1ff39e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - Hardened incoming RSS/Atom sources with public-HTTPS validation, redirect checks, response-size limits, and safe XML declaration rejection. - Added production configuration validation for secrets, encryption keys, and the public HTTPS URL, plus an administrator-facing status check. - Added Prometheus-compatible metrics and distinct liveness/readiness health probes. +- Added SQLite FTS5 indexing for public-content search, maintained automatically as posts change. ### Fixed diff --git a/app/page.tsx b/app/page.tsx index 6047923..36c8c29 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -9,8 +9,9 @@ type Query = { q?: string; tag?: string; source?: string; author?: string; from? export default async function Home({ searchParams }: { searchParams: Promise }) { const query = await searchParams; const q = query.q?.trim() || ""; const tag = query.tag?.trim() || ""; const author = query.author?.trim() || ""; const sourceId = Number(query.source) || 0; const from = query.from || ""; const to = query.to || ""; const attachments = query.attachments === "1"; const page = Math.max(1, Number(query.page) || 1); const where = ["p.visibility='PUBLIC'", "p.hidden=0"]; const args: (string | number)[] = []; - if (q) { where.push("p.content LIKE ?"); args.push(`%${q}%`); } if (tag) { where.push("p.tags_json LIKE ?"); args.push(`%${JSON.stringify(tag).slice(1, -1)}%`); } if (author) { where.push("u.username LIKE ?"); args.push(`%${author}%`); } if (sourceId) { where.push("s.id=?"); args.push(sourceId); } if (from) { where.push("date(COALESCE(p.remote_created_at,p.created_at)) >= date(?)"); args.push(from); } if (to) { where.push("date(COALESCE(p.remote_created_at,p.created_at)) <= date(?)"); args.push(to); } if (attachments) where.push("p.attachments_json <> '[]'"); - const joins = " FROM posts p JOIN users u ON u.id=p.author_id LEFT JOIN sources s ON s.id=p.source_id "; const predicate = ` WHERE ${where.join(" AND ")}`; + const ftsQuery = q.split(/\s+/).filter(Boolean).map((term) => `"${term.replaceAll('"', '""')}"`).join(" AND "); + if (q) { where.push("posts_fts MATCH ?"); args.push(ftsQuery); } if (tag) { where.push("p.tags_json LIKE ?"); args.push(`%${JSON.stringify(tag).slice(1, -1)}%`); } if (author) { where.push("u.username LIKE ?"); args.push(`%${author}%`); } if (sourceId) { where.push("s.id=?"); args.push(sourceId); } if (from) { where.push("date(COALESCE(p.remote_created_at,p.created_at)) >= date(?)"); args.push(from); } if (to) { where.push("date(COALESCE(p.remote_created_at,p.created_at)) <= date(?)"); args.push(to); } if (attachments) where.push("p.attachments_json <> '[]'"); + const joins = ` FROM posts p ${q ? "JOIN posts_fts ON posts_fts.rowid=p.id" : ""} JOIN users u ON u.id=p.author_id LEFT JOIN sources s ON s.id=p.source_id `; const predicate = ` WHERE ${where.join(" AND ")}`; const total = Number((db.prepare(`SELECT count(*) count${joins}${predicate}`).get(...args) as { count: number }).count); const pages = Math.max(1, Math.ceil(total / pageSize)); const safePage = Math.min(page, pages); const posts = db.prepare(`SELECT p.*,u.username,s.name,s.remote_display_name,s.base_url AS source_base_url,(SELECT count(*) FROM comments c WHERE c.post_id=p.id AND c.hidden=0) comment_count,(SELECT count(*) FROM reactions r WHERE r.post_id=p.id) reaction_count${joins}${predicate} ORDER BY COALESCE(p.remote_created_at,p.created_at) DESC LIMIT ? OFFSET ?`).all(...args, pageSize, (safePage - 1) * pageSize) as PublicPost[]; const sources = db.prepare("SELECT id,name FROM sources WHERE is_enabled=1 ORDER BY name").all() as { id: number; name: string }[]; diff --git a/lib/db.ts b/lib/db.ts index 0c75698..e5f1ad5 100644 --- a/lib/db.ts +++ b/lib/db.ts @@ -154,6 +154,24 @@ db.prepare("UPDATE source_members SET role='editor' WHERE role='member'").run(); db.prepare("INSERT OR IGNORE INTO schema_migrations(version) VALUES(36)").run(); db.prepare("INSERT OR IGNORE INTO schema_migrations(version) VALUES(37)").run(); applyColumnMigration(38, "sources", "attachment_archive_after_days", "ALTER TABLE sources ADD COLUMN attachment_archive_after_days INTEGER"); +const ftsSchema = ` +CREATE VIRTUAL TABLE IF NOT EXISTS posts_fts USING fts5(content, tags); +CREATE TRIGGER IF NOT EXISTS posts_fts_insert AFTER INSERT ON posts BEGIN + INSERT INTO posts_fts(rowid,content,tags) VALUES(new.id,new.content,new.tags_json); +END; +CREATE TRIGGER IF NOT EXISTS posts_fts_delete AFTER DELETE ON posts BEGIN + DELETE FROM posts_fts WHERE rowid=old.id; +END; +CREATE TRIGGER IF NOT EXISTS posts_fts_update AFTER UPDATE OF content,tags_json ON posts BEGIN + DELETE FROM posts_fts WHERE rowid=old.id; + INSERT INTO posts_fts(rowid,content,tags) VALUES(new.id,new.content,new.tags_json); +END; +`; +db.exec(ftsSchema); +const hasFtsMigration = db.prepare("SELECT 1 FROM schema_migrations WHERE version=39").get(); +if (!hasFtsMigration) { db.prepare("INSERT INTO posts_fts(rowid,content,tags) SELECT id,content,tags_json FROM posts").run(); db.prepare("INSERT INTO schema_migrations(version) VALUES(39)").run(); } +db.exec("CREATE INDEX IF NOT EXISTS posts_source_visibility_idx ON posts(source_id,visibility,hidden); CREATE INDEX IF NOT EXISTS posts_author_visibility_idx ON posts(author_id,visibility,hidden);"); +db.prepare("INSERT OR IGNORE INTO schema_migrations(version) VALUES(40)").run(); const admin = process.env.ADMIN_USERNAME; const adminPassword = process.env.ADMIN_PASSWORD; diff --git a/tests/sync.test.ts b/tests/sync.test.ts index dd769c6..ce08ae1 100644 --- a/tests/sync.test.ts +++ b/tests/sync.test.ts @@ -16,7 +16,7 @@ test("applies tracked migrations and deduplicates active pull jobs", async () => const { queuePull } = await import("../lib/sync"); const { notify } = await import("../lib/notifications"); const migrations = db.prepare("SELECT version FROM schema_migrations ORDER BY version").all() as { version: number }[]; - assert.deepEqual(migrations.map((item) => item.version), Array.from({ length: 38 }, (_, index) => index + 1)); + assert.deepEqual(migrations.map((item) => item.version), Array.from({ length: 40 }, (_, index) => index + 1)); const userId = Number(db.prepare("INSERT INTO users(username,password_hash) VALUES('sync-test','hash')").run().lastInsertRowid); const sourceId = Number(db.prepare("INSERT INTO sources(user_id,name,base_url,token_encrypted,is_enabled) VALUES(?,?,?,?,1)").run(userId, "Test", "https://example.test", "encrypted").lastInsertRowid); assert.equal(queuePull(sourceId, "manual"), true); @@ -25,6 +25,7 @@ test("applies tracked migrations and deduplicates active pull jobs", async () => assert.deepEqual(jobs, [{ kind: "pull", trigger: "manual", status: "queued" }]); const actorId = Number(db.prepare("INSERT INTO users(username,password_hash) VALUES('actor-test','hash')").run().lastInsertRowid); const postId = Number(db.prepare("INSERT INTO posts(author_id,content) VALUES(?,?)").run(userId, "Notification test").lastInsertRowid); + assert.equal(Number((db.prepare("SELECT count(*) AS count FROM posts_fts WHERE posts_fts MATCH 'Notification'").get() as { count: number }).count), 1); notify(userId, actorId, postId, "comment", "commented"); notify(userId, userId, postId, "reaction", "ignored"); assert.deepEqual(db.prepare("SELECT type,message FROM notifications WHERE user_id=?").all(userId), [{ type: "comment", message: "commented" }]); });