feat: index public search with FTS5

CHANGELOG.md

@@ -9,6 +9,7 @@
 - Hardened incoming RSS/Atom sources with public-HTTPS validation, redirect checks, response-size limits, and safe XML declaration rejection.
 - Added production configuration validation for secrets, encryption keys, and the public HTTPS URL, plus an administrator-facing status check.
 - Added Prometheus-compatible metrics and distinct liveness/readiness health probes.
+- Added SQLite FTS5 indexing for public-content search, maintained automatically as posts change.
 
 ### Fixed
 

app/page.tsx

@@ -9,8 +9,9 @@ type Query = { q?: string; tag?: string; source?: string; author?: string; from?
 export default async function Home({ searchParams }: { searchParams: Promise<Query> }) {
   const query = await searchParams; const q = query.q?.trim() || ""; const tag = query.tag?.trim() || ""; const author = query.author?.trim() || ""; const sourceId = Number(query.source) || 0; const from = query.from || ""; const to = query.to || ""; const attachments = query.attachments === "1"; const page = Math.max(1, Number(query.page) || 1);
   const where = ["p.visibility='PUBLIC'", "p.hidden=0"]; const args: (string | number)[] = [];
-  if (q) { where.push("p.content LIKE ?"); args.push(`%${q}%`); } if (tag) { where.push("p.tags_json LIKE ?"); args.push(`%${JSON.stringify(tag).slice(1, -1)}%`); } if (author) { where.push("u.username LIKE ?"); args.push(`%${author}%`); } if (sourceId) { where.push("s.id=?"); args.push(sourceId); } if (from) { where.push("date(COALESCE(p.remote_created_at,p.created_at)) >= date(?)"); args.push(from); } if (to) { where.push("date(COALESCE(p.remote_created_at,p.created_at)) <= date(?)"); args.push(to); } if (attachments) where.push("p.attachments_json <> '[]'");
+  const ftsQuery = q.split(/\s+/).filter(Boolean).map((term) => `"${term.replaceAll('"', '""')}"`).join(" AND ");
-  const joins = " FROM posts p JOIN users u ON u.id=p.author_id LEFT JOIN sources s ON s.id=p.source_id "; const predicate = ` WHERE ${where.join(" AND ")}`;
+  if (q) { where.push("posts_fts MATCH ?"); args.push(ftsQuery); } if (tag) { where.push("p.tags_json LIKE ?"); args.push(`%${JSON.stringify(tag).slice(1, -1)}%`); } if (author) { where.push("u.username LIKE ?"); args.push(`%${author}%`); } if (sourceId) { where.push("s.id=?"); args.push(sourceId); } if (from) { where.push("date(COALESCE(p.remote_created_at,p.created_at)) >= date(?)"); args.push(from); } if (to) { where.push("date(COALESCE(p.remote_created_at,p.created_at)) <= date(?)"); args.push(to); } if (attachments) where.push("p.attachments_json <> '[]'");
+  const joins = ` FROM posts p ${q ? "JOIN posts_fts ON posts_fts.rowid=p.id" : ""} JOIN users u ON u.id=p.author_id LEFT JOIN sources s ON s.id=p.source_id `; const predicate = ` WHERE ${where.join(" AND ")}`;
   const total = Number((db.prepare(`SELECT count(*) count${joins}${predicate}`).get(...args) as { count: number }).count); const pages = Math.max(1, Math.ceil(total / pageSize)); const safePage = Math.min(page, pages);
   const posts = db.prepare(`SELECT p.*,u.username,s.name,s.remote_display_name,s.base_url AS source_base_url,(SELECT count(*) FROM comments c WHERE c.post_id=p.id AND c.hidden=0) comment_count,(SELECT count(*) FROM reactions r WHERE r.post_id=p.id) reaction_count${joins}${predicate} ORDER BY COALESCE(p.remote_created_at,p.created_at) DESC LIMIT ? OFFSET ?`).all(...args, pageSize, (safePage - 1) * pageSize) as PublicPost[];
   const sources = db.prepare("SELECT id,name FROM sources WHERE is_enabled=1 ORDER BY name").all() as { id: number; name: string }[];

lib/db.ts

@@ -154,6 +154,24 @@ db.prepare("UPDATE source_members SET role='editor' WHERE role='member'").run();
 db.prepare("INSERT OR IGNORE INTO schema_migrations(version) VALUES(36)").run();
 db.prepare("INSERT OR IGNORE INTO schema_migrations(version) VALUES(37)").run();
 applyColumnMigration(38, "sources", "attachment_archive_after_days", "ALTER TABLE sources ADD COLUMN attachment_archive_after_days INTEGER");
+const ftsSchema = `
+CREATE VIRTUAL TABLE IF NOT EXISTS posts_fts USING fts5(content, tags);
+CREATE TRIGGER IF NOT EXISTS posts_fts_insert AFTER INSERT ON posts BEGIN
+  INSERT INTO posts_fts(rowid,content,tags) VALUES(new.id,new.content,new.tags_json);
+END;
+CREATE TRIGGER IF NOT EXISTS posts_fts_delete AFTER DELETE ON posts BEGIN
+  DELETE FROM posts_fts WHERE rowid=old.id;
+END;
+CREATE TRIGGER IF NOT EXISTS posts_fts_update AFTER UPDATE OF content,tags_json ON posts BEGIN
+  DELETE FROM posts_fts WHERE rowid=old.id;
+  INSERT INTO posts_fts(rowid,content,tags) VALUES(new.id,new.content,new.tags_json);
+END;
+`;
+db.exec(ftsSchema);
+const hasFtsMigration = db.prepare("SELECT 1 FROM schema_migrations WHERE version=39").get();
+if (!hasFtsMigration) { db.prepare("INSERT INTO posts_fts(rowid,content,tags) SELECT id,content,tags_json FROM posts").run(); db.prepare("INSERT INTO schema_migrations(version) VALUES(39)").run(); }
+db.exec("CREATE INDEX IF NOT EXISTS posts_source_visibility_idx ON posts(source_id,visibility,hidden); CREATE INDEX IF NOT EXISTS posts_author_visibility_idx ON posts(author_id,visibility,hidden);");
+db.prepare("INSERT OR IGNORE INTO schema_migrations(version) VALUES(40)").run();
 
 const admin = process.env.ADMIN_USERNAME;
 const adminPassword = process.env.ADMIN_PASSWORD;

tests/sync.test.ts

@@ -16,7 +16,7 @@ test("applies tracked migrations and deduplicates active pull jobs", async () =>
   const { queuePull } = await import("../lib/sync");
   const { notify } = await import("../lib/notifications");
   const migrations = db.prepare("SELECT version FROM schema_migrations ORDER BY version").all() as { version: number }[];
-  assert.deepEqual(migrations.map((item) => item.version), Array.from({ length: 38 }, (_, index) => index + 1));
+  assert.deepEqual(migrations.map((item) => item.version), Array.from({ length: 40 }, (_, index) => index + 1));
   const userId = Number(db.prepare("INSERT INTO users(username,password_hash) VALUES('sync-test','hash')").run().lastInsertRowid);
   const sourceId = Number(db.prepare("INSERT INTO sources(user_id,name,base_url,token_encrypted,is_enabled) VALUES(?,?,?,?,1)").run(userId, "Test", "https://example.test", "encrypted").lastInsertRowid);
   assert.equal(queuePull(sourceId, "manual"), true);
@@ -25,6 +25,7 @@ test("applies tracked migrations and deduplicates active pull jobs", async () =>
   assert.deepEqual(jobs, [{ kind: "pull", trigger: "manual", status: "queued" }]);
   const actorId = Number(db.prepare("INSERT INTO users(username,password_hash) VALUES('actor-test','hash')").run().lastInsertRowid);
   const postId = Number(db.prepare("INSERT INTO posts(author_id,content) VALUES(?,?)").run(userId, "Notification test").lastInsertRowid);
+  assert.equal(Number((db.prepare("SELECT count(*) AS count FROM posts_fts WHERE posts_fts MATCH 'Notification'").get() as { count: number }).count), 1);
   notify(userId, actorId, postId, "comment", "commented"); notify(userId, userId, postId, "reaction", "ignored");
   assert.deepEqual(db.prepare("SELECT type,message FROM notifications WHERE user_id=?").all(userId), [{ type: "comment", message: "commented" }]);
 });