tangsongdayo/Mebbling
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

feat: archive aged attachment cache

Browse Source
This commit is contained in:
tangsongdayo
2026-07-19 05:59:41 +08:00
parent a23bc72dd3
commit 9caf598fcb
6 changed files with 14 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+1
View File
@@ -24,6 +24,7 @@
### Added ### Added
- Added optional per-source attachment archive age: cached files are replaced with their original remote links after the configured period.
- Added opt-in Discord/ntfy alerts for exhausted sync retries and stale signed webhooks, with per-event rate limiting. - Added opt-in Discord/ntfy alerts for exhausted sync retries and stale signed webhooks, with per-event rate limiting.
- Added safe signed Memos webhook rotation: create the replacement first, then remove the previous remote webhook before committing the new credentials. - Added safe signed Memos webhook rotation: create the replacement first, then remove the previous remote webhook before committing the new credentials.
- Added configurable retention cleanup and a password-confirmed self-service account deletion flow. - Added configurable retention cleanup and a password-confirmed self-service account deletion flow.
app/api/sources/[id]/manage/route.ts
+3 -3
View File
@@ -27,9 +27,9 @@ export async function POST(req: Request, { params }: { params: Promise<{ id: str
db.prepare("UPDATE sources SET sync_tags_json=?,sync_from=?,sync_to=?,sync_attachment_mode=?,sync_batch_size=?,sync_max_posts=?,sync_cursor=NULL,sync_imported_count=0,sync_run_id=NULL WHERE id=?").run(JSON.stringify(tags), from || null, to || null, attachmentMode, batchSize, maxPosts || null, id); db.prepare("UPDATE sources SET sync_tags_json=?,sync_from=?,sync_to=?,sync_attachment_mode=?,sync_batch_size=?,sync_max_posts=?,sync_cursor=NULL,sync_imported_count=0,sync_run_id=NULL WHERE id=?").run(JSON.stringify(tags), from || null, to || null, attachmentMode, batchSize, maxPosts || null, id);
queuePull(id, "manual"); queuePull(id, "manual");
} else if (action === "set-attachment-storage") { } else if (action === "set-attachment-storage") {
if (!owner) throw new Error("Only the owner can change attachment storage"); const mode = String(form.get("mode") || "remote"); const quotaMiB = Number(form.get("quotaMiB") || 100); if (!owner) throw new Error("Only the owner can change attachment storage"); const mode = String(form.get("mode") || "remote"); const quotaMiB = Number(form.get("quotaMiB") || 100); const archiveAfterDays = Number(form.get("archiveAfterDays") || 0);
if (!["remote", "images", "all"].includes(mode) || !Number.isFinite(quotaMiB) || quotaMiB < 10 || quotaMiB > 10_240) throw new Error("Invalid attachment storage settings"); if (!["remote", "images", "all"].includes(mode) || !Number.isFinite(quotaMiB) || quotaMiB < 10 || quotaMiB > 10_240 || !Number.isInteger(archiveAfterDays) || archiveAfterDays < 0 || archiveAfterDays > 3650) throw new Error("Invalid attachment storage settings");
db.prepare("UPDATE sources SET attachment_storage_mode=?,attachment_cache_limit_bytes=?,attachment_cache_error=NULL WHERE id=?").run(mode, Math.round(quotaMiB * 1024 * 1024), id); queuePull(id, "manual"); db.prepare("UPDATE sources SET attachment_storage_mode=?,attachment_cache_limit_bytes=?,attachment_archive_after_days=?,attachment_cache_error=NULL WHERE id=?").run(mode, Math.round(quotaMiB * 1024 * 1024), archiveAfterDays || null, id); queuePull(id, "manual");
} else if (action === "test-connection") { } else if (action === "test-connection") {
if (!owner) throw new Error("Only the owner can test the connection"); if (!owner) throw new Error("Only the owner can test the connection");
try { try {
app/dashboard/page.tsx
+3 -3
View File
@@ -5,14 +5,14 @@ import { PublishForm } from "./publish-form";
import { WebhookControl } from "./webhook-control"; import { WebhookControl } from "./webhook-control";
import { InviteControl } from "./invite-control"; import { InviteControl } from "./invite-control";
type Source = { id: number; name: string; base_url: string; integration_type: "memos" | "rss"; rss_feed_url: string | null; sync_status: string; last_synced_at: string | null; last_error: string | null; webhook_secret_hash: string | null; webhook_mode: string; last_webhook_at: string | null; owner_id: number; membership_role: "owner" | "editor" | "viewer"; is_enabled: number; disabled_at: string | null; sync_tags_json: string; sync_from: string | null; sync_to: string | null; sync_attachment_mode: "all" | "images" | "none"; sync_batch_size: number; sync_max_posts: number | null; sync_cursor: string | null; sync_imported_count: number; attachment_storage_mode: "remote" | "images" | "all"; attachment_cache_limit_bytes: number; attachment_cache_error: string | null; remote_display_name: string | null; remote_avatar_url: string | null; last_connection_at: string | null; last_connection_error: string | null }; type Source = { id: number; name: string; base_url: string; integration_type: "memos" | "rss"; rss_feed_url: string | null; sync_status: string; last_synced_at: string | null; last_error: string | null; webhook_secret_hash: string | null; webhook_mode: string; last_webhook_at: string | null; owner_id: number; membership_role: "owner" | "editor" | "viewer"; is_enabled: number; disabled_at: string | null; sync_tags_json: string; sync_from: string | null; sync_to: string | null; sync_attachment_mode: "all" | "images" | "none"; sync_batch_size: number; sync_max_posts: number | null; sync_cursor: string | null; sync_imported_count: number; attachment_storage_mode: "remote" | "images" | "all"; attachment_cache_limit_bytes: number; attachment_archive_after_days: number | null; attachment_cache_error: string | null; remote_display_name: string | null; remote_avatar_url: string | null; last_connection_at: string | null; last_connection_error: string | null };
type Job = { id: number; kind: string; trigger: string | null; status: string; attempts: number; last_error: string | null; created_at: string; finished_at: string | null }; type Job = { id: number; kind: string; trigger: string | null; status: string; attempts: number; last_error: string | null; created_at: string; finished_at: string | null };
export const dynamic = "force-dynamic"; export const dynamic = "force-dynamic";
export default async function Dashboard({ searchParams }: { searchParams: Promise<{ error?: string; source?: string; sync?: string }> }) { export default async function Dashboard({ searchParams }: { searchParams: Promise<{ error?: string; source?: string; sync?: string }> }) {
const query = await searchParams; const user = await getSession(); if (!user) redirect("/login"); const query = await searchParams; const user = await getSession(); if (!user) redirect("/login");
const sourceRows = db.prepare("SELECT s.id,s.name,s.base_url,s.integration_type,s.rss_feed_url,s.sync_status,s.last_synced_at,s.last_error,s.webhook_secret_hash,s.webhook_mode,s.last_webhook_at,s.user_id AS owner_id,sm.role AS membership_role,s.is_enabled,s.disabled_at,s.sync_tags_json,s.sync_from,s.sync_to,s.sync_attachment_mode,s.sync_batch_size,s.sync_max_posts,s.sync_cursor,s.sync_imported_count,s.attachment_storage_mode,s.attachment_cache_limit_bytes,s.attachment_cache_error,s.remote_display_name,s.remote_avatar_url,s.last_connection_at,s.last_connection_error FROM sources s JOIN source_members sm ON sm.source_id=s.id WHERE sm.user_id=? ORDER BY s.id DESC").all(user.id) as Source[]; const sourceRows = db.prepare("SELECT s.id,s.name,s.base_url,s.integration_type,s.rss_feed_url,s.sync_status,s.last_synced_at,s.last_error,s.webhook_secret_hash,s.webhook_mode,s.last_webhook_at,s.user_id AS owner_id,sm.role AS membership_role,s.is_enabled,s.disabled_at,s.sync_tags_json,s.sync_from,s.sync_to,s.sync_attachment_mode,s.sync_batch_size,s.sync_max_posts,s.sync_cursor,s.sync_imported_count,s.attachment_storage_mode,s.attachment_cache_limit_bytes,s.attachment_archive_after_days,s.attachment_cache_error,s.remote_display_name,s.remote_avatar_url,s.last_connection_at,s.last_connection_error FROM sources s JOIN source_members sm ON sm.source_id=s.id WHERE sm.user_id=? ORDER BY s.id DESC").all(user.id) as Source[];
const sources = sourceRows.map((source) => ({ ...source, syncTags: (() => { try { return JSON.parse(source.sync_tags_json) as string[]; } catch { return []; } })(), members: db.prepare("SELECT u.username,u.id,sm.role FROM source_members sm JOIN users u ON u.id=sm.user_id WHERE sm.source_id=? ORDER BY sm.role DESC,u.username").all(source.id) as { username: string; id: number; role: string }[], jobs: db.prepare("SELECT id,kind,trigger,status,attempts,last_error,created_at,finished_at FROM sync_jobs WHERE source_id=? ORDER BY id DESC LIMIT 5").all(source.id) as Job[] })); const sources = sourceRows.map((source) => ({ ...source, syncTags: (() => { try { return JSON.parse(source.sync_tags_json) as string[]; } catch { return []; } })(), members: db.prepare("SELECT u.username,u.id,sm.role FROM source_members sm JOIN users u ON u.id=sm.user_id WHERE sm.source_id=? ORDER BY sm.role DESC,u.username").all(source.id) as { username: string; id: number; role: string }[], jobs: db.prepare("SELECT id,kind,trigger,status,attempts,last_error,created_at,finished_at FROM sync_jobs WHERE source_id=? ORDER BY id DESC LIMIT 5").all(source.id) as Job[] }));
const publishSources = sources.filter((source) => source.is_enabled && source.integration_type === "memos" && ["owner", "editor"].includes(source.membership_role)); const publishSources = sources.filter((source) => source.is_enabled && source.integration_type === "memos" && ["owner", "editor"].includes(source.membership_role));
return <><style>{`form:has(input[name="action"][value="rename"]){display:none}`}</style> return <><style>{`form:has(input[name="action"][value="rename"]){display:none}`}</style>
@@ -29,7 +29,7 @@ export default async function Dashboard({ searchParams }: { searchParams: Promis
{source.owner_id === user.id ? <> {source.owner_id === user.id ? <>
{source.integration_type === "memos" && <WebhookControl sourceId={source.id} configured={Boolean(source.webhook_secret_hash)} automatic={source.webhook_mode === "signed"} />} {source.integration_type === "memos" && <WebhookControl sourceId={source.id} configured={Boolean(source.webhook_secret_hash)} automatic={source.webhook_mode === "signed"} />}
<InviteControl sourceId={source.id} /> <InviteControl sourceId={source.id} />
<form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="set-attachment-storage" /><label>附件保存策略<select name="mode" defaultValue={source.attachment_storage_mode}><option value="remote">遠端連結（不佔 Hub 空間）</option><option value="images">只快取圖片</option><option value="all">完整備份附件</option></select></label><label>快取配額（MiB）<input name="quotaMiB" type="number" min="10" max="10240" defaultValue={Math.round(source.attachment_cache_limit_bytes / 1024 / 1024)} /></label><button>儲存附件策略並同步</button></form> <form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="set-attachment-storage" /><label>附件保存策略<select name="mode" defaultValue={source.attachment_storage_mode}><option value="remote">遠端連結（不佔 Hub 空間）</option><option value="images">只快取圖片</option><option value="all">完整備份附件</option></select></label><label>快取配額（MiB）<input name="quotaMiB" type="number" min="10" max="10240" defaultValue={Math.round(source.attachment_cache_limit_bytes / 1024 / 1024)} /></label><label>快取封存天數（0 為不封存）<input name="archiveAfterDays" type="number" min="0" max="3650" defaultValue={source.attachment_archive_after_days || 0} /></label><button>儲存附件策略並同步</button></form>
<details><summary>來源管理</summary><form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="rename" /><label>顯示名稱<input name="name" defaultValue={source.name} required maxLength={80} /></label><button>儲存名稱</button></form><form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="set-sync-rules" /><label>只同步標籤（逗號分隔，留白為全部）<input name="tags" defaultValue={source.syncTags.join(", ")} placeholder="旅行, 技術" /></label><div className="row"><label>開始日期<input name="from" type="date" defaultValue={source.sync_from || ""} /></label><label>結束日期<input name="to" type="date" defaultValue={source.sync_to || ""} /></label></div><label>附件<select name="attachmentMode" defaultValue={source.sync_attachment_mode}><option value="all">同步全部附件</option><option value="images">僅同步圖片</option><option value="none">不同步附件</option></select></label><div className="row"><label>每批貼文數<input name="batchSize" type="number" min="10" max="100" defaultValue={source.sync_batch_size} /></label><label>首次匯入上限（0 為不限）<input name="maxPosts" type="number" min="0" max="100000" defaultValue={source.sync_max_posts || 0} /></label></div><button>儲存同步規則並從頭批次同步</button></form><form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="test-connection" /><button>測試 Memos 連線</button></form><form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="set-enabled" /><input type="hidden" name="enabled" value={source.is_enabled ? "0" : "1"} /><button className={source.is_enabled ? "danger" : ""}>{source.is_enabled ? "停用來源" : "啟用來源"}</button></form>{source.members.length > 1 && <form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="transfer" /><label>轉移建立者<select name="username" required defaultValue=""> <option value="" disabled>選擇成員</option>{source.members.filter((member) => member.id !== user.id).map((member) => <option key={member.id} value={member.username}>{member.username}</option>)}</select></label><button>轉移所有權</button></form>}<form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="delete" /><button className="danger">刪除來源與遠端鏡像貼文</button></form></details> <details><summary>來源管理</summary><form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="rename" /><label>顯示名稱<input name="name" defaultValue={source.name} required maxLength={80} /></label><button>儲存名稱</button></form><form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="set-sync-rules" /><label>只同步標籤（逗號分隔，留白為全部）<input name="tags" defaultValue={source.syncTags.join(", ")} placeholder="旅行, 技術" /></label><div className="row"><label>開始日期<input name="from" type="date" defaultValue={source.sync_from || ""} /></label><label>結束日期<input name="to" type="date" defaultValue={source.sync_to || ""} /></label></div><label>附件<select name="attachmentMode" defaultValue={source.sync_attachment_mode}><option value="all">同步全部附件</option><option value="images">僅同步圖片</option><option value="none">不同步附件</option></select></label><div className="row"><label>每批貼文數<input name="batchSize" type="number" min="10" max="100" defaultValue={source.sync_batch_size} /></label><label>首次匯入上限（0 為不限）<input name="maxPosts" type="number" min="0" max="100000" defaultValue={source.sync_max_posts || 0} /></label></div><button>儲存同步規則並從頭批次同步</button></form><form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="test-connection" /><button>測試 Memos 連線</button></form><form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="set-enabled" /><input type="hidden" name="enabled" value={source.is_enabled ? "0" : "1"} /><button className={source.is_enabled ? "danger" : ""}>{source.is_enabled ? "停用來源" : "啟用來源"}</button></form>{source.members.length > 1 && <form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="transfer" /><label>轉移建立者<select name="username" required defaultValue=""> <option value="" disabled>選擇成員</option>{source.members.filter((member) => member.id !== user.id).map((member) => <option key={member.id} value={member.username}>{member.username}</option>)}</select></label><button>轉移所有權</button></form>}<form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="delete" /><button className="danger">刪除來源與遠端鏡像貼文</button></form></details>
</> : <form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="leave" /><button className="danger">離開共享來源</button></form>} </> : <form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="leave" /><button className="danger">離開共享來源</button></form>}
<form action="/api/sync" method="post"><input type="hidden" name="sourceId" value={source.id} /><button disabled={!source.is_enabled}>{source.is_enabled ? "立即同步" : "來源已停用"}</button></form> <form action="/api/sync" method="post"><input type="hidden" name="sourceId" value={source.id} /><button disabled={!source.is_enabled}>{source.is_enabled ? "立即同步" : "來源已停用"}</button></form>
lib/db.ts
+1
View File
@@ -151,6 +151,7 @@ db.prepare("INSERT OR IGNORE INTO schema_migrations(version) VALUES(35)").run();
db.prepare("UPDATE source_members SET role='editor' WHERE role='member'").run(); db.prepare("UPDATE source_members SET role='editor' WHERE role='member'").run();
db.prepare("INSERT OR IGNORE INTO schema_migrations(version) VALUES(36)").run(); db.prepare("INSERT OR IGNORE INTO schema_migrations(version) VALUES(36)").run();
db.prepare("INSERT OR IGNORE INTO schema_migrations(version) VALUES(37)").run(); db.prepare("INSERT OR IGNORE INTO schema_migrations(version) VALUES(37)").run();
applyColumnMigration(38, "sources", "attachment_archive_after_days", "ALTER TABLE sources ADD COLUMN attachment_archive_after_days INTEGER");
const admin = process.env.ADMIN_USERNAME; const admin = process.env.ADMIN_USERNAME;
const adminPassword = process.env.ADMIN_PASSWORD; const adminPassword = process.env.ADMIN_PASSWORD;
tests/sync.test.ts
+1 -1
View File
@@ -16,7 +16,7 @@ test("applies tracked migrations and deduplicates active pull jobs", async () =>
const { queuePull } = await import("../lib/sync"); const { queuePull } = await import("../lib/sync");
const { notify } = await import("../lib/notifications"); const { notify } = await import("../lib/notifications");
const migrations = db.prepare("SELECT version FROM schema_migrations ORDER BY version").all() as { version: number }[]; const migrations = db.prepare("SELECT version FROM schema_migrations ORDER BY version").all() as { version: number }[];
assert.deepEqual(migrations.map((item) => item.version), Array.from({ length: 37 }, (_, index) => index + 1)); assert.deepEqual(migrations.map((item) => item.version), Array.from({ length: 38 }, (_, index) => index + 1));
const userId = Number(db.prepare("INSERT INTO users(username,password_hash) VALUES('sync-test','hash')").run().lastInsertRowid); const userId = Number(db.prepare("INSERT INTO users(username,password_hash) VALUES('sync-test','hash')").run().lastInsertRowid);
const sourceId = Number(db.prepare("INSERT INTO sources(user_id,name,base_url,token_encrypted,is_enabled) VALUES(?,?,?,?,1)").run(userId, "Test", "https://example.test", "encrypted").lastInsertRowid); const sourceId = Number(db.prepare("INSERT INTO sources(user_id,name,base_url,token_encrypted,is_enabled) VALUES(?,?,?,?,1)").run(userId, "Test", "https://example.test", "encrypted").lastInsertRowid);
assert.equal(queuePull(sourceId, "manual"), true); assert.equal(queuePull(sourceId, "manual"), true);
worker/index.ts
+5 -1
View File
@@ -8,7 +8,7 @@ import { recordError } from "../lib/observability";
import { fetchRss } from "../lib/rss"; import { fetchRss } from "../lib/rss";
import { sendAlert } from "../lib/alerts"; import { sendAlert } from "../lib/alerts";
type Source = { id: number; user_id: number; base_url: string; token_encrypted: string; remote_user: string | null; integration_type: "memos" | "rss"; rss_feed_url: string | null; is_enabled: number; sync_tags_json: string; sync_from: string | null; sync_to: string | null; sync_attachment_mode: "all" | "images" | "none"; attachment_storage_mode: "remote" | "images" | "all"; attachment_cache_limit_bytes: number; sync_batch_size: number; sync_max_posts: number | null; sync_cursor: string | null; sync_imported_count: number; sync_run_id: string | null }; type Source = { id: number; user_id: number; base_url: string; token_encrypted: string; remote_user: string | null; integration_type: "memos" | "rss"; rss_feed_url: string | null; is_enabled: number; sync_tags_json: string; sync_from: string | null; sync_to: string | null; sync_attachment_mode: "all" | "images" | "none"; attachment_storage_mode: "remote" | "images" | "all"; attachment_cache_limit_bytes: number; attachment_archive_after_days: number | null; sync_batch_size: number; sync_max_posts: number | null; sync_cursor: string | null; sync_imported_count: number; sync_run_id: string | null };
type Job = { id: number; source_id: number; kind: "pull" | "push"; payload_json: string | null; attempts: number }; type Job = { id: number; source_id: number; kind: "pull" | "push"; payload_json: string | null; attempts: number };
function remoteAttachmentUrl(attachment: any, baseUrl: string) { function remoteAttachmentUrl(attachment: any, baseUrl: string) {
@@ -39,6 +39,10 @@ async function cacheAttachments(source: Source, attachments: any[]) {
} }
async function cleanupCache(source: Source) { async function cleanupCache(source: Source) {
if (source.attachment_archive_after_days) {
const oldPosts = db.prepare("SELECT id,attachments_json FROM posts WHERE source_id=? AND COALESCE(remote_created_at,created_at)<datetime('now', ?) AND attachments_json LIKE ?").all(source.id, `-${source.attachment_archive_after_days} days`, `%/uploads/cache/source-${source.id}/%`) as { id: number; attachments_json: string }[];
for (const post of oldPosts) { try { const attachments = (JSON.parse(post.attachments_json) as any[]).map((item) => String(item.url || "").startsWith(`/uploads/cache/source-${source.id}/`) && item.originalUrl ? { ...item, url: item.originalUrl } : item); db.prepare("UPDATE posts SET attachments_json=?,updated_at=CURRENT_TIMESTAMP WHERE id=?").run(JSON.stringify(attachments), post.id); } catch {} }
}
const directory = join(process.cwd(), "public", "uploads", "cache", `source-${source.id}`); let names: string[]; try { names = await readdir(directory); } catch { return; } const directory = join(process.cwd(), "public", "uploads", "cache", `source-${source.id}`); let names: string[]; try { names = await readdir(directory); } catch { return; }
const used = new Set<string>(); for (const row of db.prepare("SELECT attachments_json FROM posts WHERE source_id=?").all(source.id) as { attachments_json: string }[]) { try { for (const attachment of JSON.parse(row.attachments_json) as any[]) { const url = String(attachment.url || ""); if (url.startsWith(`/uploads/cache/source-${source.id}/`)) used.add(url.split("/").at(-1)!); } } catch {} } const used = new Set<string>(); for (const row of db.prepare("SELECT attachments_json FROM posts WHERE source_id=?").all(source.id) as { attachments_json: string }[]) { try { for (const attachment of JSON.parse(row.attachments_json) as any[]) { const url = String(attachment.url || ""); if (url.startsWith(`/uploads/cache/source-${source.id}/`)) used.add(url.split("/").at(-1)!); } } catch {} }
await Promise.all(names.filter((name) => !used.has(name)).map((name) => unlink(join(directory, name)).catch(() => undefined))); await Promise.all(names.filter((name) => !used.has(name)).map((name) => unlink(join(directory, name)).catch(() => undefined)));