feat: add sync alert webhooks

.env.example

@@ -14,6 +14,8 @@ SYNC_INTERVAL_MINUTES=60
 NOTIFICATION_RETENTION_DAYS=0
 READING_HISTORY_RETENTION_DAYS=0
 AUDIT_RETENTION_DAYS=365
+# Optional Discord webhook URL or ntfy topic URL (for example https://ntfy.sh/my-private-topic).
+ALERT_WEBHOOK_URL=
 # Optional: create the first Memos source for the bootstrap admin.
 SEED_MEMOS_NAME=
 SEED_MEMOS_URL=

CHANGELOG.md

@@ -24,6 +24,7 @@
 
 ### Added
 
+- Added opt-in Discord/ntfy alerts for exhausted sync retries and stale signed webhooks, with per-event rate limiting.
 - Added safe signed Memos webhook rotation: create the replacement first, then remove the previous remote webhook before committing the new credentials.
 - Added configurable retention cleanup and a password-confirmed self-service account deletion flow.
 - Added append-only audit records for source management, invitations, posting, and administrator moderation.

docs/OPERATIONS.md

@@ -58,3 +58,7 @@ Hub 原生附件預設只接受圖片、PDF、純文字與 Markdown。若要串
 `NOTIFICATION_RETENTION_DAYS`、`READING_HISTORY_RETENTION_DAYS` 與 `AUDIT_RETENTION_DAYS` 可設定保存天數；`0` 代表無限期保存。Worker 每日清理一次。到期或已使用的邀請連結會自動移除。
 
 使用者可在「帳號設定」以目前密碼與 `DELETE` 文字確認刪除帳號及個人資料。若帳號仍擁有來源，必須先轉移或刪除來源，避免誤刪共享內容。
+
+## 外部告警
+
+設定 `ALERT_WEBHOOK_URL` 後，Worker 會在同步重試耗盡、或簽章 Webhook 超過 7 天未收到事件時發送告警。支援 Discord incoming webhook 或 ntfy topic URL；同一事件每小時最多通知一次。

lib/alerts.ts

@@ -0,0 +1,11 @@
+import { db } from "@/lib/db";
+
+function allowed(bucket: string, seconds = 3600) {
+  const now = Math.floor(Date.now() / 1000); const row = db.prepare("SELECT reset_at FROM rate_limits WHERE bucket=?").get(bucket) as { reset_at: number } | undefined;
+  if (row && row.reset_at > now) return false;
+  db.prepare("INSERT INTO rate_limits(bucket,count,reset_at) VALUES(?,1,?) ON CONFLICT(bucket) DO UPDATE SET count=count+1,reset_at=excluded.reset_at").run(bucket, now + seconds); return true;
+}
+export async function sendAlert(bucket: string, title: string, message: string) {
+  const url = process.env.ALERT_WEBHOOK_URL?.trim(); if (!url || !allowed(`alert:${bucket}`)) return false;
+  try { const isNtfy = /(^|\.)ntfy\.sh\//.test(new URL(url).hostname + new URL(url).pathname); const response = await fetch(url, isNtfy ? { method: "POST", headers: { Title: title, Priority: "high" }, body: message, signal: AbortSignal.timeout(10_000) } : { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ content: `**${title}**\n${message}` }), signal: AbortSignal.timeout(10_000) }); if (!response.ok) throw new Error(`Alert ${response.status}`); return true; } catch { return false; }
+}

worker/index.ts

@@ -6,6 +6,7 @@ import { decrypt } from "../lib/crypto";
 import { createMemo, createRemoteFile, getMemosIdentity, listMemos, memoUrl, setMemoAttachments } from "../lib/memos";
 import { recordError } from "../lib/observability";
 import { fetchRss } from "../lib/rss";
+import { sendAlert } from "../lib/alerts";
 
 type Source = { id: number; user_id: number; base_url: string; token_encrypted: string; remote_user: string | null; integration_type: "memos" | "rss"; rss_feed_url: string | null; is_enabled: number; sync_tags_json: string; sync_from: string | null; sync_to: string | null; sync_attachment_mode: "all" | "images" | "none"; attachment_storage_mode: "remote" | "images" | "all"; attachment_cache_limit_bytes: number; sync_batch_size: number; sync_max_posts: number | null; sync_cursor: string | null; sync_imported_count: number; sync_run_id: string | null };
 type Job = { id: number; source_id: number; kind: "pull" | "push"; payload_json: string | null; attempts: number };
@@ -96,6 +97,7 @@ async function run() {
     recordError("sync", error, { sourceId: source.id, jobId: job.id, kind: job.kind, attempts: job.attempts });
     db.prepare("UPDATE sync_jobs SET status=?,last_error=?,finished_at=CASE WHEN ? THEN CURRENT_TIMESTAMP ELSE NULL END,run_after=CASE WHEN ? THEN run_after ELSE datetime('now','+5 minutes') END WHERE id=?").run(exhausted ? "failed" : "queued", message, exhausted ? 1 : 0, exhausted ? 1 : 0, job.id);
     db.prepare("UPDATE sources SET sync_status='error',last_error=?,last_connection_error=? WHERE id=?").run(message, message, source.id);
+    if (exhausted) void sendAlert(`sync:${source.id}`, "Mebbling 同步失敗", `來源 #${source.id} 已重試 5 次仍失敗：${message}`);
   }
 }
 
@@ -111,4 +113,9 @@ function retention() {
   cleanup("notifications", "created_at", days("NOTIFICATION_RETENTION_DAYS")); cleanup("reading_history", "last_read_at", days("READING_HISTORY_RETENTION_DAYS")); cleanup("audit_events", "created_at", days("AUDIT_RETENTION_DAYS"));
 }
 
-let lastRetention = 0; setInterval(() => { schedule(); if (Date.now() - lastRetention > 86_400_000) { retention(); lastRetention = Date.now(); } void run(); }, 5000); schedule(); retention(); void run();
+function webhookHealth() {
+  const stale = db.prepare("SELECT id,name,last_webhook_at FROM sources WHERE is_enabled=1 AND webhook_mode='signed' AND (last_webhook_at IS NULL OR last_webhook_at<datetime('now','-7 days'))").all() as { id: number; name: string; last_webhook_at: string | null }[];
+  for (const source of stale) void sendAlert(`webhook:${source.id}`, "Mebbling Webhook 未收到事件", `來源「${source.name}」超過 7 天未收到 Webhook；目前仍會以定期 API 同步校正。`);
+}
+
+let lastRetention = 0, lastWebhookHealth = 0; setInterval(() => { schedule(); if (Date.now() - lastRetention > 86_400_000) { retention(); lastRetention = Date.now(); } if (Date.now() - lastWebhookHealth > 3_600_000) { webhookHealth(); lastWebhookHealth = Date.now(); } void run(); }, 5000); schedule(); retention(); webhookHealth(); void run();