diff --git a/CHANGELOG.md b/CHANGELOG.md index a2050ae..3391452 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Added SQLite FTS5 indexing for public-content search, maintained automatically as posts change. - Added a repeatable FTS search benchmark and documented PostgreSQL migration decision criteria. - Added persistent alert delivery with HTTPS validation, exponential-backoff retries, and Prometheus delivery-state metrics. +- Added verified backup reports plus optional age-encrypted rclone offsite copies. ### Fixed diff --git a/README.md b/README.md index 3efdf76..a8dfba6 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,8 @@ https://你的網域/api/sync/webhook/來源ID/隨機密鑰 ├── data/ # SQLite 資料庫持久化資料 ├── Dockerfile # Web/Worker 共用映像檔 ├── docker-compose.yml # web + worker 服務與 volume 掛載 +├── scripts/ # 備份、驗證、還原與效能基準工具 +├── docs/ # 維運與擴展文件 └── .env.example # 環境變數範本 ``` diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 330c075..542838f 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -13,7 +13,11 @@ - `hub.db`:由正在執行的 SQLite 資料庫建立的一致性備份。 - `uploads.tar.gz`:Hub 本機上傳的附件。 -每次備份都會執行 SQLite `integrity_check`、驗證附件壓縮檔,並在 `SHA256SUMS` 記錄雜湊。`data/backups/` 已由 Git 排除。 +每次備份都會執行 SQLite `integrity_check`、驗證附件壓縮檔,並在 `SHA256SUMS` 記錄雜湊。`data/backups/` 已由 Git 排除。也可隨時執行不改動正式資料的驗證: + +```bash +./scripts/verify-backup.sh data/backups/<時間> +``` 可設定保留與異地複製(例如掛載的 NAS、加密磁碟或 rclone 掛載點): @@ -21,6 +25,16 @@ BACKUP_RETENTION_DAYS=30 BACKUP_OFFSITE_DIR=/mnt/nas/mebbling ./scripts/backup.sh ``` +若異地目的地可能由他人讀取,使用 `age` 加密。設定加密後,異地只會收到 `backup.tar.gz.age`,本機仍保留可供快速還原的已驗證備份: + +```bash +BACKUP_AGE_RECIPIENT=age1你的收件人公鑰 \ +BACKUP_RCLONE_REMOTE='remote:bucket/mebbling' \ +BACKUP_RETENTION_DAYS=30 ./scripts/backup.sh +``` + +`BACKUP_RCLONE_REMOTE` 可使用已在主機設定好的 rclone S3、B2、SFTP 等 remote;腳本會在缺少 `age` 或 `rclone` 時安全失敗,不會假裝已完成異地備份。加密檔要還原時,先以持有的 age identity 解密並解壓為原本的備份目錄,再使用下列還原命令。請把 `verify-backup.sh` 的成功輸出保留在 cron log 中,作為每日可用性驗證報告;完整還原演練仍應定期在隔離環境執行。 + 在 WSL 主機安裝每日 03:15 排程: ```bash @@ -61,7 +75,7 @@ Hub 原生附件預設只接受圖片、PDF、純文字與 Markdown。若要串 ## 外部告警 -設定 `ALERT_WEBHOOK_URL` 後,Worker 會在同步重試耗盡、或簽章 Webhook 超過 7 天未收到事件時發送告警。支援 Discord incoming webhook 或 ntfy topic URL;同一事件每小時最多通知一次。 +設定 `ALERT_WEBHOOK_URL` 後,Worker 會在同步重試耗盡、或簽章 Webhook 超過 7 天未收到事件時發送告警。支援 HTTPS Discord incoming webhook 或 ntfy topic URL;同一事件每小時最多通知一次。告警會保存於 SQLite,失敗時以指數退避重試、最多五次;`/api/metrics` 的 `mebbling_alert_deliveries` 可監看最終失敗。 ## 監控指標與健康檢查 diff --git a/scripts/backup.sh b/scripts/backup.sh index a3ee085..bb39398 100644 --- a/scripts/backup.sh +++ b/scripts/backup.sh @@ -3,6 +3,7 @@ set -euo pipefail # Creates a consistent SQLite backup through the running web container, then archives Hub uploads. # Optional: BACKUP_RETENTION_DAYS=30 BACKUP_OFFSITE_DIR=/mnt/backup/mebbling ./scripts/backup.sh +# For encrypted remote copies: BACKUP_AGE_RECIPIENT=age1... BACKUP_RCLONE_REMOTE='remote:bucket/mebbling' ./scripts/backup.sh root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$root_dir" stamp="$(date +%Y%m%d-%H%M%S)" @@ -23,12 +24,27 @@ docker compose exec -T -e BACKUP_PATH="/app/data/backups/$stamp/hub.db" web node ' tar -tzf "$backup_dir/uploads.tar.gz" >/dev/null (cd "$backup_dir" && sha256sum hub.db uploads.tar.gz > SHA256SUMS) +"$root_dir/scripts/verify-backup.sh" "$backup_dir" + +archive="$backup_dir/backup.tar.gz" +tar -czf "$archive" -C "$backup_dir" hub.db uploads.tar.gz SHA256SUMS +offsite_payload="$backup_dir" +if [[ -n "${BACKUP_AGE_RECIPIENT:-}" ]]; then + command -v age >/dev/null || { echo "age is required when BACKUP_AGE_RECIPIENT is set" >&2; exit 2; } + age -r "$BACKUP_AGE_RECIPIENT" -o "$archive.age" "$archive" + offsite_payload="$archive.age" +fi if [[ -n "${BACKUP_OFFSITE_DIR:-}" ]]; then destination="$BACKUP_OFFSITE_DIR/$stamp"; mkdir -p "$destination" - cp -a "$backup_dir/." "$destination/" + if [[ "$offsite_payload" == "$backup_dir" ]]; then cp -a "$backup_dir/." "$destination/"; else cp -a "$offsite_payload" "$destination/"; fi printf 'Copied verified backup to: %s\n' "$destination" fi +if [[ -n "${BACKUP_RCLONE_REMOTE:-}" ]]; then + command -v rclone >/dev/null || { echo "rclone is required when BACKUP_RCLONE_REMOTE is set" >&2; exit 2; } + if [[ "$offsite_payload" == "$backup_dir" ]]; then rclone copy "$backup_dir" "$BACKUP_RCLONE_REMOTE/$stamp"; else rclone copy "$offsite_payload" "$BACKUP_RCLONE_REMOTE/$stamp"; fi + printf 'Copied verified backup using rclone to: %s/%s\n' "$BACKUP_RCLONE_REMOTE" "$stamp" +fi if [[ -n "${BACKUP_RETENTION_DAYS:-}" ]]; then [[ "$BACKUP_RETENTION_DAYS" =~ ^[0-9]+$ ]] || { echo "BACKUP_RETENTION_DAYS must be a non-negative integer" >&2; exit 2; } find data/backups -mindepth 1 -maxdepth 1 -type d -mtime "+$BACKUP_RETENTION_DAYS" -exec rm -rf {} + diff --git a/scripts/restore.sh b/scripts/restore.sh index 80ed061..d9cff4f 100644 --- a/scripts/restore.sh +++ b/scripts/restore.sh @@ -4,8 +4,8 @@ set -euo pipefail if [[ $# -ne 1 ]]; then echo "Usage: CONFIRM_RESTORE=YES ./scripts/restore.sh data/backups/YYYYMMDD-HHMMSS" >&2; exit 2; fi if [[ "${CONFIRM_RESTORE:-}" != "YES" ]]; then echo "Refusing restore. Set CONFIRM_RESTORE=YES after verifying the backup path." >&2; exit 2; fi backup_dir="$1"; [[ -f "$backup_dir/hub.db" && -f "$backup_dir/uploads.tar.gz" && -f "$backup_dir/SHA256SUMS" ]] || { echo "Backup is incomplete" >&2; exit 2; } -(cd "$backup_dir" && sha256sum -c SHA256SUMS) root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"; cd "$root_dir" +"$root_dir/scripts/verify-backup.sh" "$backup_dir" docker compose down mkdir -p data public mv data/hub.db "data/hub.db.before-restore.$(date +%Y%m%d-%H%M%S)" 2>/dev/null || true diff --git a/scripts/verify-backup.sh b/scripts/verify-backup.sh new file mode 100644 index 0000000..39fa8e5 --- /dev/null +++ b/scripts/verify-backup.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 1 ]]; then echo "Usage: ./scripts/verify-backup.sh data/backups/YYYYMMDD-HHMMSS" >&2; exit 2; fi +root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +backup_dir="$(cd "$1" && pwd)" +[[ "$backup_dir" == "$root_dir"/* ]] || { echo "Backup must be inside the project directory" >&2; exit 2; } +[[ -f "$backup_dir/hub.db" && -f "$backup_dir/uploads.tar.gz" && -f "$backup_dir/SHA256SUMS" ]] || { echo "Backup is incomplete" >&2; exit 2; } +(cd "$backup_dir" && sha256sum -c SHA256SUMS) +tar -tzf "$backup_dir/uploads.tar.gz" >/dev/null +container_path="/app/${backup_dir#"$root_dir"/}/hub.db" +docker compose -f "$root_dir/docker-compose.yml" exec -T -e BACKUP_PATH="$container_path" web node -e ' + const Database = require("better-sqlite3"); const db = new Database(process.env.BACKUP_PATH, { readonly: true }); + const row = db.prepare("PRAGMA integrity_check").get(); db.close(); if (row.integrity_check !== "ok") { console.error("SQLite integrity check failed"); process.exit(1); } +' +printf 'Verified backup: %s\n' "$backup_dir"