feat: batch large source imports
This commit is contained in:
@@ -24,6 +24,7 @@
|
||||
|
||||
### Added
|
||||
|
||||
- Added resumable, page-token-based Memos imports with configurable batch size and first-import limit; posts are only hidden after a complete scan.
|
||||
- Added owner/member-only JSON and Markdown exports for individual posts and complete sources.
|
||||
- Added read-only RSS sources: validate a feed URL, queue recurring imports, and show imported posts under their RSS source rather than a Hub account.
|
||||
|
||||
|
||||
@@ -21,9 +21,9 @@ export async function POST(req: Request, { params }: { params: Promise<{ id: str
|
||||
} else if (action === "set-sync-rules") {
|
||||
if (!owner) throw new Error("Only the owner can change sync rules");
|
||||
const tags = String(form.get("tags") || "").split(",").map((tag) => tag.trim().replace(/^#/, "")).filter(Boolean).slice(0, 20);
|
||||
const from = String(form.get("from") || ""); const to = String(form.get("to") || ""); const attachmentMode = String(form.get("attachmentMode") || "all");
|
||||
if ((from && !/^\d{4}-\d{2}-\d{2}$/.test(from)) || (to && !/^\d{4}-\d{2}-\d{2}$/.test(to)) || (from && to && from > to) || !["all", "images", "none"].includes(attachmentMode)) throw new Error("Invalid sync rules");
|
||||
db.prepare("UPDATE sources SET sync_tags_json=?,sync_from=?,sync_to=?,sync_attachment_mode=? WHERE id=?").run(JSON.stringify(tags), from || null, to || null, attachmentMode, id);
|
||||
const from = String(form.get("from") || ""); const to = String(form.get("to") || ""); const attachmentMode = String(form.get("attachmentMode") || "all"); const batchSize = Number(form.get("batchSize") || 100); const maxPosts = Number(form.get("maxPosts") || 0);
|
||||
if ((from && !/^\d{4}-\d{2}-\d{2}$/.test(from)) || (to && !/^\d{4}-\d{2}-\d{2}$/.test(to)) || (from && to && from > to) || !["all", "images", "none"].includes(attachmentMode) || !Number.isInteger(batchSize) || batchSize < 10 || batchSize > 100 || !Number.isInteger(maxPosts) || maxPosts < 0 || maxPosts > 100000) throw new Error("Invalid sync rules");
|
||||
db.prepare("UPDATE sources SET sync_tags_json=?,sync_from=?,sync_to=?,sync_attachment_mode=?,sync_batch_size=?,sync_max_posts=?,sync_cursor=NULL,sync_imported_count=0,sync_run_id=NULL WHERE id=?").run(JSON.stringify(tags), from || null, to || null, attachmentMode, batchSize, maxPosts || null, id);
|
||||
queuePull(id, "manual");
|
||||
} else if (action === "set-attachment-storage") {
|
||||
if (!owner) throw new Error("Only the owner can change attachment storage"); const mode = String(form.get("mode") || "remote"); const quotaMiB = Number(form.get("quotaMiB") || 100);
|
||||
|
||||
@@ -4,14 +4,14 @@ import { db } from "@/lib/db";
|
||||
import { PublishForm } from "./publish-form";
|
||||
import { WebhookControl } from "./webhook-control";
|
||||
|
||||
type Source = { id: number; name: string; base_url: string; integration_type: "memos" | "rss"; rss_feed_url: string | null; sync_status: string; last_synced_at: string | null; last_error: string | null; webhook_secret_hash: string | null; webhook_mode: string; last_webhook_at: string | null; owner_id: number; is_enabled: number; disabled_at: string | null; sync_tags_json: string; sync_from: string | null; sync_to: string | null; sync_attachment_mode: "all" | "images" | "none"; attachment_storage_mode: "remote" | "images" | "all"; attachment_cache_limit_bytes: number; attachment_cache_error: string | null; remote_display_name: string | null; remote_avatar_url: string | null; last_connection_at: string | null; last_connection_error: string | null };
|
||||
type Source = { id: number; name: string; base_url: string; integration_type: "memos" | "rss"; rss_feed_url: string | null; sync_status: string; last_synced_at: string | null; last_error: string | null; webhook_secret_hash: string | null; webhook_mode: string; last_webhook_at: string | null; owner_id: number; is_enabled: number; disabled_at: string | null; sync_tags_json: string; sync_from: string | null; sync_to: string | null; sync_attachment_mode: "all" | "images" | "none"; sync_batch_size: number; sync_max_posts: number | null; sync_cursor: string | null; sync_imported_count: number; attachment_storage_mode: "remote" | "images" | "all"; attachment_cache_limit_bytes: number; attachment_cache_error: string | null; remote_display_name: string | null; remote_avatar_url: string | null; last_connection_at: string | null; last_connection_error: string | null };
|
||||
type Job = { id: number; kind: string; trigger: string | null; status: string; attempts: number; last_error: string | null; created_at: string; finished_at: string | null };
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
export default async function Dashboard({ searchParams }: { searchParams: Promise<{ error?: string; source?: string; sync?: string }> }) {
|
||||
const query = await searchParams; const user = await getSession(); if (!user) redirect("/login");
|
||||
const sourceRows = db.prepare("SELECT s.id,s.name,s.base_url,s.integration_type,s.rss_feed_url,s.sync_status,s.last_synced_at,s.last_error,s.webhook_secret_hash,s.webhook_mode,s.last_webhook_at,s.user_id AS owner_id,s.is_enabled,s.disabled_at,s.sync_tags_json,s.sync_from,s.sync_to,s.sync_attachment_mode,s.attachment_storage_mode,s.attachment_cache_limit_bytes,s.attachment_cache_error,s.remote_display_name,s.remote_avatar_url,s.last_connection_at,s.last_connection_error FROM sources s JOIN source_members sm ON sm.source_id=s.id WHERE sm.user_id=? ORDER BY s.id DESC").all(user.id) as Source[];
|
||||
const sourceRows = db.prepare("SELECT s.id,s.name,s.base_url,s.integration_type,s.rss_feed_url,s.sync_status,s.last_synced_at,s.last_error,s.webhook_secret_hash,s.webhook_mode,s.last_webhook_at,s.user_id AS owner_id,s.is_enabled,s.disabled_at,s.sync_tags_json,s.sync_from,s.sync_to,s.sync_attachment_mode,s.sync_batch_size,s.sync_max_posts,s.sync_cursor,s.sync_imported_count,s.attachment_storage_mode,s.attachment_cache_limit_bytes,s.attachment_cache_error,s.remote_display_name,s.remote_avatar_url,s.last_connection_at,s.last_connection_error FROM sources s JOIN source_members sm ON sm.source_id=s.id WHERE sm.user_id=? ORDER BY s.id DESC").all(user.id) as Source[];
|
||||
const sources = sourceRows.map((source) => ({ ...source, syncTags: (() => { try { return JSON.parse(source.sync_tags_json) as string[]; } catch { return []; } })(), members: db.prepare("SELECT u.username,u.id,sm.role FROM source_members sm JOIN users u ON u.id=sm.user_id WHERE sm.source_id=? ORDER BY sm.role DESC,u.username").all(source.id) as { username: string; id: number; role: string }[], jobs: db.prepare("SELECT id,kind,trigger,status,attempts,last_error,created_at,finished_at FROM sync_jobs WHERE source_id=? ORDER BY id DESC LIMIT 5").all(source.id) as Job[] }));
|
||||
const publishSources = sources.filter((source) => source.is_enabled && source.integration_type === "memos");
|
||||
return <><style>{`form:has(input[name="action"][value="rename"]){display:none}`}</style>
|
||||
@@ -24,11 +24,11 @@ export default async function Dashboard({ searchParams }: { searchParams: Promis
|
||||
<section className="card"><h2>連接 RSS</h2><form action="/api/sources/rss" method="post"><label>RSS Feed 網址<input name="feedUrl" type="url" required placeholder="https://example.com/feed.xml" /></label><button>驗證並訂閱</button></form><p className="muted">RSS 為唯讀來源，不需要 API Key，也不會回寫原網站。同步時會更新 Feed 內公開的項目。</p></section>
|
||||
<section><h2>已連接來源</h2>{sources.map((source) => <article className="card" key={source.id}>
|
||||
<div className="space"><strong>{source.name}</strong><span className="tag">{source.is_enabled ? source.sync_status : "disabled"}</span></div>
|
||||
<p className="meta">來源 ID：{source.id}<br />{source.integration_type === "rss" ? source.rss_feed_url : source.base_url}{source.remote_display_name && <><br />Memos 帳號：{source.remote_avatar_url && <img className="avatar" src={source.remote_avatar_url} alt="" />} {source.remote_display_name}</>}<br />成員：{source.members.map((member) => `${member.username}${member.role === "owner" ? "（建立者）" : ""}`).join("、")}<br />上次同步：{source.last_synced_at || "尚未完成"}<br />{source.integration_type === "memos" && <>附件保存：{source.attachment_storage_mode === "remote" ? "遠端連結" : source.attachment_storage_mode === "images" ? "只快取圖片" : "完整備份"}（配額 {Math.round(source.attachment_cache_limit_bytes / 1024 / 1024)} MiB）<br />連線：{source.last_connection_at ? `最近成功：${new Date(source.last_connection_at + "Z").toLocaleString("zh-TW")}` : "尚未測試"}<br />Webhook：{source.webhook_secret_hash ? (source.last_webhook_at ? (Date.now() - new Date(source.last_webhook_at + "Z").getTime() > 7 * 24 * 60 * 60 * 1000 ? `警示：超過 7 天未收到（最近：${new Date(source.last_webhook_at + "Z").toLocaleString("zh-TW")}）` : `健康（最近收到：${new Date(source.last_webhook_at + "Z").toLocaleString("zh-TW")}）`) : "已建立 URL，尚未收到呼叫") : "尚未建立 URL"}</>}{!source.is_enabled && <><br />已停用：{source.disabled_at ? new Date(source.disabled_at + "Z").toLocaleString("zh-TW") : "是"}</>}{source.last_error && <><br /><span className="error">同步：{source.last_error}</span></>}{source.last_connection_error && <><br /><span className="error">連線：{source.last_connection_error}</span></>}{source.attachment_cache_error && <><br /><span className="error">附件快取：{source.attachment_cache_error}</span></>}</p>
|
||||
<p className="meta">來源 ID：{source.id}<br />{source.integration_type === "rss" ? source.rss_feed_url : source.base_url}{source.remote_display_name && <><br />Memos 帳號：{source.remote_avatar_url && <img className="avatar" src={source.remote_avatar_url} alt="" />} {source.remote_display_name}</>}<br />成員：{source.members.map((member) => `${member.username}${member.role === "owner" ? "（建立者）" : ""}`).join("、")}<br />上次同步：{source.last_synced_at || "尚未完成"}{source.sync_cursor && <>（批次匯入中：{source.sync_imported_count} 篇）</>}<br />{source.integration_type === "memos" && <>附件保存：{source.attachment_storage_mode === "remote" ? "遠端連結" : source.attachment_storage_mode === "images" ? "只快取圖片" : "完整備份"}（配額 {Math.round(source.attachment_cache_limit_bytes / 1024 / 1024)} MiB）<br />連線：{source.last_connection_at ? `最近成功：${new Date(source.last_connection_at + "Z").toLocaleString("zh-TW")}` : "尚未測試"}<br />Webhook：{source.webhook_secret_hash ? (source.last_webhook_at ? (Date.now() - new Date(source.last_webhook_at + "Z").getTime() > 7 * 24 * 60 * 60 * 1000 ? `警示：超過 7 天未收到（最近：${new Date(source.last_webhook_at + "Z").toLocaleString("zh-TW")}）` : `健康（最近收到：${new Date(source.last_webhook_at + "Z").toLocaleString("zh-TW")}）`) : "已建立 URL，尚未收到呼叫") : "尚未建立 URL"}</>}{!source.is_enabled && <><br />已停用：{source.disabled_at ? new Date(source.disabled_at + "Z").toLocaleString("zh-TW") : "是"}</>}{source.last_error && <><br /><span className="error">同步：{source.last_error}</span></>}{source.last_connection_error && <><br /><span className="error">連線：{source.last_connection_error}</span></>}{source.attachment_cache_error && <><br /><span className="error">附件快取：{source.attachment_cache_error}</span></>}</p>
|
||||
{source.owner_id === user.id ? <>
|
||||
{source.integration_type === "memos" && <WebhookControl sourceId={source.id} configured={Boolean(source.webhook_secret_hash)} automatic={source.webhook_mode === "signed"} />}
|
||||
<form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="set-attachment-storage" /><label>附件保存策略<select name="mode" defaultValue={source.attachment_storage_mode}><option value="remote">遠端連結（不佔 Hub 空間）</option><option value="images">只快取圖片</option><option value="all">完整備份附件</option></select></label><label>快取配額（MiB）<input name="quotaMiB" type="number" min="10" max="10240" defaultValue={Math.round(source.attachment_cache_limit_bytes / 1024 / 1024)} /></label><button>儲存附件策略並同步</button></form>
|
||||
<details><summary>來源管理</summary><form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="rename" /><label>顯示名稱<input name="name" defaultValue={source.name} required maxLength={80} /></label><button>儲存名稱</button></form><form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="set-sync-rules" /><label>只同步標籤（逗號分隔，留白為全部）<input name="tags" defaultValue={source.syncTags.join(", ")} placeholder="旅行, 技術" /></label><div className="row"><label>開始日期<input name="from" type="date" defaultValue={source.sync_from || ""} /></label><label>結束日期<input name="to" type="date" defaultValue={source.sync_to || ""} /></label></div><label>附件<select name="attachmentMode" defaultValue={source.sync_attachment_mode}><option value="all">同步全部附件</option><option value="images">僅同步圖片</option><option value="none">不同步附件</option></select></label><button>儲存同步規則並同步</button></form><form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="test-connection" /><button>測試 Memos 連線</button></form><form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="set-enabled" /><input type="hidden" name="enabled" value={source.is_enabled ? "0" : "1"} /><button className={source.is_enabled ? "danger" : ""}>{source.is_enabled ? "停用來源" : "啟用來源"}</button></form>{source.members.length > 1 && <form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="transfer" /><label>轉移建立者<select name="username" required defaultValue=""> <option value="" disabled>選擇成員</option>{source.members.filter((member) => member.id !== user.id).map((member) => <option key={member.id} value={member.username}>{member.username}</option>)}</select></label><button>轉移所有權</button></form>}<form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="delete" /><button className="danger">刪除來源與遠端鏡像貼文</button></form></details>
|
||||
<details><summary>來源管理</summary><form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="rename" /><label>顯示名稱<input name="name" defaultValue={source.name} required maxLength={80} /></label><button>儲存名稱</button></form><form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="set-sync-rules" /><label>只同步標籤（逗號分隔，留白為全部）<input name="tags" defaultValue={source.syncTags.join(", ")} placeholder="旅行, 技術" /></label><div className="row"><label>開始日期<input name="from" type="date" defaultValue={source.sync_from || ""} /></label><label>結束日期<input name="to" type="date" defaultValue={source.sync_to || ""} /></label></div><label>附件<select name="attachmentMode" defaultValue={source.sync_attachment_mode}><option value="all">同步全部附件</option><option value="images">僅同步圖片</option><option value="none">不同步附件</option></select></label><div className="row"><label>每批貼文數<input name="batchSize" type="number" min="10" max="100" defaultValue={source.sync_batch_size} /></label><label>首次匯入上限（0 為不限）<input name="maxPosts" type="number" min="0" max="100000" defaultValue={source.sync_max_posts || 0} /></label></div><button>儲存同步規則並從頭批次同步</button></form><form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="test-connection" /><button>測試 Memos 連線</button></form><form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="set-enabled" /><input type="hidden" name="enabled" value={source.is_enabled ? "0" : "1"} /><button className={source.is_enabled ? "danger" : ""}>{source.is_enabled ? "停用來源" : "啟用來源"}</button></form>{source.members.length > 1 && <form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="transfer" /><label>轉移建立者<select name="username" required defaultValue=""> <option value="" disabled>選擇成員</option>{source.members.filter((member) => member.id !== user.id).map((member) => <option key={member.id} value={member.username}>{member.username}</option>)}</select></label><button>轉移所有權</button></form>}<form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="delete" /><button className="danger">刪除來源與遠端鏡像貼文</button></form></details>
|
||||
</> : <form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="leave" /><button className="danger">離開共享來源</button></form>}
|
||||
<form action="/api/sync" method="post"><input type="hidden" name="sourceId" value={source.id} /><button disabled={!source.is_enabled}>{source.is_enabled ? "立即同步" : "來源已停用"}</button></form>
|
||||
<details><summary>最近同步工作</summary>{source.jobs.length ? <ul className="job-list">{source.jobs.map((job) => <li key={job.id}><strong>{job.kind}</strong> · {job.trigger || "legacy"} · <span className="tag">{job.status}</span> · 嘗試 {job.attempts} 次<br /><span className="meta">建立：{new Date(job.created_at + "Z").toLocaleString("zh-TW")}{job.finished_at && `；完成：${new Date(job.finished_at + "Z").toLocaleString("zh-TW")}`}</span>{job.last_error && <><br /><span className="error">{job.last_error}</span></>}</li>)}</ul> : <p className="muted">尚無同步工作。</p>}</details>
|
||||
|
||||
@@ -130,6 +130,12 @@ applyColumnMigration(25, "sources", "webhook_remote_name", "ALTER TABLE sources
|
||||
applyColumnMigration(26, "sources", "webhook_signing_secret_encrypted", "ALTER TABLE sources ADD COLUMN webhook_signing_secret_encrypted TEXT");
|
||||
applyColumnMigration(27, "sources", "integration_type", "ALTER TABLE sources ADD COLUMN integration_type TEXT NOT NULL DEFAULT 'memos'");
|
||||
applyColumnMigration(28, "sources", "rss_feed_url", "ALTER TABLE sources ADD COLUMN rss_feed_url TEXT");
|
||||
applyColumnMigration(29, "sources", "sync_batch_size", "ALTER TABLE sources ADD COLUMN sync_batch_size INTEGER NOT NULL DEFAULT 100");
|
||||
applyColumnMigration(30, "sources", "sync_max_posts", "ALTER TABLE sources ADD COLUMN sync_max_posts INTEGER");
|
||||
applyColumnMigration(31, "sources", "sync_cursor", "ALTER TABLE sources ADD COLUMN sync_cursor TEXT");
|
||||
applyColumnMigration(32, "sources", "sync_imported_count", "ALTER TABLE sources ADD COLUMN sync_imported_count INTEGER NOT NULL DEFAULT 0");
|
||||
applyColumnMigration(33, "sources", "sync_run_id", "ALTER TABLE sources ADD COLUMN sync_run_id TEXT");
|
||||
applyColumnMigration(34, "posts", "last_seen_sync_run", "ALTER TABLE posts ADD COLUMN last_seen_sync_run TEXT");
|
||||
|
||||
const admin = process.env.ADMIN_USERNAME;
|
||||
const adminPassword = process.env.ADMIN_PASSWORD;
|
||||
|
||||
+8
-6
@@ -19,26 +19,28 @@ export async function getMemosIdentity(baseUrl: string, token: string) {
|
||||
return user;
|
||||
}
|
||||
export function memoUrl(baseUrl: string, memoName: string) { const id = memoName.split("/").at(-1); return id ? `${baseUrl.replace(/\/$/, "")}/m/${encodeURIComponent(id)}` : null; }
|
||||
export async function listMemos(baseUrl: string, token: string, rules: MemosSyncRules = {}) {
|
||||
const all: MemosMemo[] = []; let pageToken = "";
|
||||
export type MemosPage = { memos: MemosMemo[]; nextPageToken: string };
|
||||
export async function listMemos(baseUrl: string, token: string, rules: MemosSyncRules = {}, page: { pageToken?: string; pageSize?: number } = {}): Promise<MemosPage> {
|
||||
let pageToken = page.pageToken || "";
|
||||
const tags = rules.tags?.filter(Boolean) || []; const mode = rules.attachmentMode || "all";
|
||||
const quote = (value: string) => JSON.stringify(value); const filters = [rules.creator && `creator == ${quote(rules.creator)}`, `visibility == "PUBLIC"`, ...tags.map((tag) => `tags.exists(t, t == ${quote(tag)})`)].filter(Boolean).join(" && ");
|
||||
let useServerFilter = Boolean(filters);
|
||||
do { const params = new URLSearchParams({ pageSize: "100" }); if (pageToken) params.set("pageToken", pageToken); if (useServerFilter && filters) params.set("filter", filters); let data: any;
|
||||
do { const params = new URLSearchParams({ pageSize: String(Math.max(10, Math.min(100, page.pageSize || 100))) }); if (pageToken) params.set("pageToken", pageToken); if (useServerFilter && filters) params.set("filter", filters); let data: any;
|
||||
try { data = await (await request(`${base(baseUrl)}/memos?${params}`, token)).json(); }
|
||||
catch (error) { if (!pageToken && useServerFilter && error instanceof Error && error.message.startsWith("Memos API 400")) { useServerFilter = false; continue; } throw error; }
|
||||
all.push(...(data.memos || [])); pageToken = data.nextPageToken || ""; } while (pageToken);
|
||||
return all.filter((memo) => {
|
||||
const memos = (data.memos || []).filter((memo: MemosMemo) => {
|
||||
if (memo.visibility !== "PUBLIC") return false;
|
||||
if (rules.creator && memo.creator !== rules.creator) return false;
|
||||
if (tags.length && !tags.every((tag) => memo.tags?.includes(tag))) return false;
|
||||
const created = memo.createTime?.slice(0, 10); if (rules.from && (!created || created < rules.from)) return false; if (rules.to && (!created || created > rules.to)) return false;
|
||||
return true;
|
||||
}).map((memo) => {
|
||||
}).map((memo: MemosMemo) => {
|
||||
if (mode === "all") return memo;
|
||||
const onlyImages = <T extends { type?: string }>(items: T[] | undefined) => mode === "none" ? [] : (items || []).filter((item) => item.type?.startsWith("image/"));
|
||||
return { ...memo, attachments: onlyImages(memo.attachments), resources: onlyImages(memo.resources) };
|
||||
});
|
||||
return { memos, nextPageToken: data.nextPageToken || "" };
|
||||
} while (true);
|
||||
}
|
||||
export async function createMemo(baseUrl: string, token: string, memo: Pick<MemosMemo, "content" | "visibility"> & { attachments?: unknown[]; resources?: unknown[] }) {
|
||||
return (await request(`${base(baseUrl)}/memos`, token, { method: "POST", body: JSON.stringify({ state: "NORMAL", ...memo }) })).json() as Promise<MemosMemo>;
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
import { db } from "@/lib/db";
|
||||
|
||||
export type SyncTrigger = "manual" | "webhook" | "scheduled" | "source-created";
|
||||
export type SyncTrigger = "manual" | "webhook" | "scheduled" | "source-created" | "batch";
|
||||
|
||||
/** Queue one pull per source at a time. Returns true only when a new job was created. */
|
||||
export function queuePull(sourceId: number, trigger: SyncTrigger, payload: unknown = {}) {
|
||||
|
||||
+1
-1
@@ -16,7 +16,7 @@ test("applies tracked migrations and deduplicates active pull jobs", async () =>
|
||||
const { queuePull } = await import("../lib/sync");
|
||||
const { notify } = await import("../lib/notifications");
|
||||
const migrations = db.prepare("SELECT version FROM schema_migrations ORDER BY version").all() as { version: number }[];
|
||||
assert.deepEqual(migrations.map((item) => item.version), Array.from({ length: 28 }, (_, index) => index + 1));
|
||||
assert.deepEqual(migrations.map((item) => item.version), Array.from({ length: 34 }, (_, index) => index + 1));
|
||||
const userId = Number(db.prepare("INSERT INTO users(username,password_hash) VALUES('sync-test','hash')").run().lastInsertRowid);
|
||||
const sourceId = Number(db.prepare("INSERT INTO sources(user_id,name,base_url,token_encrypted,is_enabled) VALUES(?,?,?,?,1)").run(userId, "Test", "https://example.test", "encrypted").lastInsertRowid);
|
||||
assert.equal(queuePull(sourceId, "manual"), true);
|
||||
|
||||
+15
-13
@@ -1,13 +1,13 @@
|
||||
import { readFile, mkdir, readdir, unlink, writeFile } from "node:fs/promises";
|
||||
import { extname, join } from "node:path";
|
||||
import { createHash } from "node:crypto";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { db } from "../lib/db";
|
||||
import { decrypt } from "../lib/crypto";
|
||||
import { createMemo, createRemoteFile, getMemosIdentity, listMemos, memoUrl, setMemoAttachments } from "../lib/memos";
|
||||
import { recordError } from "../lib/observability";
|
||||
import { fetchRss } from "../lib/rss";
|
||||
|
||||
type Source = { id: number; user_id: number; base_url: string; token_encrypted: string; remote_user: string | null; integration_type: "memos" | "rss"; rss_feed_url: string | null; is_enabled: number; sync_tags_json: string; sync_from: string | null; sync_to: string | null; sync_attachment_mode: "all" | "images" | "none"; attachment_storage_mode: "remote" | "images" | "all"; attachment_cache_limit_bytes: number };
|
||||
type Source = { id: number; user_id: number; base_url: string; token_encrypted: string; remote_user: string | null; integration_type: "memos" | "rss"; rss_feed_url: string | null; is_enabled: number; sync_tags_json: string; sync_from: string | null; sync_to: string | null; sync_attachment_mode: "all" | "images" | "none"; attachment_storage_mode: "remote" | "images" | "all"; attachment_cache_limit_bytes: number; sync_batch_size: number; sync_max_posts: number | null; sync_cursor: string | null; sync_imported_count: number; sync_run_id: string | null };
|
||||
type Job = { id: number; source_id: number; kind: "pull" | "push"; payload_json: string | null; attempts: number };
|
||||
|
||||
function remoteAttachmentUrl(attachment: any, baseUrl: string) {
|
||||
@@ -43,25 +43,27 @@ async function cleanupCache(source: Source) {
|
||||
await Promise.all(names.filter((name) => !used.has(name)).map((name) => unlink(join(directory, name)).catch(() => undefined)));
|
||||
}
|
||||
|
||||
async function upsertRemote(source: Source, memo: any) {
|
||||
async function upsertRemote(source: Source, memo: any, syncRun: string) {
|
||||
const cached = await cacheAttachments(source, memo.attachments || memo.resources || []); const tags = JSON.stringify(memo.tags || []), attachments = JSON.stringify(cached.items);
|
||||
db.prepare(`INSERT INTO posts(source_id,author_id,remote_memo_name,content,visibility,tags_json,attachments_json,origin,remote_created_at,remote_updated_at,sync_status,hidden,remote_url) VALUES(?,?,?,?,?,?,?,?,?,?, 'synced',0,?) ON CONFLICT(source_id,remote_memo_name) DO UPDATE SET content=excluded.content,visibility=excluded.visibility,tags_json=excluded.tags_json,attachments_json=excluded.attachments_json,remote_updated_at=excluded.remote_updated_at,remote_url=excluded.remote_url,hidden=0,updated_at=CURRENT_TIMESTAMP`).run(source.id, source.user_id, memo.name, memo.content, memo.visibility, tags, attachments, "memos", memo.createTime || null, memo.updateTime || null, memoUrl(source.base_url, memo.name));
|
||||
db.prepare(`INSERT INTO posts(source_id,author_id,remote_memo_name,content,visibility,tags_json,attachments_json,origin,remote_created_at,remote_updated_at,sync_status,hidden,remote_url,last_seen_sync_run) VALUES(?,?,?,?,?,?,?,?,?,?, 'synced',0,?,?) ON CONFLICT(source_id,remote_memo_name) DO UPDATE SET content=excluded.content,visibility=excluded.visibility,tags_json=excluded.tags_json,attachments_json=excluded.attachments_json,remote_updated_at=excluded.remote_updated_at,remote_url=excluded.remote_url,last_seen_sync_run=excluded.last_seen_sync_run,hidden=0,updated_at=CURRENT_TIMESTAMP`).run(source.id, source.user_id, memo.name, memo.content, memo.visibility, tags, attachments, "memos", memo.createTime || null, memo.updateTime || null, memoUrl(source.base_url, memo.name), syncRun);
|
||||
return cached.errors;
|
||||
}
|
||||
|
||||
async function pull(source: Source) {
|
||||
if (source.integration_type === "rss") { const items = await fetchRss(source.rss_feed_url || ""); for (const item of items) db.prepare("INSERT INTO posts(source_id,author_id,remote_memo_name,content,visibility,tags_json,attachments_json,origin,remote_created_at,remote_updated_at,sync_status,hidden,remote_url) VALUES(?,?,?,?,?,'[]','[]','rss',?,?, 'synced',0,?) ON CONFLICT(source_id,remote_memo_name) DO UPDATE SET content=excluded.content,remote_created_at=excluded.remote_created_at,remote_url=excluded.remote_url,hidden=0,updated_at=CURRENT_TIMESTAMP").run(source.id,source.user_id,`rss:${item.id}`,item.content,"PUBLIC",item.publishedAt,item.publishedAt,item.link); const names=items.map((item)=>`rss:${item.id}`); if(names.length) db.prepare(`UPDATE posts SET hidden=1 WHERE source_id=? AND remote_memo_name NOT IN (${names.map(()=>"?").join(",")})`).run(source.id,...names); db.prepare("UPDATE sources SET sync_status='synced',last_synced_at=CURRENT_TIMESTAMP,last_error=NULL WHERE id=?").run(source.id); return; }
|
||||
if (source.integration_type === "rss") { const items = await fetchRss(source.rss_feed_url || ""); for (const item of items) db.prepare("INSERT INTO posts(source_id,author_id,remote_memo_name,content,visibility,tags_json,attachments_json,origin,remote_created_at,remote_updated_at,sync_status,hidden,remote_url) VALUES(?,?,?,?,?,'[]','[]','rss',?,?, 'synced',0,?) ON CONFLICT(source_id,remote_memo_name) DO UPDATE SET content=excluded.content,remote_created_at=excluded.remote_created_at,remote_url=excluded.remote_url,hidden=0,updated_at=CURRENT_TIMESTAMP").run(source.id,source.user_id,`rss:${item.id}`,item.content,"PUBLIC",item.publishedAt,item.publishedAt,item.link); const names=items.map((item)=>`rss:${item.id}`); if(names.length) db.prepare(`UPDATE posts SET hidden=1 WHERE source_id=? AND remote_memo_name NOT IN (${names.map(()=>"?").join(",")})`).run(source.id,...names); db.prepare("UPDATE sources SET sync_status='synced',last_synced_at=CURRENT_TIMESTAMP,last_error=NULL WHERE id=?").run(source.id); return false; }
|
||||
const token = decrypt(source.token_encrypted); const identity = await getMemosIdentity(source.base_url, token); const creator = identity.name;
|
||||
const rules = { creator, tags: JSON.parse(source.sync_tags_json || "[]") as string[], from: source.sync_from, to: source.sync_to, attachmentMode: source.sync_attachment_mode };
|
||||
const memos = await listMemos(source.base_url, token, rules);
|
||||
const cacheErrors: string[] = []; for (const memo of memos) cacheErrors.push(...await upsertRemote(source, memo));
|
||||
const names = memos.map((memo) => memo.name);
|
||||
if (names.length) { const placeholders = names.map(() => "?").join(","); db.prepare(`UPDATE posts SET hidden=1,updated_at=CURRENT_TIMESTAMP WHERE source_id=? AND remote_memo_name IS NOT NULL AND remote_memo_name NOT IN (${placeholders})`).run(source.id, ...names); }
|
||||
else db.prepare("UPDATE posts SET hidden=1,updated_at=CURRENT_TIMESTAMP WHERE source_id=? AND remote_memo_name IS NOT NULL").run(source.id);
|
||||
const remaining = source.sync_max_posts ? Math.max(0, source.sync_max_posts - source.sync_imported_count) : source.sync_batch_size;
|
||||
if (remaining === 0) { db.prepare("UPDATE sources SET sync_cursor=NULL,sync_status='synced',last_synced_at=CURRENT_TIMESTAMP WHERE id=?").run(source.id); return false; }
|
||||
const page = await listMemos(source.base_url, token, rules, { pageToken: source.sync_cursor || "", pageSize: Math.min(source.sync_batch_size, remaining) }); const memos = page.memos; const syncRun = source.sync_run_id || randomUUID();
|
||||
const cacheErrors: string[] = []; for (const memo of memos) cacheErrors.push(...await upsertRemote(source, memo, syncRun));
|
||||
const imported = source.sync_imported_count + memos.length; const capped = Boolean(source.sync_max_posts && imported >= source.sync_max_posts); const complete = !page.nextPageToken && !capped; const more = Boolean(page.nextPageToken) && !capped;
|
||||
if (complete) db.prepare("UPDATE posts SET hidden=1,updated_at=CURRENT_TIMESTAMP WHERE source_id=? AND origin='memos' AND COALESCE(last_seen_sync_run,'')<>?").run(source.id, syncRun);
|
||||
await cleanupCache(source);
|
||||
const avatar = identity.avatarUrl || identity.avatar || null; const avatarUrl = avatar?.startsWith("/") ? `${source.base_url.replace(/\/$/, "")}${avatar}` : avatar;
|
||||
const name = identity.displayName || identity.nickname || identity.username || identity.name;
|
||||
db.prepare("UPDATE sources SET name=?,remote_user=?,sync_status='synced',last_synced_at=CURRENT_TIMESTAMP,last_error=NULL,last_connection_at=CURRENT_TIMESTAMP,last_connection_error=NULL,attachment_cache_error=?,remote_display_name=?,remote_avatar_url=? WHERE id=?").run(name, creator, cacheErrors.length ? `${cacheErrors.length} 個附件快取失敗；可按「立即同步」重試。` : null, name, avatarUrl, source.id);
|
||||
db.prepare("UPDATE sources SET name=?,remote_user=?,sync_status=?,last_synced_at=CASE WHEN ? THEN NULL ELSE CURRENT_TIMESTAMP END,last_error=NULL,last_connection_at=CURRENT_TIMESTAMP,last_connection_error=NULL,attachment_cache_error=?,remote_display_name=?,remote_avatar_url=?,sync_cursor=?,sync_imported_count=?,sync_run_id=? WHERE id=?").run(name, creator, more ? 'importing' : 'synced', more ? 1 : 0, cacheErrors.length ? `${cacheErrors.length} 個附件快取失敗；可按「立即同步」重試。` : null, name, avatarUrl, more ? page.nextPageToken : null, more ? imported : 0, more ? syncRun : null, source.id);
|
||||
return more;
|
||||
}
|
||||
|
||||
async function push(source: Source, payload: any) {
|
||||
@@ -86,9 +88,9 @@ async function run() {
|
||||
const source = db.prepare("SELECT * FROM sources WHERE id=?").get(job.source_id) as Source | undefined;
|
||||
if (!source || !source.is_enabled) { db.prepare("UPDATE sync_jobs SET status='cancelled',last_error='Source is disabled or deleted',finished_at=CURRENT_TIMESTAMP WHERE id=?").run(job.id); return; }
|
||||
try {
|
||||
if (job.kind === "pull") await pull(source); else if (job.kind === "push") await push(source, JSON.parse(job.payload_json || "{}"));
|
||||
const more = job.kind === "pull" ? await pull(source) : false; if (job.kind === "push") await push(source, JSON.parse(job.payload_json || "{}"));
|
||||
db.prepare("UPDATE sync_jobs SET status='done',finished_at=CURRENT_TIMESTAMP WHERE id=?").run(job.id);
|
||||
db.prepare("UPDATE sources SET sync_status='synced',last_error=NULL,last_synced_at=CURRENT_TIMESTAMP WHERE id=?").run(source.id);
|
||||
if (more) db.prepare("INSERT INTO sync_jobs(source_id,kind,trigger) VALUES(?,'pull','batch')").run(source.id); else db.prepare("UPDATE sources SET sync_status='synced',last_error=NULL,last_synced_at=CURRENT_TIMESTAMP WHERE id=?").run(source.id);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Sync failure"; const exhausted = job.attempts + 1 >= 5;
|
||||
recordError("sync", error, { sourceId: source.id, jobId: job.id, kind: job.kind, attempts: job.attempts });
|
||||
|
||||
Reference in New Issue
Block a user