feat: batch large source imports

lib/db.ts

@@ -130,6 +130,12 @@ applyColumnMigration(25, "sources", "webhook_remote_name", "ALTER TABLE sources
 applyColumnMigration(26, "sources", "webhook_signing_secret_encrypted", "ALTER TABLE sources ADD COLUMN webhook_signing_secret_encrypted TEXT");
 applyColumnMigration(27, "sources", "integration_type", "ALTER TABLE sources ADD COLUMN integration_type TEXT NOT NULL DEFAULT 'memos'");
 applyColumnMigration(28, "sources", "rss_feed_url", "ALTER TABLE sources ADD COLUMN rss_feed_url TEXT");
+applyColumnMigration(29, "sources", "sync_batch_size", "ALTER TABLE sources ADD COLUMN sync_batch_size INTEGER NOT NULL DEFAULT 100");
+applyColumnMigration(30, "sources", "sync_max_posts", "ALTER TABLE sources ADD COLUMN sync_max_posts INTEGER");
+applyColumnMigration(31, "sources", "sync_cursor", "ALTER TABLE sources ADD COLUMN sync_cursor TEXT");
+applyColumnMigration(32, "sources", "sync_imported_count", "ALTER TABLE sources ADD COLUMN sync_imported_count INTEGER NOT NULL DEFAULT 0");
+applyColumnMigration(33, "sources", "sync_run_id", "ALTER TABLE sources ADD COLUMN sync_run_id TEXT");
+applyColumnMigration(34, "posts", "last_seen_sync_run", "ALTER TABLE posts ADD COLUMN last_seen_sync_run TEXT");
 
 const admin = process.env.ADMIN_USERNAME;
 const adminPassword = process.env.ADMIN_PASSWORD;

lib/memos.ts

@@ -19,26 +19,28 @@ export async function getMemosIdentity(baseUrl: string, token: string) {
   return user;
 }
 export function memoUrl(baseUrl: string, memoName: string) { const id = memoName.split("/").at(-1); return id ? `${baseUrl.replace(/\/$/, "")}/m/${encodeURIComponent(id)}` : null; }
-export async function listMemos(baseUrl: string, token: string, rules: MemosSyncRules = {}) {
-  const all: MemosMemo[] = []; let pageToken = "";
+export type MemosPage = { memos: MemosMemo[]; nextPageToken: string };
+export async function listMemos(baseUrl: string, token: string, rules: MemosSyncRules = {}, page: { pageToken?: string; pageSize?: number } = {}): Promise<MemosPage> {
+  let pageToken = page.pageToken || "";
   const tags = rules.tags?.filter(Boolean) || []; const mode = rules.attachmentMode || "all";
   const quote = (value: string) => JSON.stringify(value); const filters = [rules.creator && `creator == ${quote(rules.creator)}`, `visibility == "PUBLIC"`, ...tags.map((tag) => `tags.exists(t, t == ${quote(tag)})`)].filter(Boolean).join(" && ");
   let useServerFilter = Boolean(filters);
-  do { const params = new URLSearchParams({ pageSize: "100" }); if (pageToken) params.set("pageToken", pageToken); if (useServerFilter && filters) params.set("filter", filters); let data: any;
+  do { const params = new URLSearchParams({ pageSize: String(Math.max(10, Math.min(100, page.pageSize || 100))) }); if (pageToken) params.set("pageToken", pageToken); if (useServerFilter && filters) params.set("filter", filters); let data: any;
     try { data = await (await request(`${base(baseUrl)}/memos?${params}`, token)).json(); }
     catch (error) { if (!pageToken && useServerFilter && error instanceof Error && error.message.startsWith("Memos API 400")) { useServerFilter = false; continue; } throw error; }
-    all.push(...(data.memos || [])); pageToken = data.nextPageToken || ""; } while (pageToken);
-  return all.filter((memo) => {
+    const memos = (data.memos || []).filter((memo: MemosMemo) => {
     if (memo.visibility !== "PUBLIC") return false;
     if (rules.creator && memo.creator !== rules.creator) return false;
     if (tags.length && !tags.every((tag) => memo.tags?.includes(tag))) return false;
     const created = memo.createTime?.slice(0, 10); if (rules.from && (!created || created < rules.from)) return false; if (rules.to && (!created || created > rules.to)) return false;
     return true;
-  }).map((memo) => {
+    }).map((memo: MemosMemo) => {
     if (mode === "all") return memo;
     const onlyImages = <T extends { type?: string }>(items: T[] | undefined) => mode === "none" ? [] : (items || []).filter((item) => item.type?.startsWith("image/"));
     return { ...memo, attachments: onlyImages(memo.attachments), resources: onlyImages(memo.resources) };
-  });
+    });
+    return { memos, nextPageToken: data.nextPageToken || "" };
+  } while (true);
 }
 export async function createMemo(baseUrl: string, token: string, memo: Pick<MemosMemo, "content" | "visibility"> & { attachments?: unknown[]; resources?: unknown[] }) {
   return (await request(`${base(baseUrl)}/memos`, token, { method: "POST", body: JSON.stringify({ state: "NORMAL", ...memo }) })).json() as Promise<MemosMemo>;

lib/sync.ts

@@ -1,6 +1,6 @@
 import { db } from "@/lib/db";
 
-export type SyncTrigger = "manual" | "webhook" | "scheduled" | "source-created";
+export type SyncTrigger = "manual" | "webhook" | "scheduled" | "source-created" | "batch";
 
 /** Queue one pull per source at a time. Returns true only when a new job was created. */
 export function queuePull(sourceId: number, trigger: SyncTrigger, payload: unknown = {}) {