tangsongdayo/Mebbling
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

feat: batch large source imports

Browse Source
This commit is contained in:
tangsongdayo
2026-07-19 05:43:03 +08:00
parent 591981b7c8
commit 9e159910ca
8 changed files with 40 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/dashboard/page.tsx
+4 -4
View File
@@ -4,14 +4,14 @@ import { db } from "@/lib/db";
import { PublishForm } from "./publish-form";
import { WebhookControl } from "./webhook-control";
type Source = { id: number; name: string; base_url: string; integration_type: "memos" | "rss"; rss_feed_url: string | null; sync_status: string; last_synced_at: string | null; last_error: string | null; webhook_secret_hash: string | null; webhook_mode: string; last_webhook_at: string | null; owner_id: number; is_enabled: number; disabled_at: string | null; sync_tags_json: string; sync_from: string | null; sync_to: string | null; sync_attachment_mode: "all" | "images" | "none"; attachment_storage_mode: "remote" | "images" | "all"; attachment_cache_limit_bytes: number; attachment_cache_error: string | null; remote_display_name: string | null; remote_avatar_url: string | null; last_connection_at: string | null; last_connection_error: string | null };
type Source = { id: number; name: string; base_url: string; integration_type: "memos" | "rss"; rss_feed_url: string | null; sync_status: string; last_synced_at: string | null; last_error: string | null; webhook_secret_hash: string | null; webhook_mode: string; last_webhook_at: string | null; owner_id: number; is_enabled: number; disabled_at: string | null; sync_tags_json: string; sync_from: string | null; sync_to: string | null; sync_attachment_mode: "all" | "images" | "none"; sync_batch_size: number; sync_max_posts: number | null; sync_cursor: string | null; sync_imported_count: number; attachment_storage_mode: "remote" | "images" | "all"; attachment_cache_limit_bytes: number; attachment_cache_error: string | null; remote_display_name: string | null; remote_avatar_url: string | null; last_connection_at: string | null; last_connection_error: string | null };
type Job = { id: number; kind: string; trigger: string | null; status: string; attempts: number; last_error: string | null; created_at: string; finished_at: string | null };
export const dynamic = "force-dynamic";
export default async function Dashboard({ searchParams }: { searchParams: Promise<{ error?: string; source?: string; sync?: string }> }) {
const query = await searchParams; const user = await getSession(); if (!user) redirect("/login");
const sourceRows = db.prepare("SELECT s.id,s.name,s.base_url,s.integration_type,s.rss_feed_url,s.sync_status,s.last_synced_at,s.last_error,s.webhook_secret_hash,s.webhook_mode,s.last_webhook_at,s.user_id AS owner_id,s.is_enabled,s.disabled_at,s.sync_tags_json,s.sync_from,s.sync_to,s.sync_attachment_mode,s.attachment_storage_mode,s.attachment_cache_limit_bytes,s.attachment_cache_error,s.remote_display_name,s.remote_avatar_url,s.last_connection_at,s.last_connection_error FROM sources s JOIN source_members sm ON sm.source_id=s.id WHERE sm.user_id=? ORDER BY s.id DESC").all(user.id) as Source[];
const sourceRows = db.prepare("SELECT s.id,s.name,s.base_url,s.integration_type,s.rss_feed_url,s.sync_status,s.last_synced_at,s.last_error,s.webhook_secret_hash,s.webhook_mode,s.last_webhook_at,s.user_id AS owner_id,s.is_enabled,s.disabled_at,s.sync_tags_json,s.sync_from,s.sync_to,s.sync_attachment_mode,s.sync_batch_size,s.sync_max_posts,s.sync_cursor,s.sync_imported_count,s.attachment_storage_mode,s.attachment_cache_limit_bytes,s.attachment_cache_error,s.remote_display_name,s.remote_avatar_url,s.last_connection_at,s.last_connection_error FROM sources s JOIN source_members sm ON sm.source_id=s.id WHERE sm.user_id=? ORDER BY s.id DESC").all(user.id) as Source[];
const sources = sourceRows.map((source) => ({ ...source, syncTags: (() => { try { return JSON.parse(source.sync_tags_json) as string[]; } catch { return []; } })(), members: db.prepare("SELECT u.username,u.id,sm.role FROM source_members sm JOIN users u ON u.id=sm.user_id WHERE sm.source_id=? ORDER BY sm.role DESC,u.username").all(source.id) as { username: string; id: number; role: string }[], jobs: db.prepare("SELECT id,kind,trigger,status,attempts,last_error,created_at,finished_at FROM sync_jobs WHERE source_id=? ORDER BY id DESC LIMIT 5").all(source.id) as Job[] }));
const publishSources = sources.filter((source) => source.is_enabled && source.integration_type === "memos");
return <><style>{`form:has(input[name="action"][value="rename"]){display:none}`}</style>
@@ -24,11 +24,11 @@ export default async function Dashboard({ searchParams }: { searchParams: Promis
<section className="card"><h2>連接 RSS</h2><form action="/api/sources/rss" method="post"><label>RSS Feed 網址<input name="feedUrl" type="url" required placeholder="https://example.com/feed.xml" /></label><button>驗證並訂閱</button></form><p className="muted">RSS 為唯讀來源，不需要 API Key，也不會回寫原網站。同步時會更新 Feed 內公開的項目。</p></section>
<section><h2>已連接來源</h2>{sources.map((source) => <article className="card" key={source.id}>
<div className="space"><strong>{source.name}</strong><span className="tag">{source.is_enabled ? source.sync_status : "disabled"}</span></div>
<p className="meta">來源 ID：{source.id}<br />{source.integration_type === "rss" ? source.rss_feed_url : source.base_url}{source.remote_display_name && <><br />Memos 帳號：{source.remote_avatar_url && <img className="avatar" src={source.remote_avatar_url} alt="" />} {source.remote_display_name}</>}<br />成員：{source.members.map((member) => `${member.username}${member.role === "owner" ? "（建立者）" : ""}`).join("、")}<br />上次同步：{source.last_synced_at || "尚未完成"}<br />{source.integration_type === "memos" && <>附件保存：{source.attachment_storage_mode === "remote" ? "遠端連結" : source.attachment_storage_mode === "images" ? "只快取圖片" : "完整備份"}（配額 {Math.round(source.attachment_cache_limit_bytes / 1024 / 1024)} MiB）<br />連線：{source.last_connection_at ? `最近成功：${new Date(source.last_connection_at + "Z").toLocaleString("zh-TW")}` : "尚未測試"}<br />Webhook：{source.webhook_secret_hash ? (source.last_webhook_at ? (Date.now() - new Date(source.last_webhook_at + "Z").getTime() > 7 * 24 * 60 * 60 * 1000 ? `警示：超過 7 天未收到（最近：${new Date(source.last_webhook_at + "Z").toLocaleString("zh-TW")}）` : `健康（最近收到：${new Date(source.last_webhook_at + "Z").toLocaleString("zh-TW")}）`) : "已建立 URL，尚未收到呼叫") : "尚未建立 URL"}</>}{!source.is_enabled && <><br />已停用：{source.disabled_at ? new Date(source.disabled_at + "Z").toLocaleString("zh-TW") : "是"}</>}{source.last_error && <><br /><span className="error">同步：{source.last_error}</span></>}{source.last_connection_error && <><br /><span className="error">連線：{source.last_connection_error}</span></>}{source.attachment_cache_error && <><br /><span className="error">附件快取：{source.attachment_cache_error}</span></>}</p>
<p className="meta">來源 ID：{source.id}<br />{source.integration_type === "rss" ? source.rss_feed_url : source.base_url}{source.remote_display_name && <><br />Memos 帳號：{source.remote_avatar_url && <img className="avatar" src={source.remote_avatar_url} alt="" />} {source.remote_display_name}</>}<br />成員：{source.members.map((member) => `${member.username}${member.role === "owner" ? "（建立者）" : ""}`).join("、")}<br />上次同步：{source.last_synced_at || "尚未完成"}{source.sync_cursor && <>（批次匯入中：{source.sync_imported_count} 篇）</>}<br />{source.integration_type === "memos" && <>附件保存：{source.attachment_storage_mode === "remote" ? "遠端連結" : source.attachment_storage_mode === "images" ? "只快取圖片" : "完整備份"}（配額 {Math.round(source.attachment_cache_limit_bytes / 1024 / 1024)} MiB）<br />連線：{source.last_connection_at ? `最近成功：${new Date(source.last_connection_at + "Z").toLocaleString("zh-TW")}` : "尚未測試"}<br />Webhook：{source.webhook_secret_hash ? (source.last_webhook_at ? (Date.now() - new Date(source.last_webhook_at + "Z").getTime() > 7 * 24 * 60 * 60 * 1000 ? `警示：超過 7 天未收到（最近：${new Date(source.last_webhook_at + "Z").toLocaleString("zh-TW")}）` : `健康（最近收到：${new Date(source.last_webhook_at + "Z").toLocaleString("zh-TW")}）`) : "已建立 URL，尚未收到呼叫") : "尚未建立 URL"}</>}{!source.is_enabled && <><br />已停用：{source.disabled_at ? new Date(source.disabled_at + "Z").toLocaleString("zh-TW") : "是"}</>}{source.last_error && <><br /><span className="error">同步：{source.last_error}</span></>}{source.last_connection_error && <><br /><span className="error">連線：{source.last_connection_error}</span></>}{source.attachment_cache_error && <><br /><span className="error">附件快取：{source.attachment_cache_error}</span></>}</p>
{source.owner_id === user.id ? <>
{source.integration_type === "memos" && <WebhookControl sourceId={source.id} configured={Boolean(source.webhook_secret_hash)} automatic={source.webhook_mode === "signed"} />}
<form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="set-attachment-storage" /><label>附件保存策略<select name="mode" defaultValue={source.attachment_storage_mode}><option value="remote">遠端連結（不佔 Hub 空間）</option><option value="images">只快取圖片</option><option value="all">完整備份附件</option></select></label><label>快取配額（MiB）<input name="quotaMiB" type="number" min="10" max="10240" defaultValue={Math.round(source.attachment_cache_limit_bytes / 1024 / 1024)} /></label><button>儲存附件策略並同步</button></form>
<details><summary>來源管理</summary><form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="rename" /><label>顯示名稱<input name="name" defaultValue={source.name} required maxLength={80} /></label><button>儲存名稱</button></form><form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="set-sync-rules" /><label>只同步標籤（逗號分隔，留白為全部）<input name="tags" defaultValue={source.syncTags.join(", ")} placeholder="旅行, 技術" /></label><div className="row"><label>開始日期<input name="from" type="date" defaultValue={source.sync_from || ""} /></label><label>結束日期<input name="to" type="date" defaultValue={source.sync_to || ""} /></label></div><label>附件<select name="attachmentMode" defaultValue={source.sync_attachment_mode}><option value="all">同步全部附件</option><option value="images">僅同步圖片</option><option value="none">不同步附件</option></select></label><button>儲存同步規則並同步</button></form><form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="test-connection" /><button>測試 Memos 連線</button></form><form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="set-enabled" /><input type="hidden" name="enabled" value={source.is_enabled ? "0" : "1"} /><button className={source.is_enabled ? "danger" : ""}>{source.is_enabled ? "停用來源" : "啟用來源"}</button></form>{source.members.length > 1 && <form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="transfer" /><label>轉移建立者<select name="username" required defaultValue=""> <option value="" disabled>選擇成員</option>{source.members.filter((member) => member.id !== user.id).map((member) => <option key={member.id} value={member.username}>{member.username}</option>)}</select></label><button>轉移所有權</button></form>}<form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="delete" /><button className="danger">刪除來源與遠端鏡像貼文</button></form></details>
<details><summary>來源管理</summary><form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="rename" /><label>顯示名稱<input name="name" defaultValue={source.name} required maxLength={80} /></label><button>儲存名稱</button></form><form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="set-sync-rules" /><label>只同步標籤（逗號分隔，留白為全部）<input name="tags" defaultValue={source.syncTags.join(", ")} placeholder="旅行, 技術" /></label><div className="row"><label>開始日期<input name="from" type="date" defaultValue={source.sync_from || ""} /></label><label>結束日期<input name="to" type="date" defaultValue={source.sync_to || ""} /></label></div><label>附件<select name="attachmentMode" defaultValue={source.sync_attachment_mode}><option value="all">同步全部附件</option><option value="images">僅同步圖片</option><option value="none">不同步附件</option></select></label><div className="row"><label>每批貼文數<input name="batchSize" type="number" min="10" max="100" defaultValue={source.sync_batch_size} /></label><label>首次匯入上限（0 為不限）<input name="maxPosts" type="number" min="0" max="100000" defaultValue={source.sync_max_posts || 0} /></label></div><button>儲存同步規則並從頭批次同步</button></form><form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="test-connection" /><button>測試 Memos 連線</button></form><form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="set-enabled" /><input type="hidden" name="enabled" value={source.is_enabled ? "0" : "1"} /><button className={source.is_enabled ? "danger" : ""}>{source.is_enabled ? "停用來源" : "啟用來源"}</button></form>{source.members.length > 1 && <form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="transfer" /><label>轉移建立者<select name="username" required defaultValue=""> <option value="" disabled>選擇成員</option>{source.members.filter((member) => member.id !== user.id).map((member) => <option key={member.id} value={member.username}>{member.username}</option>)}</select></label><button>轉移所有權</button></form>}<form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="delete" /><button className="danger">刪除來源與遠端鏡像貼文</button></form></details>
</> : <form action={`/api/sources/${source.id}/manage`} method="post"><input type="hidden" name="action" value="leave" /><button className="danger">離開共享來源</button></form>}
<form action="/api/sync" method="post"><input type="hidden" name="sourceId" value={source.id} /><button disabled={!source.is_enabled}>{source.is_enabled ? "立即同步" : "來源已停用"}</button></form>
<details><summary>最近同步工作</summary>{source.jobs.length ? <ul className="job-list">{source.jobs.map((job) => <li key={job.id}><strong>{job.kind}</strong> · {job.trigger || "legacy"} · <span className="tag">{job.status}</span> · 嘗試 {job.attempts} 次<br /><span className="meta">建立：{new Date(job.created_at + "Z").toLocaleString("zh-TW")}{job.finished_at && `；完成：${new Date(job.finished_at + "Z").toLocaleString("zh-TW")}`}</span>{job.last_error && <><br /><span className="error">{job.last_error}</span></>}</li>)}</ul> : <p className="muted">尚無同步工作。</p>}</details>