diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1c90d57..1e16cf6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,16 @@
本專案遵循 [Semantic Versioning](https://semver.org/lang/zh-TW/);版本 `0.x` 表示功能仍可能調整。
+## [0.6.0] - Unreleased
+
+### Added
+
+- 文章顯示時隱藏已辨識的內文 hashtag,保留原始 Markdown 與文章底部標籤。
+- 具時間範圍篩選的公開標籤雲與標籤導覽入口。
+- 瀏覽器端自動儲存的發文草稿與 Markdown 預覽。
+- 來源附件保存資料結構:遠端連結、僅圖片快取或完整快取,並提供來源配額與失效快取清理。
+- 標籤別名/合併的資料結構,供後續管理介面使用。
+
## [0.5.0] - Unreleased
### Added
diff --git a/README.md b/README.md
index 4c6fa34..957ac19 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
自架的 Memos 公開貼文 Hub。將朋友各自 Memos 中的公開貼文集中展示,同時保留 Hub 內的留言、表情回應與發文功能。
-目前開發版本:`v0.5.0`(尚未發布)。版本變更請見 [CHANGELOG.md](CHANGELOG.md)。
+目前開發版本:`v0.6.0`(尚未發布)。版本變更請見 [CHANGELOG.md](CHANGELOG.md)。
## 功能
@@ -22,6 +22,8 @@
- 控制台可測試 Token/Memos 連線、顯示遠端名稱與頭像,並提示 webhook 長時間未收到事件的狀態。
- 同源請求保護、SQLite 共用登入/webhook 限流、附件白名單與可選掃毒服務。
- 管理員可審核檢舉、隱藏貼文、停權帳號與協助重設密碼;提供健康檢查與 JSON 結構化日誌。
+- 文章以底部標籤為主,避免內文 hashtag 重複;提供時間範圍標籤雲、草稿自動儲存與預覽。
+- 遠端附件預設直連;來源可選擇只快取圖片或完整快取,並受每來源配額限制。
## 快速啟動(WSL/Docker)
diff --git a/app/api/sources/[id]/manage/route.ts b/app/api/sources/[id]/manage/route.ts
index 97faa23..46dff6e 100644
--- a/app/api/sources/[id]/manage/route.ts
+++ b/app/api/sources/[id]/manage/route.ts
@@ -28,6 +28,10 @@ export async function POST(req: Request, { params }: { params: Promise<{ id: str
if ((from && !/^\d{4}-\d{2}-\d{2}$/.test(from)) || (to && !/^\d{4}-\d{2}-\d{2}$/.test(to)) || (from && to && from > to) || !["all", "images", "none"].includes(attachmentMode)) throw new Error("Invalid sync rules");
db.prepare("UPDATE sources SET sync_tags_json=?,sync_from=?,sync_to=?,sync_attachment_mode=? WHERE id=?").run(JSON.stringify(tags), from || null, to || null, attachmentMode, id);
queuePull(id, "manual");
+ } else if (action === "set-attachment-storage") {
+ if (!owner) throw new Error("Only the owner can change attachment storage"); const mode = String(form.get("mode") || "remote"); const quotaMiB = Number(form.get("quotaMiB") || 100);
+ if (!["remote", "images", "all"].includes(mode) || !Number.isFinite(quotaMiB) || quotaMiB < 10 || quotaMiB > 10_240) throw new Error("Invalid attachment storage settings");
+ db.prepare("UPDATE sources SET attachment_storage_mode=?,attachment_cache_limit_bytes=?,attachment_cache_error=NULL WHERE id=?").run(mode, Math.round(quotaMiB * 1024 * 1024), id); queuePull(id, "manual");
} else if (action === "test-connection") {
if (!owner) throw new Error("Only the owner can test the connection");
try {
diff --git a/app/components/markdown.tsx b/app/components/markdown.tsx
index 6071cf4..36b61c9 100644
--- a/app/components/markdown.tsx
+++ b/app/components/markdown.tsx
@@ -1,8 +1,9 @@
import ReactMarkdown from "react-markdown";
import rehypeHighlight from "rehype-highlight";
import remarkGfm from "remark-gfm";
+import { withoutInlineTags } from "@/lib/tags";
/** Raw HTML is intentionally not enabled, so Memos content cannot inject script or markup. */
-export function Markdown({ content, compact = false }: { content: string; compact?: boolean }) {
- return
{content}
;
+export function Markdown({ content, tags = [], compact = false }: { content: string; tags?: string[]; compact?: boolean }) {
+ return {withoutInlineTags(content, tags)}
;
}
diff --git a/app/components/post-card.tsx b/app/components/post-card.tsx
index a5dc08b..f4150b8 100644
--- a/app/components/post-card.tsx
+++ b/app/components/post-card.tsx
@@ -1,10 +1,11 @@
import Link from "next/link";
import { Attachments } from "./attachments";
import { Markdown } from "./markdown";
+import { canonicalTags } from "@/lib/tags";
export type PublicPost = { id: number; source_id: number | null; content: string; tags_json: string; attachments_json: string; created_at: string; username: string; name: string | null; source_base_url: string | null; comment_count: number; reaction_count: number };
export function PostCard({ post }: { post: PublicPost }) {
- let tags: string[] = []; try { tags = JSON.parse(post.tags_json); } catch { /* Ignore malformed legacy tags. */ }
- return @{post.username}{post.name ? ` · ${post.name}` : ""}{new Date(post.created_at).toLocaleString("zh-TW")}
{tags.map((tag) => #{tag})}{post.source_id && 來源}閱讀全文 · 💬 {post.comment_count} 🙂 {post.reaction_count}
;
+ let tags: string[] = []; try { tags = canonicalTags(JSON.parse(post.tags_json)); } catch { /* Ignore malformed legacy tags. */ }
+ return @{post.username}{post.name ? ` · ${post.name}` : ""}{new Date(post.created_at).toLocaleString("zh-TW")}
{tags.map((tag) => #{tag})}{post.source_id && 來源}閱讀全文 · 💬 {post.comment_count} 🙂 {post.reaction_count}
;
}
diff --git a/app/dashboard/page.tsx b/app/dashboard/page.tsx
index 03151e9..77a6924 100644
--- a/app/dashboard/page.tsx
+++ b/app/dashboard/page.tsx
@@ -4,14 +4,14 @@ import { db } from "@/lib/db";
import { PublishForm } from "./publish-form";
import { WebhookControl } from "./webhook-control";
-type Source = { id: number; name: string; base_url: string; sync_status: string; last_synced_at: string | null; last_error: string | null; webhook_secret_hash: string | null; last_webhook_at: string | null; owner_id: number; is_enabled: number; disabled_at: string | null; sync_tags_json: string; sync_from: string | null; sync_to: string | null; sync_attachment_mode: "all" | "images" | "none"; remote_display_name: string | null; remote_avatar_url: string | null; last_connection_at: string | null; last_connection_error: string | null };
+type Source = { id: number; name: string; base_url: string; sync_status: string; last_synced_at: string | null; last_error: string | null; webhook_secret_hash: string | null; last_webhook_at: string | null; owner_id: number; is_enabled: number; disabled_at: string | null; sync_tags_json: string; sync_from: string | null; sync_to: string | null; sync_attachment_mode: "all" | "images" | "none"; attachment_storage_mode: "remote" | "images" | "all"; attachment_cache_limit_bytes: number; attachment_cache_error: string | null; remote_display_name: string | null; remote_avatar_url: string | null; last_connection_at: string | null; last_connection_error: string | null };
type Job = { id: number; kind: string; trigger: string | null; status: string; attempts: number; last_error: string | null; created_at: string; finished_at: string | null };
export const dynamic = "force-dynamic";
export default async function Dashboard({ searchParams }: { searchParams: Promise<{ error?: string; source?: string; sync?: string }> }) {
const query = await searchParams; const user = await getSession(); if (!user) redirect("/login");
- const sourceRows = db.prepare("SELECT s.id,s.name,s.base_url,s.sync_status,s.last_synced_at,s.last_error,s.webhook_secret_hash,s.last_webhook_at,s.user_id AS owner_id,s.is_enabled,s.disabled_at,s.sync_tags_json,s.sync_from,s.sync_to,s.sync_attachment_mode,s.remote_display_name,s.remote_avatar_url,s.last_connection_at,s.last_connection_error FROM sources s JOIN source_members sm ON sm.source_id=s.id WHERE sm.user_id=? ORDER BY s.id DESC").all(user.id) as Source[];
+ const sourceRows = db.prepare("SELECT s.id,s.name,s.base_url,s.sync_status,s.last_synced_at,s.last_error,s.webhook_secret_hash,s.last_webhook_at,s.user_id AS owner_id,s.is_enabled,s.disabled_at,s.sync_tags_json,s.sync_from,s.sync_to,s.sync_attachment_mode,s.attachment_storage_mode,s.attachment_cache_limit_bytes,s.attachment_cache_error,s.remote_display_name,s.remote_avatar_url,s.last_connection_at,s.last_connection_error FROM sources s JOIN source_members sm ON sm.source_id=s.id WHERE sm.user_id=? ORDER BY s.id DESC").all(user.id) as Source[];
const sources = sourceRows.map((source) => ({ ...source, syncTags: (() => { try { return JSON.parse(source.sync_tags_json) as string[]; } catch { return []; } })(), members: db.prepare("SELECT u.username,u.id,sm.role FROM source_members sm JOIN users u ON u.id=sm.user_id WHERE sm.source_id=? ORDER BY sm.role DESC,u.username").all(source.id) as { username: string; id: number; role: string }[], jobs: db.prepare("SELECT id,kind,trigger,status,attempts,last_error,created_at,finished_at FROM sync_jobs WHERE source_id=? ORDER BY id DESC LIMIT 5").all(source.id) as Job[] }));
const publishSources = sources.filter((source) => source.is_enabled);
return <>
@@ -23,9 +23,10 @@ export default async function Dashboard({ searchParams }: { searchParams: Promis
連接 Memos
Token 會使用伺服器金鑰加密保存。同一個 Memos 帳號與網址會自動共用來源,不會建立重複貼文。
已連接來源
{sources.map((source) =>
{source.name}{source.is_enabled ? source.sync_status : "disabled"}
- 來源 ID:{source.id}
{source.base_url}{source.remote_display_name && <>
Memos 帳號:{source.remote_avatar_url &&
} {source.remote_display_name}>}
成員:{source.members.map((member) => `${member.username}${member.role === "owner" ? "(建立者)" : ""}`).join("、")}
上次同步:{source.last_synced_at || "尚未完成"}
連線:{source.last_connection_at ? `最近成功:${new Date(source.last_connection_at + "Z").toLocaleString("zh-TW")}` : "尚未測試"}
Webhook:{source.webhook_secret_hash ? (source.last_webhook_at ? (Date.now() - new Date(source.last_webhook_at + "Z").getTime() > 7 * 24 * 60 * 60 * 1000 ? `警示:超過 7 天未收到(最近:${new Date(source.last_webhook_at + "Z").toLocaleString("zh-TW")})` : `健康(最近收到:${new Date(source.last_webhook_at + "Z").toLocaleString("zh-TW")})`) : "已建立 URL,尚未收到呼叫") : "尚未建立 URL"}{!source.is_enabled && <>
已停用:{source.disabled_at ? new Date(source.disabled_at + "Z").toLocaleString("zh-TW") : "是"}>}{source.last_error && <>
同步:{source.last_error}>}{source.last_connection_error && <>
連線:{source.last_connection_error}>}
+ 來源 ID:{source.id}
{source.base_url}{source.remote_display_name && <>
Memos 帳號:{source.remote_avatar_url &&
} {source.remote_display_name}>}
成員:{source.members.map((member) => `${member.username}${member.role === "owner" ? "(建立者)" : ""}`).join("、")}
上次同步:{source.last_synced_at || "尚未完成"}
附件保存:{source.attachment_storage_mode === "remote" ? "遠端連結" : source.attachment_storage_mode === "images" ? "只快取圖片" : "完整備份"}(配額 {Math.round(source.attachment_cache_limit_bytes / 1024 / 1024)} MiB)
連線:{source.last_connection_at ? `最近成功:${new Date(source.last_connection_at + "Z").toLocaleString("zh-TW")}` : "尚未測試"}
Webhook:{source.webhook_secret_hash ? (source.last_webhook_at ? (Date.now() - new Date(source.last_webhook_at + "Z").getTime() > 7 * 24 * 60 * 60 * 1000 ? `警示:超過 7 天未收到(最近:${new Date(source.last_webhook_at + "Z").toLocaleString("zh-TW")})` : `健康(最近收到:${new Date(source.last_webhook_at + "Z").toLocaleString("zh-TW")})`) : "已建立 URL,尚未收到呼叫") : "尚未建立 URL"}{!source.is_enabled && <>
已停用:{source.disabled_at ? new Date(source.disabled_at + "Z").toLocaleString("zh-TW") : "是"}>}{source.last_error && <>
同步:{source.last_error}>}{source.last_connection_error && <>
連線:{source.last_connection_error}>}{source.attachment_cache_error && <>
附件快取:{source.attachment_cache_error}>}
{source.owner_id === user.id ? <>
+
來源管理
{source.members.length > 1 && }
> : }
diff --git a/app/dashboard/publish-form.tsx b/app/dashboard/publish-form.tsx
index f0b013f..113840e 100644
--- a/app/dashboard/publish-form.tsx
+++ b/app/dashboard/publish-form.tsx
@@ -1,34 +1,15 @@
"use client";
-import { FormEvent, useState } from "react";
+import { FormEvent, useEffect, useMemo, useState } from "react";
type Source = { id: number; name: string };
+const draftKey = "mebbling:publish-draft";
export function PublishForm({ sources }: { sources: Source[] }) {
- const [error, setError] = useState("");
- const [submitting, setSubmitting] = useState(false);
-
- async function submit(event: FormEvent) {
- event.preventDefault();
- setSubmitting(true); setError("");
- try {
- const response = await fetch("/api/posts", { method: "POST", body: new FormData(event.currentTarget), headers: { Accept: "application/json" } });
- const result = await response.json();
- if (!response.ok) throw new Error(result.error || "發佈失敗");
- window.location.assign(`/posts/${result.id}`);
- } catch (reason) {
- setError(reason instanceof Error ? reason.message : "發佈失敗");
- setSubmitting(false);
- }
- }
-
- return ;
+ const [error, setError] = useState(""); const [submitting, setSubmitting] = useState(false); const [content, setContent] = useState(""); const [tags, setTags] = useState(""); const [sourceId, setSourceId] = useState(String(sources[0]?.id || "")); const [preview, setPreview] = useState(false);
+ useEffect(() => { try { const saved = JSON.parse(localStorage.getItem(draftKey) || "{}"); setContent(saved.content || ""); setTags(saved.tags || ""); if (saved.sourceId && sources.some((source) => String(source.id) === saved.sourceId)) setSourceId(saved.sourceId); } catch {} }, [sources]);
+ useEffect(() => { localStorage.setItem(draftKey, JSON.stringify({ content, tags, sourceId })); }, [content, tags, sourceId]);
+ const draftState = useMemo(() => content ? "草稿已自動儲存於此瀏覽器" : "", [content]);
+ async function submit(event: FormEvent) { event.preventDefault(); setSubmitting(true); setError(""); try { const response = await fetch("/api/posts", { method: "POST", body: new FormData(event.currentTarget), headers: { Accept: "application/json" } }); const result = await response.json(); if (!response.ok) throw new Error(result.error || "發佈失敗"); localStorage.removeItem(draftKey); window.location.assign(`/posts/${result.id}`); } catch (reason) { setError(reason instanceof Error ? reason.message : "發佈失敗"); setSubmitting(false); } }
+ return ;
}
diff --git a/app/layout.tsx b/app/layout.tsx
index 9a54334..ea8af18 100644
--- a/app/layout.tsx
+++ b/app/layout.tsx
@@ -6,5 +6,5 @@ export const metadata = { title: "Mebbling", description: "聚合朋友公開筆
export default async function RootLayout({ children }: { children: React.ReactNode }) {
const user = await getSession();
const unread = user ? Number((db.prepare("SELECT count(*) count FROM notifications WHERE user_id=? AND read_at IS NULL").get(user.id) as { count: number }).count) : 0;
- return Mebbling{children};
+ return Mebbling{children};
}
diff --git a/app/posts/[id]/page.tsx b/app/posts/[id]/page.tsx
index 7fa3b3a..b410d34 100644
--- a/app/posts/[id]/page.tsx
+++ b/app/posts/[id]/page.tsx
@@ -4,6 +4,7 @@ import { db } from "@/lib/db";
import { getSession } from "@/lib/auth";
import { Attachments } from "@/app/components/attachments";
import { Markdown } from "@/app/components/markdown";
+import { canonicalTags } from "@/lib/tags";
export const dynamic = "force-dynamic";
@@ -22,7 +23,7 @@ export default async function PostPage({ params, searchParams }: { params: Promi
const bookmark = user ? db.prepare("SELECT kind FROM bookmarks WHERE user_id=? AND post_id=?").get(user.id, id) as { kind: string } | undefined : undefined;
const comments = db.prepare("SELECT c.*,u.username FROM comments c JOIN users u ON u.id=c.author_id WHERE c.post_id=? AND c.hidden=0 ORDER BY c.created_at").all(id) as any[];
const reactions = db.prepare("SELECT emoji,count(*) count FROM reactions WHERE post_id=? GROUP BY emoji").all(id) as any[];
- return @{post.username} · {post.remote_display_name || post.name || "Hub"} · {new Date(post.created_at).toLocaleString("zh-TW")}{post.remote_url && <> · 在 Memos 開啟>}
+ let tags: string[] = []; try { tags = canonicalTags(JSON.parse(post.tags_json)); } catch {} return @{post.username} · {post.remote_display_name || post.name || "Hub"} · {new Date(post.created_at).toLocaleString("zh-TW")}{post.remote_url && <> · 在 Memos 開啟>}
{reactions.map((reaction: any) => {reaction.emoji} {reaction.count})}{user && <>>}{user && ["👍", "❤️", "🎉", "🤔"].map((emoji) => )}
留言
{user ? <>檢舉這篇貼文
{query.reported && 已收到檢舉,管理員會審核。
} > : 請先登入以留言、互動或檢舉。
}{comments.map((comment) => @{comment.username}{comment.content}
{new Date(comment.created_at).toLocaleString("zh-TW")} )}
;
diff --git a/app/tags/page.tsx b/app/tags/page.tsx
new file mode 100644
index 0000000..a16447a
--- /dev/null
+++ b/app/tags/page.tsx
@@ -0,0 +1,12 @@
+import Link from "next/link";
+import { db } from "@/lib/db";
+import { canonicalTag } from "@/lib/tags";
+
+export const dynamic = "force-dynamic";
+export default async function TagsPage({ searchParams }: { searchParams: Promise<{ days?: string }> }) {
+ const query = await searchParams; const days = [30, 90, 365, 0].includes(Number(query.days)) ? Number(query.days) : 0;
+ const rows = db.prepare(`SELECT tags_json FROM posts WHERE visibility='PUBLIC' AND hidden=0 ${days ? "AND created_at >= datetime('now', ?)" : ""}`).all(...(days ? [`-${days} days`] : [])) as { tags_json: string }[];
+ const counts = new Map(); for (const row of rows) { try { for (const tag of new Set(JSON.parse(row.tags_json) as string[])) { const canonical = canonicalTag(tag); counts.set(canonical, (counts.get(canonical) || 0) + 1); } } catch {} }
+ const tags = [...counts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0], "zh-Hant")); const max = Math.max(...tags.map(([, count]) => count), 1);
+ return <>標籤雲
依公開貼文使用次數呈現,共 {tags.length} 個標籤。
{tags.map(([tag, count]) => #{tag}{count})}>;
+}
diff --git a/docker-compose.yml b/docker-compose.yml
index d527df2..8c916e3 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,10 +1,10 @@
services:
web:
- image: mebbling:${MEBBLING_VERSION:-0.5.0}
+ image: mebbling:${MEBBLING_VERSION:-0.6.0}
build:
context: .
args:
- APP_VERSION: "${MEBBLING_VERSION:-0.5.0}"
+ APP_VERSION: "${MEBBLING_VERSION:-0.6.0}"
ports: ["8088:3000"]
env_file: .env
environment: { DATABASE_PATH: /app/data/hub.db }
@@ -13,11 +13,11 @@ services:
- ./public/uploads:/app/public/uploads
restart: unless-stopped
worker:
- image: mebbling:${MEBBLING_VERSION:-0.5.0}
+ image: mebbling:${MEBBLING_VERSION:-0.6.0}
build:
context: .
args:
- APP_VERSION: "${MEBBLING_VERSION:-0.5.0}"
+ APP_VERSION: "${MEBBLING_VERSION:-0.6.0}"
command: npm run worker
env_file: .env
environment: { DATABASE_PATH: /app/data/hub.db }
diff --git a/lib/db.ts b/lib/db.ts
index 94661fb..854f23c 100644
--- a/lib/db.ts
+++ b/lib/db.ts
@@ -88,6 +88,10 @@ CREATE TABLE IF NOT EXISTS error_events (
id INTEGER PRIMARY KEY, scope TEXT NOT NULL, message TEXT NOT NULL, context_json TEXT,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
+CREATE TABLE IF NOT EXISTS tag_aliases (
+ alias TEXT PRIMARY KEY COLLATE NOCASE, canonical TEXT NOT NULL COLLATE NOCASE,
+ created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
`);
db.exec("INSERT OR IGNORE INTO source_members(source_id,user_id,role) SELECT id,user_id,'owner' FROM sources");
@@ -117,6 +121,10 @@ applyColumnMigration(16, "sources", "last_connection_error", "ALTER TABLE source
applyColumnMigration(17, "posts", "remote_url", "ALTER TABLE posts ADD COLUMN remote_url TEXT");
db.prepare("INSERT OR IGNORE INTO schema_migrations(version) VALUES(18)").run();
db.prepare("INSERT OR IGNORE INTO schema_migrations(version) VALUES(19)").run();
+applyColumnMigration(20, "sources", "attachment_storage_mode", "ALTER TABLE sources ADD COLUMN attachment_storage_mode TEXT NOT NULL DEFAULT 'remote'");
+applyColumnMigration(21, "sources", "attachment_cache_limit_bytes", "ALTER TABLE sources ADD COLUMN attachment_cache_limit_bytes INTEGER NOT NULL DEFAULT 104857600");
+applyColumnMigration(22, "sources", "attachment_cache_error", "ALTER TABLE sources ADD COLUMN attachment_cache_error TEXT");
+db.prepare("INSERT OR IGNORE INTO schema_migrations(version) VALUES(23)").run();
const admin = process.env.ADMIN_USERNAME;
const adminPassword = process.env.ADMIN_PASSWORD;
diff --git a/lib/tags.ts b/lib/tags.ts
new file mode 100644
index 0000000..6ae651c
--- /dev/null
+++ b/lib/tags.ts
@@ -0,0 +1,19 @@
+import { db } from "@/lib/db";
+
+export function canonicalTag(tag: string) {
+ const clean = tag.trim().replace(/^#/, "");
+ const alias = db.prepare("SELECT canonical FROM tag_aliases WHERE alias=? COLLATE NOCASE").get(clean) as { canonical: string } | undefined;
+ return alias?.canonical || clean;
+}
+
+export function canonicalTags(tags: string[]) { return [...new Set(tags.map(canonicalTag).filter(Boolean))]; }
+
+/** Removes only known tags from normal Markdown lines; fenced code is always untouched. */
+export function withoutInlineTags(content: string, tags: string[]) {
+ let fenced = false;
+ return content.split("\n").map((line) => {
+ if (/^\s*```/.test(line)) { fenced = !fenced; return line; }
+ if (fenced) return line;
+ return tags.reduce((text, tag) => text.replace(new RegExp(`(^|\\s)#${tag.replace(/[.*+?^${}()|[\\]\\\\]/g, "\\$&")}(?=\\s|$|[,。!?、,.!?])`, "gu"), "$1").replace(/ {2,}/g, " "), line);
+ }).join("\n");
+}
diff --git a/package-lock.json b/package-lock.json
index d0573d6..414cfe4 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "mebbling",
- "version": "0.5.0",
+ "version": "0.6.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "mebbling",
- "version": "0.5.0",
+ "version": "0.6.0",
"license": "PolyForm-Noncommercial-1.0.0",
"dependencies": {
"bcryptjs": "^3.0.3",
diff --git a/package.json b/package.json
index cdf62f8..8dfe888 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "mebbling",
- "version": "0.5.0",
+ "version": "0.6.0",
"description": "",
"private": true,
"scripts": {
diff --git a/tests/sync.test.ts b/tests/sync.test.ts
index c55d52f..87fb738 100644
--- a/tests/sync.test.ts
+++ b/tests/sync.test.ts
@@ -16,7 +16,7 @@ test("applies tracked migrations and deduplicates active pull jobs", async () =>
const { queuePull } = await import("../lib/sync");
const { notify } = await import("../lib/notifications");
const migrations = db.prepare("SELECT version FROM schema_migrations ORDER BY version").all() as { version: number }[];
- assert.deepEqual(migrations.map((item) => item.version), Array.from({ length: 19 }, (_, index) => index + 1));
+ assert.deepEqual(migrations.map((item) => item.version), Array.from({ length: 23 }, (_, index) => index + 1));
const userId = Number(db.prepare("INSERT INTO users(username,password_hash) VALUES('sync-test','hash')").run().lastInsertRowid);
const sourceId = Number(db.prepare("INSERT INTO sources(user_id,name,base_url,token_encrypted,is_enabled) VALUES(?,?,?,?,1)").run(userId, "Test", "https://example.test", "encrypted").lastInsertRowid);
assert.equal(queuePull(sourceId, "manual"), true);
diff --git a/worker/index.ts b/worker/index.ts
index 8b8c685..212ab0e 100644
--- a/worker/index.ts
+++ b/worker/index.ts
@@ -1,25 +1,60 @@
-import { readFile } from "node:fs/promises";
-import { join } from "node:path";
+import { readFile, mkdir, readdir, unlink, writeFile } from "node:fs/promises";
+import { extname, join } from "node:path";
+import { createHash } from "node:crypto";
import { db } from "../lib/db";
import { decrypt } from "../lib/crypto";
import { createMemo, createRemoteFile, getMemosIdentity, listMemos, memoUrl, setMemoAttachments } from "../lib/memos";
import { recordError } from "../lib/observability";
-type Source = { id: number; user_id: number; base_url: string; token_encrypted: string; is_enabled: number; sync_tags_json: string; sync_from: string | null; sync_to: string | null; sync_attachment_mode: "all" | "images" | "none" };
+type Source = { id: number; user_id: number; base_url: string; token_encrypted: string; is_enabled: number; sync_tags_json: string; sync_from: string | null; sync_to: string | null; sync_attachment_mode: "all" | "images" | "none"; attachment_storage_mode: "remote" | "images" | "all"; attachment_cache_limit_bytes: number };
type Job = { id: number; source_id: number; kind: "pull" | "push"; payload_json: string | null; attempts: number };
-function upsertRemote(source: Source, memo: any) {
- const tags = JSON.stringify(memo.tags || []), attachments = JSON.stringify(memo.attachments || memo.resources || []);
+function remoteAttachmentUrl(attachment: any, baseUrl: string) {
+ if (attachment.url || attachment.externalLink) return attachment.url || attachment.externalLink;
+ if (!attachment.name || !attachment.filename) return null;
+ return `${baseUrl.replace(/\/$/, "")}/file/${attachment.name.split("/").map(encodeURIComponent).join("/")}/${encodeURIComponent(attachment.filename)}`;
+}
+
+async function cacheAttachments(source: Source, attachments: any[]) {
+ if (source.attachment_storage_mode === "remote") return attachments;
+ const directory = join(process.cwd(), "public", "uploads", "cache", `source-${source.id}`); await mkdir(directory, { recursive: true });
+ let used = 0;
+ for (const row of db.prepare("SELECT attachments_json FROM posts WHERE source_id=?").all(source.id) as { attachments_json: string }[]) { try { used += (JSON.parse(row.attachments_json) as any[]).filter((item) => String(item.url || "").startsWith(`/uploads/cache/source-${source.id}/`)).reduce((sum, item) => sum + Number(item.size || 0), 0); } catch {} }
+ const result: any[] = [];
+ for (const attachment of attachments) {
+ const url = remoteAttachmentUrl(attachment, source.base_url); const type = attachment.type || "";
+ if (!url || (source.attachment_storage_mode === "images" && !type.startsWith("image/"))) { result.push(attachment); continue; }
+ try {
+ const target = new URL(url); if (target.origin !== new URL(source.base_url).origin) throw new Error("Attachment host is not the source host");
+ const response = await fetch(url, { signal: AbortSignal.timeout(15_000) }); if (!response.ok) throw new Error(`Attachment download ${response.status}`);
+ const body = Buffer.from(await response.arrayBuffer()); if (used + body.length > source.attachment_cache_limit_bytes) throw new Error("Attachment cache quota exceeded");
+ const filename = attachment.filename || attachment.name || "attachment"; const key = createHash("sha256").update(url).digest("hex").slice(0, 24) + extname(filename);
+ await writeFile(join(directory, key), body); used += body.length;
+ result.push({ ...attachment, originalUrl: url, url: `/uploads/cache/source-${source.id}/${key}`, type: type || response.headers.get("content-type") || "application/octet-stream", size: body.length });
+ } catch (error) { recordError("attachment-cache", error, { sourceId: source.id, url }); result.push(attachment); }
+ }
+ return result;
+}
+
+async function cleanupCache(source: Source) {
+ const directory = join(process.cwd(), "public", "uploads", "cache", `source-${source.id}`); let names: string[]; try { names = await readdir(directory); } catch { return; }
+ const used = new Set(); for (const row of db.prepare("SELECT attachments_json FROM posts WHERE source_id=?").all(source.id) as { attachments_json: string }[]) { try { for (const attachment of JSON.parse(row.attachments_json) as any[]) { const url = String(attachment.url || ""); if (url.startsWith(`/uploads/cache/source-${source.id}/`)) used.add(url.split("/").at(-1)!); } } catch {} }
+ await Promise.all(names.filter((name) => !used.has(name)).map((name) => unlink(join(directory, name)).catch(() => undefined)));
+}
+
+async function upsertRemote(source: Source, memo: any) {
+ const tags = JSON.stringify(memo.tags || []), attachments = JSON.stringify(await cacheAttachments(source, memo.attachments || memo.resources || []));
db.prepare(`INSERT INTO posts(source_id,author_id,remote_memo_name,content,visibility,tags_json,attachments_json,origin,remote_created_at,remote_updated_at,sync_status,hidden,remote_url) VALUES(?,?,?,?,?,?,?,?,?,?, 'synced',0,?) ON CONFLICT(source_id,remote_memo_name) DO UPDATE SET content=excluded.content,visibility=excluded.visibility,tags_json=excluded.tags_json,attachments_json=excluded.attachments_json,remote_updated_at=excluded.remote_updated_at,remote_url=excluded.remote_url,hidden=0,updated_at=CURRENT_TIMESTAMP`).run(source.id, source.user_id, memo.name, memo.content, memo.visibility, tags, attachments, "memos", memo.createTime || null, memo.updateTime || null, memoUrl(source.base_url, memo.name));
}
async function pull(source: Source) {
const token = decrypt(source.token_encrypted); const rules = { tags: JSON.parse(source.sync_tags_json || "[]") as string[], from: source.sync_from, to: source.sync_to, attachmentMode: source.sync_attachment_mode };
const [memos, identity] = await Promise.all([listMemos(source.base_url, token, rules), getMemosIdentity(source.base_url, token)]);
- for (const memo of memos) upsertRemote(source, memo);
+ for (const memo of memos) await upsertRemote(source, memo);
const names = memos.map((memo) => memo.name);
if (names.length) { const placeholders = names.map(() => "?").join(","); db.prepare(`UPDATE posts SET hidden=1,updated_at=CURRENT_TIMESTAMP WHERE source_id=? AND remote_memo_name IS NOT NULL AND remote_memo_name NOT IN (${placeholders})`).run(source.id, ...names); }
else db.prepare("UPDATE posts SET hidden=1,updated_at=CURRENT_TIMESTAMP WHERE source_id=? AND remote_memo_name IS NOT NULL").run(source.id);
+ await cleanupCache(source);
const avatar = identity.avatarUrl || identity.avatar || null; const avatarUrl = avatar?.startsWith("/") ? `${source.base_url.replace(/\/$/, "")}${avatar}` : avatar;
db.prepare("UPDATE sources SET sync_status='synced',last_synced_at=CURRENT_TIMESTAMP,last_error=NULL,last_connection_at=CURRENT_TIMESTAMP,last_connection_error=NULL,remote_display_name=?,remote_avatar_url=? WHERE id=?").run(identity.nickname || identity.username || identity.name, avatarUrl, source.id);
}