diff --git a/README.md b/README.md index e29234e9..12fc8902 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ The public demo resets every day at 3:00 AM (China Standard Time) and restores s - **Public Note Sharing**: Share a note publicly and stop sharing it at any time. - **WeChat Article Clipping on Mobile**: Share a WeChat Official Account article to EdgeEver on your phone to extract its content and save it as an editable note. - **Smart Local Image Compression**: Client-side WebP compression reduces file sizes by 50%-90% before uploading, saving storage and speeding up page loads without extra server costs. -- **Universal File Attachments**: Attach and preview PDFs, Office documents, zip files, audio, and video directly within notes. +- **Universal File Attachments**: Attach and preview PDFs, Office documents, zip files, audio, and video directly within notes. Retryable 8 MiB multipart uploads, streamed downloads, streaming ZIP export/restore, chunked desktop offline staging, and disk-backed WebDAV backups safely support attachments up to 1 GiB across Cloudflare and Docker deployments. - **Batch Operations & Flexible Sorting**: Easily merge or relocate multiple notes, with drag-and-drop notebook reordering. - **Offline Drafts & Queueing**: Draft and edit uninterrupted while offline; changes automatically sync once reconnected. - **Brute-Force Login Protection**: Server-side account- and IP-based failed-login throttling with automatic cooldowns helps protect private notes against brute-force and password-spraying attacks. diff --git a/README.zh-CN.md b/README.zh-CN.md index cae4045f..fedba3db 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -52,7 +52,7 @@ EdgeEver 是一款现代化的开源笔记工作区。它为你找回经典印 - **公开笔记分享**:支持公开分享笔记,并可随时取消分享。 - **移动 App 微信公众号文章剪藏**:在手机上将微信公众号文章分享至 EdgeEver,即可提取正文并保存为可继续编辑的笔记。 - **智能前端图片压缩**:图片上传前在浏览器端静默完成压缩,常见截图与大图精简 50%-90% 体积,加载更迅速、存储更省心。 -- **通用文件附件支持**:支持轻松上传并插入 PDF、Office 文档、压缩包及音视频等各种附件。 +- **通用文件附件支持**:支持轻松上传并插入 PDF、Office 文档、压缩包及音视频等各种附件;Cloudflare 与 Docker 部署均通过可重试的 8 MiB 分片上传、流式下载、流式 ZIP 导出/恢复、桌面端离线分片暂存和磁盘暂存 WebDAV 备份,安全支持最大 1 GiB 附件。 - **高效多选与批量操作**:支持笔记批量合并、批量移动,以及笔记本拖拽排序与层级调整。 - **离线草稿与同步队列**:网络不稳定时自动保存离线草稿,恢复连线后自动入队同步。 - **登录防暴力破解保护**:服务端按账号与 IP 记录失败登录并自动限流、冷却,降低暴力破解与密码喷洒攻击风险,守护私密笔记数据。 diff --git a/apps/api/src/backup-routes.test.mjs b/apps/api/src/backup-routes.test.mjs index f273cef3..d7507e19 100644 --- a/apps/api/src/backup-routes.test.mjs +++ b/apps/api/src/backup-routes.test.mjs @@ -84,6 +84,7 @@ const createDependencies = (overrides = {}) => ({ restoreJsonMemos: async () => {}, restoreJsonNotebooks: async () => {}, sha256Bytes: async () => "checksum", + initiateResourceRestoreUpload: async () => { throw new Error("Unexpected restore upload"); }, ...overrides, }); @@ -173,6 +174,52 @@ describe("backup route contracts", () => { expect(restored.slice(1)).toEqual(["ws_1", payload.notebooks]); }); + test("initializes a resumable resource restore with the original resource id", async () => { + let received; + const metadata = { + id: "res_restore", + memoId: "memo_1", + originalMemoId: null, + kind: "attachment", + mimeType: "application/octet-stream", + filename: "archive.bin", + byteSize: 600 * 1024 * 1024, + sha256: "checksum", + width: null, + height: null, + createdAt: "2026-08-08T00:00:00.000Z", + updatedAt: "2026-08-08T00:00:00.000Z", + archivePath: "resources/res_restore/archive.bin", + }; + const response = await createApp(createDependencies({ + initiateResourceRestoreUpload: async (_context, input) => { + received = input; + return { + id: "upload_restore", + resourceId: input.id, + partSize: 8 * 1024 * 1024, + partCount: 75, + byteSize: input.byteSize, + expiresAt: "2026-09-01T00:00:00.000Z", + }; + }, + }), userAuth).request( + "/api/v1/restores/json/resources/res_restore/uploads", + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(metadata), + }, + { storage: { db: createDatabase(), resources: {} } }, + ); + + expect(response.status).toBe(201); + expect(received).toEqual(metadata); + expect(await response.json()).toMatchObject({ + upload: { resourceId: "res_restore", partCount: 75 }, + }); + }); + test("maps damaged revision JSON to an empty document", () => { expect(mapJsonBackupRevision({ id: "rev_1", diff --git a/apps/api/src/backup-routes.ts b/apps/api/src/backup-routes.ts index f0e006db..c4e7ca4d 100644 --- a/apps/api/src/backup-routes.ts +++ b/apps/api/src/backup-routes.ts @@ -14,6 +14,7 @@ import { import { zValidator } from "@hono/zod-validator"; import type { Hono } from "hono"; import type { AppEnv } from "./api-context"; +import { AppError } from "./app-error"; import { isoNow, parseJsonArray } from "./entity-utils"; import { apiError, badRequest, conflict, notFound } from "./http-errors"; import { resolveObjectStorage } from "./object-storage"; @@ -26,6 +27,7 @@ import { } from "./resource-service"; import { getWorkspaceId, requireScopes, requireUser } from "./request-auth"; import type { DatabaseAdapter } from "./storage-contract"; +import type { initiateResourceRestoreUpload as initiateResourceRestoreUploadService } from "./resource-upload-service"; export type BackupMemoDetailRow = { id: string; @@ -82,8 +84,11 @@ type BackupRouteDependencies = { notebooks: JsonBackupNotebook[], ) => Promise; sha256Bytes: (bytes: Uint8Array) => Promise; + initiateResourceRestoreUpload: typeof initiateResourceRestoreUploadService; }; +const MAX_LEGACY_RESTORE_BYTES = 100 * 1024 * 1024; + const parseRevisionDoc = (json: string): TiptapDoc => { try { const value = JSON.parse(json); @@ -243,10 +248,38 @@ export const registerBackupRoutes = ( }, ); + app.post("/api/v1/restores/json/resources/:id/uploads", async (context) => { + const denied = requireUser(context); + if (denied) return denied; + const metadata = await context.req.json().catch(() => null); + if (!metadata || typeof metadata !== "object" || (metadata as { id?: unknown }).id !== context.req.param("id")) { + return badRequest(context, "Restore resource metadata is invalid."); + } + try { + const upload = await dependencies.initiateResourceRestoreUpload(context, metadata); + return context.json({ upload }, 201); + } catch (error) { + if (error instanceof AppError) { + return apiError(context, error.code, error.message, error.status); + } + throw error; + } + }); + app.put("/api/v1/restores/json/resources/:id", async (context) => { const denied = requireUser(context); if (denied) return denied; + const declaredRequestBytes = Number(context.req.header("Content-Length")); + if (Number.isFinite(declaredRequestBytes) && declaredRequestBytes > MAX_LEGACY_RESTORE_BYTES + 1024 * 1024) { + return apiError( + context, + "multipart_upload_required", + "Backup resources larger than 100 MiB must use the resumable restore API.", + 413, + ); + } + const form = await context.req.raw.formData(); const file = form.get("file"); const metadataValue = form.get("metadata"); @@ -273,7 +306,7 @@ export const registerBackupRoutes = ( } const maxBytes = metadata.kind === "image" ? MAX_IMAGE_UPLOAD_BYTES : MAX_ATTACHMENT_UPLOAD_BYTES; - if (file.size <= 0 || file.size > maxBytes) { + if (file.size <= 0 || file.size > Math.min(maxBytes, MAX_LEGACY_RESTORE_BYTES)) { return apiError(context, "upload_too_large", "Backup resource size is invalid.", 413); } diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 95f40920..abbbaf1b 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -76,6 +76,13 @@ import { registerObjectStorageRoutes } from "./object-storage-routes"; import { registerAiRoutes } from "./ai-routes"; import { registerAiPromptRoutes } from "./ai-prompt-routes"; import { registerResourceRoutes } from "./resource-routes"; +import { + abortResourceUpload, + completeResourceUpload, + initiateResourceRestoreUpload, + initiateResourceUpload, + uploadResourcePart, +} from "./resource-upload-service"; import { registerPluginDistributionRoutes } from "./plugin-distribution-routes"; import { registerSyncRoutes } from "./sync-routes"; import { registerMemoRoutes } from "./memo-routes"; @@ -333,6 +340,7 @@ registerBackupRoutes(app, { restoreJsonMemos: restoreJsonMemosService, restoreJsonNotebooks: restoreJsonNotebooksService, sha256Bytes, + initiateResourceRestoreUpload: (...args) => initiateResourceRestoreUpload(...args), }); registerMcpRoutes(app, { @@ -347,6 +355,10 @@ registerResourceRoutes(app, { createImageResource: (...args) => createImageResource(...args), getMemoDetail: (...args) => getMemoDetail(...args), getResourceRow: (...args) => getResourceRow(...args), + initiateResourceUpload: (...args) => initiateResourceUpload(...args), + uploadResourcePart: (...args) => uploadResourcePart(...args), + completeResourceUpload: (...args) => completeResourceUpload(...args), + abortResourceUpload: (...args) => abortResourceUpload(...args), }); app.post("/api/v1/demo/reset", async (c) => { diff --git a/apps/api/src/resource-routes.test.mjs b/apps/api/src/resource-routes.test.mjs index 8beb6c29..a4649fe5 100644 --- a/apps/api/src/resource-routes.test.mjs +++ b/apps/api/src/resource-routes.test.mjs @@ -55,7 +55,7 @@ const createEnvironment = () => ({ }, }); -const createApp = (auth = agentAuth, getResourceRow = async () => null) => { +const createApp = (auth = agentAuth, getResourceRow = async () => null, overrides = {}) => { const app = new Hono(); app.onError((error, context) => context.json({ error: { message: error.message } }, 500)); app.use("/api/v1/*", async (context, next) => { @@ -68,6 +68,11 @@ const createApp = (auth = agentAuth, getResourceRow = async () => null) => { createImageResource: async () => { throw new Error("Unexpected upload"); }, getMemoDetail: async () => null, getResourceRow, + initiateResourceUpload: async () => { throw new Error("Unexpected multipart initialization"); }, + uploadResourcePart: async () => { throw new Error("Unexpected multipart part"); }, + completeResourceUpload: async () => { throw new Error("Unexpected multipart completion"); }, + abortResourceUpload: async () => { throw new Error("Unexpected multipart abort"); }, + ...overrides, }); return app; }; @@ -108,6 +113,75 @@ describe("resource route contracts", () => { expect(await response.json()).toMatchObject({ error: { code: "forbidden" } }); }); + test("initializes a bounded multipart upload after memo authorization", async () => { + let input; + const response = await createApp( + { ...agentAuth, scopes: ["write:resources"] }, + async () => null, + { + getMemoDetail: async () => ({ id: "memo_1" }), + initiateResourceUpload: async (_context, value) => { + input = value; + return { + id: "upload_1", + resourceId: "res_1", + partSize: 8 * 1024 * 1024, + partCount: 2, + byteSize: 9 * 1024 * 1024, + expiresAt: "2026-09-01T00:00:00.000Z", + }; + }, + }, + ).request( + "/api/v1/memos/memo_1/resource-uploads", + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + filename: "archive.zip", + mimeType: "application/zip", + byteSize: 9 * 1024 * 1024, + }), + }, + createEnvironment(), + ); + + expect(response.status).toBe(201); + expect(input).toEqual({ + memoId: "memo_1", + filename: "archive.zip", + mimeType: "application/zip", + byteSize: 9 * 1024 * 1024, + }); + expect(await response.json()).toMatchObject({ upload: { id: "upload_1", partCount: 2 } }); + }); + + test("streams an exact-sized upload part to the multipart service", async () => { + let streamed = ""; + const response = await createApp( + { ...agentAuth, scopes: ["write:resources"] }, + async () => null, + { + uploadResourcePart: async (_context, uploadId, partNumber, body, byteSize) => { + streamed = await new Response(body).text(); + expect({ uploadId, partNumber, byteSize }).toEqual({ uploadId: "upload_1", partNumber: 2, byteSize: 4 }); + return { partNumber, byteSize }; + }, + }, + ).request( + "/api/v1/resource-uploads/upload_1/parts/2", + { + method: "PUT", + headers: { "Content-Type": "application/octet-stream", "Content-Length": "4" }, + body: "part", + }, + createEnvironment(), + ); + + expect(response.status).toBe(200); + expect(streamed).toBe("part"); + }); + test("serves PDF attachments with UTF-8 filenames inline using an ASCII-safe header", async () => { const environment = createEnvironment(); environment.storage.resources = { diff --git a/apps/api/src/resource-routes.ts b/apps/api/src/resource-routes.ts index e33f82f9..75a23ea8 100644 --- a/apps/api/src/resource-routes.ts +++ b/apps/api/src/resource-routes.ts @@ -20,6 +20,7 @@ import { type ResourceRow, type ResourceStatsRow, } from "./resource-service"; +import type { initiateResourceUpload as initiateResourceUploadService } from "./resource-upload-service"; import { getAuditActor, getWorkspaceId, requireScopes } from "./request-auth"; import type { DatabaseAdapter } from "./storage-contract"; @@ -51,6 +52,29 @@ type ResourceRouteDependencies = { resourceId: string, includeDeleted?: boolean, ) => Promise; + initiateResourceUpload: typeof initiateResourceUploadService; + uploadResourcePart: ( + context: AppContext, + uploadId: string, + partNumber: number, + body: ReadableStream, + byteSize: number, + ) => Promise<{ partNumber: number; byteSize: number }>; + completeResourceUpload: ( + context: AppContext, + uploadId: string, + actor: AuditActor, + ) => Promise; + abortResourceUpload: (context: AppContext, uploadId: string) => Promise; +}; + +const MAX_LEGACY_RESOURCE_UPLOAD_BYTES = 100 * 1024 * 1024; + +const appErrorResponse = (context: AppContext, error: unknown) => { + if (error instanceof AppError) { + return apiError(context, error.code, error.message, error.status); + } + throw error; }; export const registerResourceRoutes = ( @@ -96,6 +120,19 @@ export const registerResourceRoutes = ( const denied = requireScopes(context, "write:resources"); if (denied) return denied; + const declaredRequestBytes = Number(context.req.header("Content-Length")); + if ( + Number.isFinite(declaredRequestBytes) + && declaredRequestBytes > MAX_LEGACY_RESOURCE_UPLOAD_BYTES + 1024 * 1024 + ) { + return apiError( + context, + "multipart_upload_required", + "Files larger than 100 MiB must use the resumable multipart upload API.", + 413, + ); + } + const memoId = context.req.param("id"); const memo = await dependencies.getMemoDetail( context.env.storage.db, @@ -109,6 +146,14 @@ export const registerResourceRoutes = ( if (!(file instanceof File)) { return badRequest(context, "Expected multipart form field named file."); } + if (file.size > MAX_LEGACY_RESOURCE_UPLOAD_BYTES) { + return apiError( + context, + "multipart_upload_required", + "Files larger than 100 MiB must use the resumable multipart upload API.", + 413, + ); + } const input = { memoId, @@ -123,15 +168,101 @@ export const registerResourceRoutes = ( ? await dependencies.createImageResource(context, { ...input, source: "upload" }) : await dependencies.createAttachmentResource(context, input); } catch (error) { - if (error instanceof AppError) { - return apiError(context, error.code, error.message, error.status); - } - throw error; + return appErrorResponse(context, error); } return context.json({ resource }, 201); }); + app.post("/api/v1/memos/:id/resource-uploads", async (context) => { + const denied = requireScopes(context, "write:resources"); + if (denied) return denied; + + const memoId = context.req.param("id"); + const memo = await dependencies.getMemoDetail( + context.env.storage.db, + getWorkspaceId(context), + memoId, + ); + if (!memo) return notFound(context, "Memo not found"); + + const payload = await context.req.json().catch(() => null) as Record | null; + const filename = typeof payload?.filename === "string" ? payload.filename : ""; + const mimeType = typeof payload?.mimeType === "string" && payload.mimeType.trim() + ? payload.mimeType.trim() + : "application/octet-stream"; + const byteSize = Number(payload?.byteSize); + if (!filename.trim() || !Number.isSafeInteger(byteSize) || byteSize <= 0) { + return badRequest(context, "filename and a positive integer byteSize are required."); + } + + try { + const upload = await dependencies.initiateResourceUpload(context, { + memoId, + filename, + mimeType, + byteSize, + }); + return context.json({ upload }, 201); + } catch (error) { + return appErrorResponse(context, error); + } + }); + + app.put("/api/v1/resource-uploads/:id/parts/:partNumber", async (context) => { + const denied = requireScopes(context, "write:resources"); + if (denied) return denied; + + const body = context.req.raw.body; + const byteSize = Number(context.req.header("Content-Length")); + if (!body || !Number.isSafeInteger(byteSize) || byteSize <= 0) { + return apiError( + context, + "invalid_upload_part", + "Multipart upload parts require a positive Content-Length header.", + 411, + ); + } + try { + const part = await dependencies.uploadResourcePart( + context, + context.req.param("id"), + Number(context.req.param("partNumber")), + body, + byteSize, + ); + return context.json({ part }); + } catch (error) { + return appErrorResponse(context, error); + } + }); + + app.post("/api/v1/resource-uploads/:id/complete", async (context) => { + const denied = requireScopes(context, "write:resources"); + if (denied) return denied; + try { + const resource = await dependencies.completeResourceUpload( + context, + context.req.param("id"), + getAuditActor(context), + ); + return context.json({ resource }, 201); + } catch (error) { + return appErrorResponse(context, error); + } + }); + + app.delete("/api/v1/resource-uploads/:id", async (context) => { + const denied = requireScopes(context, "write:resources"); + if (denied) return denied; + try { + await dependencies.abortResourceUpload(context, context.req.param("id")); + return context.json({ ok: true }); + } catch (error) { + return appErrorResponse(context, error); + } + }); + app.get("/api/v1/resources/:id/blob", async (context) => { const denied = requireScopes(context, "read:resources"); if (denied) return denied; diff --git a/apps/api/src/resource-service.ts b/apps/api/src/resource-service.ts index d0518b03..abfe7a27 100644 --- a/apps/api/src/resource-service.ts +++ b/apps/api/src/resource-service.ts @@ -13,7 +13,7 @@ import { getWorkspaceId } from "./request-auth"; import type { DatabaseAdapter } from "./storage-contract"; export const MAX_IMAGE_UPLOAD_BYTES = 100 * 1024 * 1024; -export const MAX_ATTACHMENT_UPLOAD_BYTES = 100 * 1024 * 1024; +export const MAX_ATTACHMENT_UPLOAD_BYTES = 1024 * 1024 * 1024; export const SUPPORTED_IMAGE_MIME_TYPES = new Set([ "image/png", @@ -106,13 +106,13 @@ export const validateImageUpload = (mimeType: string, size: number) => { ); } if (size <= 0 || size > MAX_IMAGE_UPLOAD_BYTES) { - throw new AppError("upload_too_large", "Image must be between 1 byte and 50 MB.", 413); + throw new AppError("upload_too_large", "Image must be between 1 byte and 100 MiB.", 413); } }; export const validateAttachmentUpload = (size: number) => { if (size <= 0 || size > MAX_ATTACHMENT_UPLOAD_BYTES) { - throw new AppError("upload_too_large", "Attachment must be between 1 byte and 100 MB.", 413); + throw new AppError("upload_too_large", "Attachment must be between 1 byte and 1 GiB.", 413); } }; diff --git a/apps/api/src/resource-upload-integration.test.mjs b/apps/api/src/resource-upload-integration.test.mjs new file mode 100644 index 00000000..4b06a2df --- /dev/null +++ b/apps/api/src/resource-upload-integration.test.mjs @@ -0,0 +1,141 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { Database } from "bun:sqlite"; +import { mkdtemp, readdir, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fetchEdgeEverApp } from "./index.ts"; +import { createSelfHostedStorageAdapter } from "./self-hosted-storage-adapter.ts"; + +const temporaryDirectories = []; +afterEach(async () => { + await Promise.all(temporaryDirectories.splice(0).map((directory) => rm(directory, { recursive: true, force: true }))); +}); + +const requestJson = async (environment, path, init = {}) => { + const response = await fetchEdgeEverApp( + new Request(`http://edgeever.test${path}`, init), + environment, + { waitUntil: () => undefined, passThroughOnException: () => undefined }, + ); + const body = await response.json(); + if (!response.ok) throw new Error(`${response.status}: ${JSON.stringify(body)}`); + return body; +}; + +describe("resource multipart upload integration", () => { + test("assembles a large resource through the shared API and filesystem adapter", async () => { + const directory = await mkdtemp(`${tmpdir()}/edgeever-upload-integration-`); + temporaryDirectories.push(directory); + const database = new Database(join(directory, "edgeever.sqlite"), { create: true }); + database.exec("PRAGMA foreign_keys = ON"); + const migrationDirectory = new URL("../../../migrations/", import.meta.url); + for (const name of (await readdir(migrationDirectory)).filter((value) => value.endsWith(".sql")).sort()) { + database.exec(await readFile(new URL(name, migrationDirectory), "utf8")); + } + database.query( + "INSERT OR IGNORE INTO workspaces (id, name, is_personal) VALUES ('ws_default', 'Personal', 1)", + ).run(); + + const resourcesDirectory = join(directory, "resources"); + const environment = { + storage: createSelfHostedStorageAdapter(database, resourcesDirectory), + EDGE_EVER_ALLOW_UNAUTHENTICATED: "true", + EDGE_EVER_RUNTIME: "self-hosted-bun", + }; + const notebook = await requestJson(environment, "/api/v1/notebooks", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ name: "Uploads" }), + }); + const memo = await requestJson(environment, "/api/v1/memos", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ notebookId: notebook.notebook.id, title: "Multipart", contentMarkdown: "" }), + }); + + const file = new Uint8Array(8 * 1024 * 1024 + 3); + file.fill(7); + file.set([1, 2, 3], file.byteLength - 3); + const started = await requestJson(environment, `/api/v1/memos/${memo.memo.id}/resource-uploads`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ filename: "archive.bin", mimeType: "application/octet-stream", byteSize: file.byteLength }), + }); + expect(started.upload.partCount).toBe(2); + + for (let partNumber = 1; partNumber <= started.upload.partCount; partNumber += 1) { + const start = (partNumber - 1) * started.upload.partSize; + const chunk = file.slice(start, Math.min(start + started.upload.partSize, file.byteLength)); + await requestJson(environment, `/api/v1/resource-uploads/${started.upload.id}/parts/${partNumber}`, { + method: "PUT", + headers: { "Content-Type": "application/octet-stream", "Content-Length": String(chunk.byteLength) }, + body: chunk, + }); + } + const completed = await requestJson(environment, `/api/v1/resource-uploads/${started.upload.id}/complete`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: "{}", + }); + + expect(completed.resource).toMatchObject({ + memoId: memo.memo.id, + filename: "archive.bin", + byteSize: file.byteLength, + sha256: null, + }); + const row = database.query("SELECT object_key FROM resources WHERE id = ?").get(completed.resource.id); + expect(await readFile(join(resourcesDirectory, row.object_key))).toEqual(file); + expect(database.query("SELECT COUNT(*) AS count FROM resource_uploads").get().count).toBe(0); + + const restoredBytes = new Uint8Array([9, 8, 7, 6, 5]); + const restoredMetadata = { + id: "res_restored", + memoId: memo.memo.id, + originalMemoId: null, + kind: "attachment", + mimeType: "application/octet-stream", + filename: "restored.bin", + byteSize: restoredBytes.byteLength, + sha256: "backup-checksum", + width: null, + height: null, + createdAt: "2026-08-31T00:00:00.000Z", + updatedAt: "2026-08-31T00:00:00.000Z", + archivePath: "resources/res_restored/restored.bin", + }; + const restoreStarted = await requestJson( + environment, + "/api/v1/restores/json/resources/res_restored/uploads", + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(restoredMetadata), + }, + ); + expect(restoreStarted.upload.resourceId).toBe("res_restored"); + await requestJson(environment, `/api/v1/resource-uploads/${restoreStarted.upload.id}/parts/1`, { + method: "PUT", + headers: { + "Content-Type": "application/octet-stream", + "Content-Length": String(restoredBytes.byteLength), + }, + body: restoredBytes, + }); + const restored = await requestJson(environment, `/api/v1/resource-uploads/${restoreStarted.upload.id}/complete`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: "{}", + }); + expect(restored.resource).toMatchObject({ + id: "res_restored", + memoId: memo.memo.id, + filename: "restored.bin", + byteSize: restoredBytes.byteLength, + sha256: "backup-checksum", + }); + const restoredRow = database.query("SELECT object_key FROM resources WHERE id = ?").get("res_restored"); + expect(await readFile(join(resourcesDirectory, restoredRow.object_key))).toEqual(restoredBytes); + database.close(); + }); +}); diff --git a/apps/api/src/resource-upload-service.ts b/apps/api/src/resource-upload-service.ts new file mode 100644 index 00000000..2df664f4 --- /dev/null +++ b/apps/api/src/resource-upload-service.ts @@ -0,0 +1,502 @@ +import { + JsonBackupResourceMetadataSchema, + type JsonBackupResource, + type Resource, +} from "@edgeever/shared"; +import { auditStatement } from "./audit"; +import type { AppContext, AuditActor } from "./api-context"; +import { AppError } from "./app-error"; +import { createId, isoNow } from "./entity-utils"; +import { resolveObjectStorage } from "./object-storage"; +import { + SUPPORTED_IMAGE_MIME_TYPES, + getResourceRow, + inferImageExtension, + mapResource, + MAX_ATTACHMENT_UPLOAD_BYTES, + MAX_IMAGE_UPLOAD_BYTES, + normalizeFilename, + validateAttachmentUpload, + validateImageUpload, +} from "./resource-service"; +import { getWorkspaceId } from "./request-auth"; +import type { BlobUploadedPart } from "./storage-contract"; + +export const RESOURCE_UPLOAD_PART_BYTES = 8 * 1024 * 1024; +export const RESOURCE_UPLOAD_TTL_MS = 24 * 60 * 60 * 1000; + +export type ResourceUploadRow = { + id: string; + workspace_id: string; + memo_id: string; + resource_id: string; + storage_config_id: string; + bucket_name: string; + object_key: string; + provider_upload_id: string; + kind: "image" | "attachment"; + mime_type: string; + filename: string; + byte_size: number; + part_size: number; + part_count: number; + restore_metadata_json: string | null; + created_at: string; + updated_at: string; + expires_at: string; +}; + +type ResourceUploadPartRow = { + part_number: number; + etag: string; + byte_size: number; +}; + +const uploadNotFound = () => new AppError("upload_not_found", "Resource upload was not found.", 404); + +const assertActiveUpload = (upload: ResourceUploadRow | null): ResourceUploadRow => { + if (!upload) throw uploadNotFound(); + if (Date.parse(upload.expires_at) <= Date.now()) { + throw new AppError("upload_expired", "Resource upload has expired.", 410); + } + return upload; +}; + +const uploadHttpMetadata = (mimeType: string) => ({ + httpMetadata: { + contentType: mimeType, + cacheControl: "private, max-age=3600", + }, +}); + +const cleanupExpiredResourceUploads = async (context: AppContext) => { + const expired = await context.env.storage.db.prepare( + `SELECT id, workspace_id, memo_id, resource_id, storage_config_id, bucket_name, object_key, + provider_upload_id, kind, mime_type, filename, byte_size, part_size, part_count, restore_metadata_json, + created_at, updated_at, expires_at + FROM resource_uploads + WHERE workspace_id = ? AND expires_at <= ? + ORDER BY expires_at ASC LIMIT 10`, + ).bind(getWorkspaceId(context), isoNow()).all(); + + for (const upload of expired.results) { + try { + const destination = await resolveObjectStorage(context.env, upload.storage_config_id); + await destination.store + .resumeMultipartUpload(upload.object_key, upload.provider_upload_id) + .abort(); + } catch { + // Provider multipart uploads have their own expiry lifecycle. Remove the + // inaccessible application session even if the provider already did so. + } + await context.env.storage.db.prepare("DELETE FROM resource_uploads WHERE id = ?").bind(upload.id).run(); + } +}; + +export const getResourceUpload = ( + context: AppContext, + uploadId: string, +) => context.env.storage.db.prepare( + `SELECT id, workspace_id, memo_id, resource_id, storage_config_id, bucket_name, object_key, + provider_upload_id, kind, mime_type, filename, byte_size, part_size, part_count, restore_metadata_json, + created_at, updated_at, expires_at + FROM resource_uploads WHERE id = ? AND workspace_id = ?`, +).bind(uploadId, getWorkspaceId(context)).first(); + +export const initiateResourceUpload = async ( + context: AppContext, + input: { memoId: string; filename: string; mimeType: string; byteSize: number }, +) => { + await cleanupExpiredResourceUploads(context); + const kind = SUPPORTED_IMAGE_MIME_TYPES.has(input.mimeType) ? "image" : "attachment"; + if (kind === "image") validateImageUpload(input.mimeType, input.byteSize); + else validateAttachmentUpload(input.byteSize); + + const uploadId = createId("upload"); + const resourceId = createId("res"); + const workspaceId = getWorkspaceId(context); + const filename = normalizeFilename(input.filename) || resourceId; + const extension = kind === "image" ? inferImageExtension(filename, input.mimeType) : ""; + const objectKey = `workspaces/${workspaceId}/memos/${input.memoId}/${resourceId}${extension}`; + const destination = await resolveObjectStorage(context.env); + const multipart = await destination.store.createMultipartUpload( + objectKey, + uploadHttpMetadata(input.mimeType), + ); + const now = isoNow(); + const expiresAt = new Date(Date.now() + RESOURCE_UPLOAD_TTL_MS).toISOString(); + const partCount = Math.ceil(input.byteSize / RESOURCE_UPLOAD_PART_BYTES); + + try { + await context.env.storage.db.prepare( + `INSERT INTO resource_uploads ( + id, workspace_id, memo_id, resource_id, storage_config_id, bucket_name, object_key, + provider_upload_id, kind, mime_type, filename, byte_size, part_size, part_count, restore_metadata_json, + created_at, updated_at, expires_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ).bind( + uploadId, + workspaceId, + input.memoId, + resourceId, + destination.configId, + destination.bucketName, + objectKey, + multipart.uploadId, + kind, + input.mimeType, + filename, + input.byteSize, + RESOURCE_UPLOAD_PART_BYTES, + partCount, + null, + now, + now, + expiresAt, + ).run(); + } catch (error) { + await multipart.abort().catch(() => undefined); + throw error; + } + + return { + id: uploadId, + resourceId, + partSize: RESOURCE_UPLOAD_PART_BYTES, + partCount, + byteSize: input.byteSize, + expiresAt, + }; +}; + +export const initiateResourceRestoreUpload = async ( + context: AppContext, + input: unknown, +) => { + await cleanupExpiredResourceUploads(context); + const parsed = JsonBackupResourceMetadataSchema.safeParse(input); + if (!parsed.success) { + throw new AppError("invalid_restore_resource", "Restore resource metadata is invalid.", 400); + } + const metadata = parsed.data as JsonBackupResource; + const maxBytes = metadata.kind === "image" ? MAX_IMAGE_UPLOAD_BYTES : MAX_ATTACHMENT_UPLOAD_BYTES; + if (!Number.isSafeInteger(metadata.byteSize) || metadata.byteSize <= 0 || metadata.byteSize > maxBytes) { + throw new AppError("upload_too_large", "Backup resource size is invalid.", 413); + } + + const workspaceId = getWorkspaceId(context); + const memo = await context.env.storage.db.prepare( + "SELECT id FROM memos WHERE id = ? AND workspace_id = ? LIMIT 1", + ).bind(metadata.memoId, workspaceId).first<{ id: string }>(); + if (!memo) { + throw new AppError("not_found", "Restore target memo not found.", 404); + } + const foreignResource = await context.env.storage.db.prepare( + `SELECT r.id FROM resources r INNER JOIN memos m ON m.id = r.memo_id + WHERE r.id = ? AND m.workspace_id <> ? LIMIT 1`, + ).bind(metadata.id, workspaceId).first<{ id: string }>(); + if (foreignResource) { + throw new AppError( + "cross_workspace_id_conflict", + "Backup resource ID is already used by another user.", + 409, + ); + } + + const uploadId = createId("upload"); + const filename = normalizeFilename(metadata.filename || "") || `${metadata.kind}-${metadata.id}`; + const mimeType = metadata.mimeType?.trim() || "application/octet-stream"; + const objectKey = `workspaces/${workspaceId}/restores/${metadata.memoId}/${metadata.id}/${Date.now()}-${filename}`; + const destination = await resolveObjectStorage(context.env); + const multipart = await destination.store.createMultipartUpload( + objectKey, + uploadHttpMetadata(mimeType), + ); + const now = isoNow(); + const expiresAt = new Date(Date.now() + RESOURCE_UPLOAD_TTL_MS).toISOString(); + const partCount = Math.ceil(metadata.byteSize / RESOURCE_UPLOAD_PART_BYTES); + + try { + await context.env.storage.db.prepare( + `INSERT INTO resource_uploads ( + id, workspace_id, memo_id, resource_id, storage_config_id, bucket_name, object_key, + provider_upload_id, kind, mime_type, filename, byte_size, part_size, part_count, restore_metadata_json, + created_at, updated_at, expires_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ).bind( + uploadId, + workspaceId, + metadata.memoId, + metadata.id, + destination.configId, + destination.bucketName, + objectKey, + multipart.uploadId, + metadata.kind, + mimeType, + filename, + metadata.byteSize, + RESOURCE_UPLOAD_PART_BYTES, + partCount, + JSON.stringify(metadata), + now, + now, + expiresAt, + ).run(); + } catch (error) { + await multipart.abort().catch(() => undefined); + throw error; + } + + return { + id: uploadId, + resourceId: metadata.id, + partSize: RESOURCE_UPLOAD_PART_BYTES, + partCount, + byteSize: metadata.byteSize, + expiresAt, + }; +}; + +export const uploadResourcePart = async ( + context: AppContext, + uploadId: string, + partNumber: number, + body: ReadableStream, + byteSize: number, +) => { + const upload = assertActiveUpload(await getResourceUpload(context, uploadId)); + if (!Number.isInteger(partNumber) || partNumber < 1 || partNumber > upload.part_count) { + throw new AppError("invalid_upload_part", "Resource upload part number is invalid.", 400); + } + const expectedBytes = partNumber === upload.part_count + ? upload.byte_size - upload.part_size * (upload.part_count - 1) + : upload.part_size; + if (byteSize !== expectedBytes) { + throw new AppError( + "invalid_upload_part", + `Resource upload part ${partNumber} must contain exactly ${expectedBytes} bytes.`, + 400, + ); + } + + const destination = await resolveObjectStorage(context.env, upload.storage_config_id); + const multipart = destination.store.resumeMultipartUpload(upload.object_key, upload.provider_upload_id); + const part = await multipart.uploadPart(partNumber, body); + const now = isoNow(); + await context.env.storage.db.batch([ + context.env.storage.db.prepare( + `INSERT INTO resource_upload_parts (upload_id, part_number, etag, byte_size, created_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(upload_id, part_number) DO UPDATE SET + etag = excluded.etag, byte_size = excluded.byte_size, created_at = excluded.created_at`, + ).bind(upload.id, part.partNumber, part.etag, byteSize, now), + context.env.storage.db.prepare( + "UPDATE resource_uploads SET updated_at = ? WHERE id = ?", + ).bind(now, upload.id), + ]); + return { partNumber: part.partNumber, byteSize }; +}; + +const completeRestoredResourceUpload = async ( + context: AppContext, + upload: ResourceUploadRow, + actor: AuditActor, +): Promise => { + let metadataInput: unknown; + try { + metadataInput = JSON.parse(upload.restore_metadata_json ?? "null"); + } catch { + throw new AppError("invalid_restore_resource", "Restore resource metadata is invalid.", 400); + } + const parsed = JsonBackupResourceMetadataSchema.safeParse(metadataInput); + if (!parsed.success || parsed.data.id !== upload.resource_id || parsed.data.memoId !== upload.memo_id) { + throw new AppError("invalid_restore_resource", "Restore resource metadata is invalid.", 400); + } + const metadata = parsed.data as JsonBackupResource; + const originalMemo = metadata.originalMemoId + ? await context.env.storage.db.prepare( + "SELECT id FROM memos WHERE id = ? AND workspace_id = ? LIMIT 1", + ).bind(metadata.originalMemoId, upload.workspace_id).first<{ id: string }>() + : null; + const previous = await context.env.storage.db.prepare( + `SELECT r.object_key, r.storage_config_id FROM resources r + INNER JOIN memos m ON m.id = r.memo_id + WHERE r.id = ? AND m.workspace_id = ? LIMIT 1`, + ).bind(upload.resource_id, upload.workspace_id).first<{ + object_key: string; + storage_config_id: string; + }>(); + const now = isoNow(); + + try { + await context.env.storage.db.batch([ + context.env.storage.db.prepare( + `INSERT INTO resources ( + id, memo_id, original_memo_id, bucket_name, object_key, storage_config_id, kind, mime_type, filename, + byte_size, sha256, width, height, metadata_json, is_deleted, created_at, updated_at, deleted_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, NULL) + ON CONFLICT(id) DO UPDATE SET + memo_id = excluded.memo_id, + original_memo_id = excluded.original_memo_id, + bucket_name = excluded.bucket_name, + object_key = excluded.object_key, + storage_config_id = excluded.storage_config_id, + kind = excluded.kind, + mime_type = excluded.mime_type, + filename = excluded.filename, + byte_size = excluded.byte_size, + sha256 = excluded.sha256, + width = excluded.width, + height = excluded.height, + metadata_json = excluded.metadata_json, + is_deleted = 0, + updated_at = excluded.updated_at, + deleted_at = NULL`, + ).bind( + metadata.id, + metadata.memoId, + originalMemo?.id ?? null, + upload.bucket_name, + upload.object_key, + upload.storage_config_id, + metadata.kind, + metadata.mimeType ?? upload.mime_type, + upload.filename, + upload.byte_size, + metadata.sha256, + metadata.width, + metadata.height, + JSON.stringify({ source: "edgeever-zip-import" }), + metadata.createdAt, + now, + ), + auditStatement( + context.env.storage.db, + actor.actorType, + actor.actorId, + "resource.restore", + "resource", + metadata.id, + { memoId: metadata.memoId, byteSize: upload.byte_size, multipart: true }, + ), + context.env.storage.db.prepare("DELETE FROM resource_uploads WHERE id = ?").bind(upload.id), + ]); + } catch (error) { + const destination = await resolveObjectStorage(context.env, upload.storage_config_id); + await destination.store.delete(upload.object_key).catch(() => undefined); + throw error; + } + + if (previous?.object_key && previous.object_key !== upload.object_key) { + const previousStorage = await resolveObjectStorage(context.env, previous.storage_config_id); + await previousStorage.store.delete(previous.object_key).catch(() => undefined); + } + + const resource = await getResourceRow(context.env.storage.db, upload.workspace_id, upload.resource_id); + if (!resource) throw new AppError("not_found", "Resource not found", 404); + return mapResource(resource); +}; + +export const completeResourceUpload = async ( + context: AppContext, + uploadId: string, + actor: AuditActor, +): Promise => { + const upload = assertActiveUpload(await getResourceUpload(context, uploadId)); + const rows = await context.env.storage.db.prepare( + `SELECT part_number, etag, byte_size FROM resource_upload_parts + WHERE upload_id = ? ORDER BY part_number ASC`, + ).bind(upload.id).all(); + if (rows.results.length !== upload.part_count) { + throw new AppError("upload_incomplete", "Not all resource upload parts have been received.", 409); + } + + let totalBytes = 0; + const parts: BlobUploadedPart[] = rows.results.map((part, index) => { + const expectedPartNumber = index + 1; + const expectedBytes = expectedPartNumber === upload.part_count + ? upload.byte_size - upload.part_size * (upload.part_count - 1) + : upload.part_size; + if (part.part_number !== expectedPartNumber || part.byte_size !== expectedBytes) { + throw new AppError("upload_incomplete", "Resource upload parts do not match the expected file size.", 409); + } + totalBytes += part.byte_size; + return { partNumber: part.part_number, etag: part.etag }; + }); + if (totalBytes !== upload.byte_size) { + throw new AppError("upload_incomplete", "Resource upload size does not match the expected file size.", 409); + } + + const destination = await resolveObjectStorage(context.env, upload.storage_config_id); + const multipart = destination.store.resumeMultipartUpload(upload.object_key, upload.provider_upload_id); + await multipart.complete(parts); + + if (upload.restore_metadata_json) { + return completeRestoredResourceUpload(context, upload, actor); + } + + const now = isoNow(); + const metadata = upload.kind === "image" + ? { + source: "upload", + originalFilename: upload.filename, + originalMimeType: upload.mime_type, + originalByteSize: upload.byte_size, + compression: "disabled", + } + : { originalFilename: upload.filename }; + try { + await context.env.storage.db.batch([ + context.env.storage.db.prepare( + `INSERT INTO resources ( + id, memo_id, bucket_name, object_key, storage_config_id, kind, mime_type, filename, + byte_size, sha256, width, height, metadata_json, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ).bind( + upload.resource_id, + upload.memo_id, + upload.bucket_name, + upload.object_key, + upload.storage_config_id, + upload.kind, + upload.mime_type, + upload.filename, + upload.byte_size, + null, + null, + null, + JSON.stringify(metadata), + now, + now, + ), + auditStatement( + context.env.storage.db, + actor.actorType, + actor.actorId, + "resource.create", + "resource", + upload.resource_id, + { memoId: upload.memo_id, mimeType: upload.mime_type, byteSize: upload.byte_size, multipart: true }, + ), + context.env.storage.db.prepare("DELETE FROM resource_uploads WHERE id = ?").bind(upload.id), + ]); + } catch (error) { + await destination.store.delete(upload.object_key).catch(() => undefined); + throw error; + } + + const resource = await getResourceRow(context.env.storage.db, upload.workspace_id, upload.resource_id); + if (!resource) throw new AppError("not_found", "Resource not found", 404); + return mapResource(resource); +}; + +export const abortResourceUpload = async (context: AppContext, uploadId: string) => { + const upload = await getResourceUpload(context, uploadId); + if (!upload) return; + const destination = await resolveObjectStorage(context.env, upload.storage_config_id); + await destination.store + .resumeMultipartUpload(upload.object_key, upload.provider_upload_id) + .abort(); + await context.env.storage.db.prepare("DELETE FROM resource_uploads WHERE id = ?").bind(upload.id).run(); +}; diff --git a/apps/api/src/s3-compatible-storage-adapter.ts b/apps/api/src/s3-compatible-storage-adapter.ts index a195315c..ada0c626 100644 --- a/apps/api/src/s3-compatible-storage-adapter.ts +++ b/apps/api/src/s3-compatible-storage-adapter.ts @@ -1,11 +1,15 @@ import { + AbortMultipartUploadCommand, + CompleteMultipartUploadCommand, + CreateMultipartUploadCommand, DeleteObjectsCommand, GetObjectCommand, PutObjectCommand, S3Client, + UploadPartCommand, } from "@aws-sdk/client-s3"; import type { Readable } from "node:stream"; -import type { BlobStoreAdapter, BlobObjectAdapter, StorageAdapter } from "./storage-contract"; +import type { BlobMultipartUploadAdapter, BlobStoreAdapter, BlobObjectAdapter, StorageAdapter } from "./storage-contract"; import type { SqliteDatabaseLike } from "./self-hosted-storage-adapter"; import { createSelfHostedStorageAdapter } from "./self-hosted-storage-adapter"; @@ -71,61 +75,111 @@ const createS3BlobStore = ( ? { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey } : undefined, }), -): BlobStoreAdapter => ({ - async get(objectKey, options): Promise { - try { - const range = options?.range; - const result = await client.send(new GetObjectCommand({ +): BlobStoreAdapter => { + const resumeMultipartUpload = (objectKey: string, uploadId: string): BlobMultipartUploadAdapter => ({ + uploadId, + async uploadPart(partNumber, value) { + const result = await client.send(new UploadPartCommand({ Bucket: config.bucket, Key: objectKey, - Range: range ? `bytes=${range.offset}-${range.offset + range.length - 1}` : undefined, + UploadId: uploadId, + PartNumber: partNumber, + Body: value as never, })); - if (!result.Body) { - return null; - } - - const body = await toWebStream(result.Body); - return { - body, - size: totalSizeFromContentRange(result.ContentRange) ?? result.ContentLength ?? 0, - range: range ? { offset: range.offset, length: result.ContentLength ?? range.length } : undefined, - writeHttpMetadata: (headers) => { - if (result.ContentType) headers.set("Content-Type", result.ContentType); - if (result.CacheControl) headers.set("Cache-Control", result.CacheControl); - if (result.ContentDisposition) headers.set("Content-Disposition", result.ContentDisposition); - if (result.ETag) headers.set("ETag", result.ETag); - if (result.LastModified) headers.set("Last-Modified", result.LastModified.toUTCString()); - }, - }; - } catch (error) { - if (isMissingObjectError(error)) return null; - throw error; - } - }, - - async put(objectKey, value, options) { - const metadata = options as { httpMetadata?: Record } | undefined; - const httpMetadata = metadata?.httpMetadata ?? {}; - await client.send(new PutObjectCommand({ - Bucket: config.bucket, - Key: objectKey, - Body: value as never, - ContentType: httpMetadata.contentType, - CacheControl: httpMetadata.cacheControl, - ContentDisposition: httpMetadata.contentDisposition, - })); - }, - - async delete(objectKeys) { - const keys = Array.isArray(objectKeys) ? objectKeys : [objectKeys]; - for (let index = 0; index < keys.length; index += 1000) { - await client.send(new DeleteObjectsCommand({ + if (!result.ETag) throw new Error("Multipart upload part did not return an ETag."); + return { partNumber, etag: result.ETag }; + }, + async complete(parts) { + await client.send(new CompleteMultipartUploadCommand({ Bucket: config.bucket, - Delete: { Objects: keys.slice(index, index + 1000).map((Key) => ({ Key })) }, + Key: objectKey, + UploadId: uploadId, + MultipartUpload: { + Parts: parts.map((part) => ({ PartNumber: part.partNumber, ETag: part.etag })), + }, })); - } - }, -}); + }, + async abort() { + await client.send(new AbortMultipartUploadCommand({ + Bucket: config.bucket, + Key: objectKey, + UploadId: uploadId, + })); + }, + }); + + return { + async get(objectKey, options): Promise { + try { + const range = options?.range; + const result = await client.send(new GetObjectCommand({ + Bucket: config.bucket, + Key: objectKey, + Range: range ? `bytes=${range.offset}-${range.offset + range.length - 1}` : undefined, + })); + if (!result.Body) { + return null; + } + + const body = await toWebStream(result.Body); + return { + body, + size: totalSizeFromContentRange(result.ContentRange) ?? result.ContentLength ?? 0, + range: range ? { offset: range.offset, length: result.ContentLength ?? range.length } : undefined, + writeHttpMetadata: (headers) => { + if (result.ContentType) headers.set("Content-Type", result.ContentType); + if (result.CacheControl) headers.set("Cache-Control", result.CacheControl); + if (result.ContentDisposition) headers.set("Content-Disposition", result.ContentDisposition); + if (result.ETag) headers.set("ETag", result.ETag); + if (result.LastModified) headers.set("Last-Modified", result.LastModified.toUTCString()); + }, + }; + } catch (error) { + if (isMissingObjectError(error)) return null; + throw error; + } + }, + + async put(objectKey, value, options) { + const metadata = options as { httpMetadata?: Record } | undefined; + const httpMetadata = metadata?.httpMetadata ?? {}; + await client.send(new PutObjectCommand({ + Bucket: config.bucket, + Key: objectKey, + Body: value as never, + ContentType: httpMetadata.contentType, + CacheControl: httpMetadata.cacheControl, + ContentDisposition: httpMetadata.contentDisposition, + })); + }, + + async createMultipartUpload(objectKey, options) { + const metadata = options as { httpMetadata?: Record } | undefined; + const httpMetadata = metadata?.httpMetadata ?? {}; + const result = await client.send(new CreateMultipartUploadCommand({ + Bucket: config.bucket, + Key: objectKey, + ContentType: httpMetadata.contentType, + CacheControl: httpMetadata.cacheControl, + ContentDisposition: httpMetadata.contentDisposition, + })); + if (!result.UploadId) throw new Error("Multipart upload initialization did not return an upload ID."); + return resumeMultipartUpload(objectKey, result.UploadId); + }, + + resumeMultipartUpload, + + async delete(objectKeys) { + const keys = Array.isArray(objectKeys) ? objectKeys : [objectKeys]; + for (let index = 0; index < keys.length; index += 1000) { + await client.send(new DeleteObjectsCommand({ + Bucket: config.bucket, + Delete: { Objects: keys.slice(index, index + 1000).map((Key) => ({ Key })) }, + })); + } + }, + }; +}; /** * S3-compatible storage for self-hosted deployments. The SQLite adapter is diff --git a/apps/api/src/self-hosted-storage-adapter.ts b/apps/api/src/self-hosted-storage-adapter.ts index 0e64564b..1adc1c3f 100644 --- a/apps/api/src/self-hosted-storage-adapter.ts +++ b/apps/api/src/self-hosted-storage-adapter.ts @@ -1,7 +1,10 @@ -import { mkdir, open, readFile, unlink, writeFile } from "node:fs/promises"; +import { createReadStream } from "node:fs"; +import { mkdir, open, rename, rm, stat, unlink, writeFile } from "node:fs/promises"; import { dirname, resolve, sep } from "node:path"; +import { Readable } from "node:stream"; import type { BlobObjectAdapter, + BlobMultipartUploadAdapter, BlobStoreAdapter, DatabaseAdapter, DatabaseQueryResult, @@ -95,37 +98,89 @@ const safeObjectPath = (rootDirectory: string, objectKey: string) => { return target; }; +const localMultipartUpload = ( + rootDirectory: string, + objectKey: string, + uploadId: string, +): BlobMultipartUploadAdapter => { + const uploadDirectory = safeObjectPath(rootDirectory, `.uploads/${uploadId}`); + const partPath = (partNumber: number) => resolve(uploadDirectory, `${partNumber}.part`); + + return { + uploadId, + async uploadPart(partNumber, value) { + await mkdir(uploadDirectory, { recursive: true }); + const bytes = value instanceof ReadableStream + ? new Uint8Array(await new Response(value).arrayBuffer()) + : value instanceof Blob + ? new Uint8Array(await value.arrayBuffer()) + : value instanceof ArrayBuffer + ? new Uint8Array(value) + : new Uint8Array(value.buffer, value.byteOffset, value.byteLength); + await writeFile(partPath(partNumber), bytes); + return { partNumber, etag: `local-${partNumber}-${bytes.byteLength}` }; + }, + async complete(parts) { + const target = safeObjectPath(rootDirectory, objectKey); + const temporaryTarget = `${target}.upload-${uploadId}`; + await mkdir(dirname(target), { recursive: true }); + const output = await open(temporaryTarget, "w"); + try { + for (const part of [...parts].sort((left, right) => left.partNumber - right.partNumber)) { + const source = partPath(part.partNumber); + const info = await stat(source); + if (part.etag !== `local-${part.partNumber}-${info.size}`) { + throw new Error(`Multipart upload part ${part.partNumber} does not match its recorded ETag.`); + } + const input = await open(source, "r"); + try { + for await (const chunk of input.createReadStream()) { + await output.write(chunk); + } + } finally { + await input.close(); + } + } + } catch (error) { + await output.close(); + await rm(temporaryTarget, { force: true }); + throw error; + } + await output.close(); + await rename(temporaryTarget, target); + await rm(uploadDirectory, { recursive: true, force: true }); + }, + async abort() { + await rm(uploadDirectory, { recursive: true, force: true }); + }, + }; +}; + const createLocalBlobStore = (rootDirectory: string): BlobStoreAdapter => ({ async get(objectKey, options): Promise { const target = safeObjectPath(rootDirectory, objectKey); try { - if (options?.range) { - const handle = await open(target, "r"); - try { - const { size } = await handle.stat(); - const bytes = new Uint8Array(options.range.length); - const { bytesRead } = await handle.read(bytes, 0, bytes.byteLength, options.range.offset); - const bodyBytes = bytes.subarray(0, bytesRead); - return { - body: new Response(bodyBytes).body as ReadableStream, - size, - range: { offset: options.range.offset, length: bytesRead }, - writeHttpMetadata: (headers) => { - headers.set("Content-Length", String(bytesRead)); - }, - }; - } finally { - await handle.close(); - } - } - - const bytes = await readFile(target); + const { size } = await stat(target); + const requestedRange = options?.range; + const rangeLength = requestedRange + ? Math.max(0, Math.min(requestedRange.length, size - requestedRange.offset)) + : size; + const nodeStream = createReadStream(target, requestedRange + ? { + start: requestedRange.offset, + end: requestedRange.offset + rangeLength - 1, + } + : undefined); + const body = Readable.toWeb(nodeStream) as ReadableStream; return { - body: new Response(bytes).body as ReadableStream, - size: bytes.byteLength, + body, + size, + ...(requestedRange + ? { range: { offset: requestedRange.offset, length: rangeLength } } + : {}), writeHttpMetadata: (headers) => { - headers.set("Content-Length", String(bytes.byteLength)); + headers.set("Content-Length", String(rangeLength)); }, }; } catch (error) { @@ -158,6 +213,14 @@ const createLocalBlobStore = (rootDirectory: string): BlobStoreAdapter => ({ throw new Error("Unsupported local resource payload"); }, + async createMultipartUpload(objectKey) { + return localMultipartUpload(rootDirectory, objectKey, crypto.randomUUID()); + }, + + resumeMultipartUpload(objectKey, uploadId) { + return localMultipartUpload(rootDirectory, objectKey, uploadId); + }, + async delete(objectKeys) { const keys = Array.isArray(objectKeys) ? objectKeys : [objectKeys]; await Promise.all(keys.map(async (objectKey) => { diff --git a/apps/api/src/storage-contract.ts b/apps/api/src/storage-contract.ts index 4558e8bc..baa67fd9 100644 --- a/apps/api/src/storage-contract.ts +++ b/apps/api/src/storage-contract.ts @@ -47,6 +47,21 @@ export type BlobObjectAdapter = { writeHttpMetadata: (headers: Headers) => void; }; +export type BlobUploadedPart = { + partNumber: number; + etag: string; +}; + +export type BlobMultipartUploadAdapter = { + readonly uploadId: string; + uploadPart: ( + partNumber: number, + value: ReadableStream | ArrayBuffer | ArrayBufferView | Blob, + ) => Promise; + complete: (parts: BlobUploadedPart[]) => Promise; + abort: () => Promise; +}; + /** * Deliberately uses unknown for provider-specific upload metadata. The API * passes through metadata such as content type and cache control, while a @@ -55,6 +70,8 @@ export type BlobObjectAdapter = { export type BlobStoreAdapter = { get: (key: string, options?: BlobGetOptions) => Promise; put: (key: string, value: unknown, options?: unknown) => Promise; + createMultipartUpload: (key: string, options?: unknown) => Promise; + resumeMultipartUpload: (key: string, uploadId: string) => BlobMultipartUploadAdapter; delete: (keys: string | string[]) => Promise; }; diff --git a/apps/api/src/worker-s3-blob-store.test.mjs b/apps/api/src/worker-s3-blob-store.test.mjs index 72580e30..e8e751d4 100644 --- a/apps/api/src/worker-s3-blob-store.test.mjs +++ b/apps/api/src/worker-s3-blob-store.test.mjs @@ -70,4 +70,37 @@ describe("Worker S3-compatible blob store", () => { expect(object?.range).toEqual({ offset: 2, length: 4 }); expect(await new Response(object?.body).text()).toBe("2345"); }); + + test("signs the complete multipart lifecycle", async () => { + const requests = []; + globalThis.fetch = async (request) => { + requests.push(request); + const url = new URL(request.url); + if (request.method === "POST" && url.searchParams.has("uploads")) { + return new Response("provider-1"); + } + if (request.method === "PUT") return new Response(null, { headers: { etag: '"part-1"' } }); + if (request.method === "POST" && url.searchParams.get("uploadId") === "provider-1") { + expect(await request.text()).toContain("1"); + return new Response(""); + } + return new Response(null, { status: 405 }); + }; + const store = createWorkerS3BlobStore({ + endpoint: "https://objects.example.com", + region: "us-east-1", + bucket: "edgeever", + accessKeyId: "access-key", + secretAccessKey: "secret-key", + forcePathStyle: true, + objectPrefix: "notes", + }); + + const upload = await store.createMultipartUpload("archive.bin"); + const part = await upload.uploadPart(1, new Uint8Array([1, 2, 3])); + await upload.complete([part]); + + expect(upload.uploadId).toBe("provider-1"); + expect(requests.map((request) => request.method)).toEqual(["POST", "PUT", "POST"]); + }); }); diff --git a/apps/api/src/worker-s3-blob-store.ts b/apps/api/src/worker-s3-blob-store.ts index c2907588..bf32b15a 100644 --- a/apps/api/src/worker-s3-blob-store.ts +++ b/apps/api/src/worker-s3-blob-store.ts @@ -1,5 +1,5 @@ import { AwsClient } from "aws4fetch"; -import type { BlobObjectAdapter, BlobStoreAdapter } from "./storage-contract"; +import type { BlobMultipartUploadAdapter, BlobObjectAdapter, BlobStoreAdapter } from "./storage-contract"; export type WorkerS3Config = { endpoint: string; @@ -41,6 +41,18 @@ const totalSizeFromResponse = (response: Response) => { return match ? Number(match[1]) : Number(response.headers.get("content-length") ?? 0); }; +const xmlText = (value: string, tag: string) => { + const match = new RegExp(`<${tag}>([^<]+)`).exec(value); + return match?.[1]?.trim() ?? null; +}; + +const escapeXml = (value: string) => value + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); + export const createWorkerS3BlobStore = (config: WorkerS3Config): BlobStoreAdapter => { const client = new AwsClient({ accessKeyId: config.accessKeyId, @@ -49,6 +61,43 @@ export const createWorkerS3BlobStore = (config: WorkerS3Config): BlobStoreAdapte service: "s3", }); + const resumeMultipartUpload = (objectKey: string, uploadId: string): BlobMultipartUploadAdapter => ({ + uploadId, + async uploadPart(partNumber, value) { + const url = createObjectUrl(config, objectKey); + url.searchParams.set("partNumber", String(partNumber)); + url.searchParams.set("uploadId", uploadId); + const response = await client.fetch(url.toString(), { method: "PUT", body: value as BodyInit }); + if (!response.ok) throw await responseError("Multipart part upload", response); + const etag = response.headers.get("etag"); + if (!etag) throw new Error("Multipart upload part did not return an ETag."); + return { partNumber, etag }; + }, + async complete(parts) { + const url = createObjectUrl(config, objectKey); + url.searchParams.set("uploadId", uploadId); + const body = `${parts + .map((part) => `${part.partNumber}${escapeXml(part.etag)}`) + .join("")}`; + const response = await client.fetch(url.toString(), { + method: "POST", + headers: { "content-type": "application/xml" }, + body, + }); + if (!response.ok) throw await responseError("Multipart upload completion", response); + const result = await response.text(); + if (/)/.test(result)) { + throw new Error(`Multipart upload completion failed: ${result.replace(/\s+/g, " ").trim().slice(0, 300)}`); + } + }, + async abort() { + const url = createObjectUrl(config, objectKey); + url.searchParams.set("uploadId", uploadId); + const response = await client.fetch(url.toString(), { method: "DELETE" }); + if (!response.ok && response.status !== 404) throw await responseError("Multipart upload abort", response); + }, + }); + return { async get(objectKey, options): Promise { const headers = new Headers(); @@ -89,6 +138,24 @@ export const createWorkerS3BlobStore = (config: WorkerS3Config): BlobStoreAdapte if (!response.ok) throw await responseError("Object upload", response); }, + async createMultipartUpload(objectKey, options) { + const metadata = options as { httpMetadata?: Record } | undefined; + const httpMetadata = metadata?.httpMetadata ?? {}; + const headers = new Headers(); + if (httpMetadata.contentType) headers.set("content-type", httpMetadata.contentType); + if (httpMetadata.cacheControl) headers.set("cache-control", httpMetadata.cacheControl); + if (httpMetadata.contentDisposition) headers.set("content-disposition", httpMetadata.contentDisposition); + const url = createObjectUrl(config, objectKey); + url.searchParams.set("uploads", ""); + const response = await client.fetch(url.toString(), { method: "POST", headers }); + if (!response.ok) throw await responseError("Multipart upload initialization", response); + const uploadId = xmlText(await response.text(), "UploadId"); + if (!uploadId) throw new Error("Multipart upload initialization did not return an upload ID."); + return resumeMultipartUpload(objectKey, uploadId); + }, + + resumeMultipartUpload, + async delete(objectKeys) { for (const objectKey of Array.isArray(objectKeys) ? objectKeys : [objectKeys]) { const response = await client.fetch(createObjectUrl(config, objectKey).toString(), { method: "DELETE" }); diff --git a/apps/desktop/src/main/index.mjs b/apps/desktop/src/main/index.mjs index 8bce2c41..e7dae244 100644 --- a/apps/desktop/src/main/index.mjs +++ b/apps/desktop/src/main/index.mjs @@ -1,18 +1,24 @@ import { app, BrowserWindow, Menu, Tray, nativeImage, ipcMain, session, net, protocol, shell, dialog, safeStorage, clipboard, powerMonitor } from "electron"; -import { existsSync } from "node:fs"; -import { appendFile, mkdir, readdir, readFile, rename, unlink, writeFile } from "node:fs/promises"; +import { createReadStream, existsSync } from "node:fs"; +import { appendFile, mkdir, open, readdir, readFile, rename, rm, stat, unlink, writeFile } from "node:fs/promises"; import { basename, join } from "node:path"; +import { Readable } from "node:stream"; import { execFile, spawn } from "node:child_process"; import { fileURLToPath } from "node:url"; import { release as operatingSystemRelease } from "node:os"; import { SidecarRpcClient } from "./rpc.mjs"; import { resourceRequestHeaders } from "./resource-request.mjs"; -import { cachedResourceResponse, isSafeResourceId, resourceIdFromRequest } from "./resource-url.mjs"; +import { isSafeResourceId, parseByteRangeHeader, resourceIdFromRequest } from "./resource-url.mjs"; import { isSupportedAssociatedFile } from "./file-association.mjs"; import { accountDataDirectory, accountScopeKey } from "./account-scope.mjs"; import { rotateDiagnosticLog } from "./diagnostic-log.mjs"; import { restrictDirectory, restrictFile } from "./file-permissions.mjs"; -import { normalizeStagedResourceInput, remapStagedResourceMetadata } from "./staged-resource.mjs"; +import { + STAGED_RESOURCE_PART_BYTES, + normalizeStagedResourceMetadataInput, + normalizeStagedResourcePart, + remapStagedResourceMetadata, +} from "./staged-resource.mjs"; import { isMountedDiskImageVolume, isMountedInstallerPath, @@ -550,11 +556,37 @@ const handleResourceProtocolRequest = async (request) => { const bytesPath = join(directory, `${resourceId}.bin`); const metadataPath = join(directory, `${resourceId}.json`); + const fileResponse = async (path, contentType, rangeHeader) => { + const { size } = await stat(path); + const range = parseByteRangeHeader(rangeHeader, size); + const headers = new Headers({ + "Accept-Ranges": "bytes", + "Cache-Control": "no-store", + "Content-Type": contentType || "application/octet-stream", + }); + if (range.kind === "invalid") { + headers.set("Content-Range", `bytes */${size}`); + return new Response(null, { status: 416, headers }); + } + const selected = range.kind === "range" ? range : { offset: 0, length: size }; + headers.set("Content-Length", String(selected.length)); + if (range.kind === "range") { + headers.set("Content-Range", `bytes ${selected.offset}-${selected.offset + selected.length - 1}/${size}`); + } + const stream = createReadStream(path, { + start: selected.offset, + end: selected.offset + selected.length - 1, + }); + return new Response(Readable.toWeb(stream), { + status: range.kind === "range" ? 206 : 200, + headers, + }); + }; + try { - const bytes = await readFile(bytesPath); let metadata = {}; try { metadata = JSON.parse(await readFile(metadataPath, "utf8")); } catch {} - return cachedResourceResponse(bytes, metadata.contentType, request.headers.get("range")); + return await fileResponse(bytesPath, metadata.contentType, request.headers.get("range")); } catch { // Fall through to the instance while online, then persist the response. } @@ -568,7 +600,6 @@ const handleResourceProtocolRequest = async (request) => { if (rangeHeader) headers.set("range", rangeHeader); const response = await net.fetch(sourceUrl, { headers }); if (!response.ok) return new Response("Resource request failed", { status: response.status }); - const body = Buffer.from(await response.arrayBuffer()); if (response.status === 206) { const responseHeaders = new Headers({ "Accept-Ranges": response.headers.get("accept-ranges") || "bytes", @@ -579,15 +610,51 @@ const handleResourceProtocolRequest = async (request) => { const value = response.headers.get(name); if (value) responseHeaders.set(name, value); } - return new Response(body, { status: 206, headers: responseHeaders }); + return new Response(response.body, { status: 206, headers: responseHeaders }); } + if (!response.body) return new Response("Resource response body is empty", { status: 502 }); await mkdir(directory, { recursive: true }); await restrictDirectory(directory); - await writeFile(bytesPath, body, { mode: 0o600 }); - await writeFile(metadataPath, JSON.stringify({ contentType: response.headers.get("content-type") || "application/octet-stream" }), { mode: 0o600 }); - await restrictFile(bytesPath); - await restrictFile(metadataPath); - return cachedResourceResponse(body, response.headers.get("content-type"), null); + const temporaryPath = `${bytesPath}.download-${crypto.randomUUID()}`; + const output = await open(temporaryPath, "w", 0o600); + const reader = response.body.getReader(); + let finished = false; + const cleanup = async () => { + if (!finished) { + finished = true; + await output.close().catch(() => {}); + await rm(temporaryPath, { force: true }).catch(() => {}); + } + }; + const streamedBody = new ReadableStream({ + async pull(controller) { + try { + const next = await reader.read(); + if (next.done) { + await output.close(); + await rename(temporaryPath, bytesPath); + await writeFile(metadataPath, JSON.stringify({ contentType: response.headers.get("content-type") || "application/octet-stream" }), { mode: 0o600 }); + await restrictFile(bytesPath); + await restrictFile(metadataPath); + finished = true; + controller.close(); + return; + } + await output.write(next.value); + controller.enqueue(next.value); + } catch (error) { + await cleanup(); + controller.error(error); + } + }, + async cancel(reason) { + await reader.cancel(reason).catch(() => {}); + await cleanup(); + }, + }); + const responseHeaders = new Headers(response.headers); + responseHeaders.set("Cache-Control", "no-store"); + return new Response(streamedBody, { status: 200, headers: responseHeaders }); } catch (error) { void writeDiagnostic("resource.cache-failed", { resourceId, message: error.message }); return new Response("Resource unavailable", { status: 504 }); @@ -604,10 +671,13 @@ const registerResourceProtocol = () => { const directory = stagedResourceDirectory(); try { const metadata = JSON.parse(await readFile(join(directory, `${stagedId}.json`), "utf8")); - const bytes = await readFile(join(directory, `${stagedId}.bin`)); - return new Response(bytes, { + const path = join(directory, `${stagedId}.bin`); + const { size } = await stat(path); + const stream = createReadStream(path); + return new Response(Readable.toWeb(stream), { headers: { "Content-Type": metadata.type || "application/octet-stream", + "Content-Length": String(size), "Cache-Control": "no-store", }, }); @@ -1251,21 +1321,60 @@ const startApplication = async () => { }); ipcMain.handle("desktop:download-update", () => downloadTrustedDesktopUpdate("manual-download")); ipcMain.handle("desktop:install-update", () => installDownloadedUpdate()); - ipcMain.handle("desktop:stage-resource", async (_event, input) => { - const { memoId, name, type, bytes } = normalizeStagedResourceInput(input); + ipcMain.handle("desktop:stage-resource-begin", async (_event, input) => { + const { memoId, name, type, size } = normalizeStagedResourceMetadataInput(input); const id = `stage_${Date.now()}_${Math.random().toString(36).slice(2)}`; const directory = stagedResourceDirectory(); await mkdir(directory, { recursive: true }); await restrictDirectory(directory); - const metadata = { id, memoId, name, type, size: bytes.byteLength }; + const metadata = { id, memoId, name, type, size }; + const pendingMetadataPath = join(directory, `${id}.pending.json`); + const pendingBytesPath = join(directory, `${id}.pending.bin`); + await writeFile(pendingMetadataPath, JSON.stringify(metadata), { mode: 0o600 }); + await writeFile(pendingBytesPath, new Uint8Array(), { mode: 0o600 }); + await restrictFile(pendingMetadataPath); + await restrictFile(pendingBytesPath); + return { id, partSize: STAGED_RESOURCE_PART_BYTES }; + }); + ipcMain.handle("desktop:stage-resource-append", async (_event, id, value) => { + if (!isSafeResourceId(id)) throw new Error("Invalid staged resource id"); + const bytes = normalizeStagedResourcePart(value); + const directory = stagedResourceDirectory(); + const pendingMetadataPath = join(directory, `${id}.pending.json`); + const pendingBytesPath = join(directory, `${id}.pending.bin`); + const metadata = JSON.parse(await readFile(pendingMetadataPath, "utf8")); + const current = await stat(pendingBytesPath); + if (current.size + bytes.byteLength > metadata.size) { + throw new Error("Staged resource received more bytes than declared"); + } + await appendFile(pendingBytesPath, bytes); + return { receivedBytes: current.size + bytes.byteLength }; + }); + ipcMain.handle("desktop:stage-resource-complete", async (_event, id) => { + if (!isSafeResourceId(id)) throw new Error("Invalid staged resource id"); + const directory = stagedResourceDirectory(); + const pendingMetadataPath = join(directory, `${id}.pending.json`); + const pendingBytesPath = join(directory, `${id}.pending.bin`); + const metadata = JSON.parse(await readFile(pendingMetadataPath, "utf8")); + const current = await stat(pendingBytesPath); + if (current.size !== metadata.size) throw new Error("Staged resource is incomplete"); const metadataPath = join(directory, `${id}.json`); const bytesPath = join(directory, `${id}.bin`); + await rename(pendingBytesPath, bytesPath); await writeFile(metadataPath, JSON.stringify(metadata), { mode: 0o600 }); - await writeFile(bytesPath, Buffer.from(bytes), { mode: 0o600 }); await restrictFile(metadataPath); await restrictFile(bytesPath); + await unlink(pendingMetadataPath).catch(() => {}); return { id }; }); + ipcMain.handle("desktop:stage-resource-abort", async (_event, id) => { + if (!isSafeResourceId(id)) throw new Error("Invalid staged resource id"); + const directory = stagedResourceDirectory(); + await Promise.all([ + unlink(join(directory, `${id}.pending.json`)).catch(() => {}), + unlink(join(directory, `${id}.pending.bin`)).catch(() => {}), + ]); + }); ipcMain.handle("desktop:list-staged-resources", async () => { const directory = stagedResourceDirectory(); try { await mkdir(directory, { recursive: true }); await restrictDirectory(directory); } catch {} @@ -1302,6 +1411,25 @@ const startApplication = async () => { const bytes = await readFile(join(directory, `${id}.bin`)); return { ...metadata, bytes: new Uint8Array(bytes) }; }); + ipcMain.handle("desktop:read-staged-resource-part", async (_event, id, start, length) => { + if (!isSafeResourceId(id)) throw new Error("Invalid staged resource id"); + if (!Number.isSafeInteger(start) || start < 0) throw new Error("Invalid staged resource offset"); + if (!Number.isSafeInteger(length) || length <= 0 || length > STAGED_RESOURCE_PART_BYTES) { + throw new Error("Invalid staged resource part length"); + } + const path = join(stagedResourceDirectory(), `${id}.bin`); + const details = await stat(path); + if (start >= details.size || start + length > details.size) throw new Error("Invalid staged resource range"); + const handle = await open(path, "r"); + try { + const bytes = Buffer.allocUnsafe(length); + const { bytesRead } = await handle.read(bytes, 0, length, start); + if (bytesRead !== length) throw new Error("Staged resource part is incomplete"); + return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytesRead); + } finally { + await handle.close(); + } + }); ipcMain.handle("desktop:read-resource", async (_event, id) => { if (!isSafeResourceId(id)) throw new Error("Invalid resource id"); const response = await handleResourceProtocolRequest(new Request( @@ -1319,6 +1447,8 @@ const startApplication = async () => { await Promise.all([ unlink(join(directory, `${id}.json`)).catch(() => {}), unlink(join(directory, `${id}.bin`)).catch(() => {}), + unlink(join(directory, `${id}.pending.json`)).catch(() => {}), + unlink(join(directory, `${id}.pending.bin`)).catch(() => {}), ]); }); diff --git a/apps/desktop/src/main/staged-resource.mjs b/apps/desktop/src/main/staged-resource.mjs index d119b2f8..49a7b3da 100644 --- a/apps/desktop/src/main/staged-resource.mjs +++ b/apps/desktop/src/main/staged-resource.mjs @@ -1,25 +1,36 @@ -export const MAX_STAGED_RESOURCE_BYTES = 256 * 1024 * 1024; +export const MAX_STAGED_RESOURCE_BYTES = 1024 * 1024 * 1024; +export const STAGED_RESOURCE_PART_BYTES = 8 * 1024 * 1024; const MAX_STAGED_RESOURCE_NAME_LENGTH = 512; const MAX_STAGED_RESOURCE_TYPE_LENGTH = 256; const MAX_STAGED_RESOURCE_MEMO_ID_LENGTH = 160; const normalizeMemoId = (value) => typeof value === "string" ? value.trim() : ""; -export const normalizeStagedResourceInput = (input) => { +export const normalizeStagedResourceMetadataInput = (input) => { if (!input || typeof input !== "object") throw new Error("Invalid staged resource input"); const memoId = normalizeMemoId(input.memoId); const name = typeof input.name === "string" ? input.name.trim() : ""; const type = typeof input.type === "string" && input.type.trim() ? input.type.trim() : "application/octet-stream"; - const bytes = input.bytes instanceof Uint8Array - ? input.bytes - : input.bytes instanceof ArrayBuffer - ? new Uint8Array(input.bytes) - : null; + const size = Number(input.size); if (!memoId || memoId.length > MAX_STAGED_RESOURCE_MEMO_ID_LENGTH) throw new Error("Invalid staged resource memo id"); if (!name || name.length > MAX_STAGED_RESOURCE_NAME_LENGTH || /[\u0000-\u001f\u007f]/.test(name)) throw new Error("Invalid staged resource name"); if (type.length > MAX_STAGED_RESOURCE_TYPE_LENGTH || /[\u0000-\u001f\u007f]/.test(type)) throw new Error("Invalid staged resource type"); - if (!bytes || bytes.byteLength > MAX_STAGED_RESOURCE_BYTES) throw new Error("Staged resource exceeds the 256 MiB limit"); - return { memoId, name, type, bytes }; + if (!Number.isSafeInteger(size) || size <= 0 || size > MAX_STAGED_RESOURCE_BYTES) { + throw new Error("Staged resource must be between 1 byte and 1 GiB"); + } + return { memoId, name, type, size }; +}; + +export const normalizeStagedResourcePart = (value) => { + const bytes = value instanceof Uint8Array + ? value + : value instanceof ArrayBuffer + ? new Uint8Array(value) + : null; + if (!bytes || bytes.byteLength <= 0 || bytes.byteLength > STAGED_RESOURCE_PART_BYTES) { + throw new Error("Invalid staged resource part"); + } + return bytes; }; export const remapStagedResourceMetadata = (metadata, mappings) => { diff --git a/apps/desktop/src/main/staged-resource.test.mjs b/apps/desktop/src/main/staged-resource.test.mjs index 191c1ab5..5dbce2ea 100644 --- a/apps/desktop/src/main/staged-resource.test.mjs +++ b/apps/desktop/src/main/staged-resource.test.mjs @@ -1,16 +1,27 @@ import { describe, expect, test } from "bun:test"; -import { MAX_STAGED_RESOURCE_BYTES, normalizeStagedResourceInput, remapStagedResourceMetadata } from "./staged-resource.mjs"; +import { + MAX_STAGED_RESOURCE_BYTES, + STAGED_RESOURCE_PART_BYTES, + normalizeStagedResourceMetadataInput, + normalizeStagedResourcePart, + remapStagedResourceMetadata, +} from "./staged-resource.mjs"; describe("desktop staged resource input", () => { - test("normalizes valid structured-clone payloads", () => { - const input = normalizeStagedResourceInput({ memoId: "memo-1", name: " photo.png ", type: " image/png ", bytes: new ArrayBuffer(3) }); - expect(input).toMatchObject({ memoId: "memo-1", name: "photo.png", type: "image/png" }); - expect(input.bytes).toBeInstanceOf(Uint8Array); + test("normalizes valid metadata without accepting the complete file", () => { + const input = normalizeStagedResourceMetadataInput({ memoId: "memo-1", name: " photo.png ", type: " image/png ", size: 3 }); + expect(input).toEqual({ memoId: "memo-1", name: "photo.png", type: "image/png", size: 3 }); }); - test("rejects control characters and invalid or oversized payloads", () => { - expect(() => normalizeStagedResourceInput({ memoId: "memo-1", name: "bad\u0000name", bytes: new Uint8Array() })).toThrow("Invalid staged resource name"); - expect(() => normalizeStagedResourceInput({ memoId: "memo-1", name: "large.bin", bytes: { byteLength: MAX_STAGED_RESOURCE_BYTES + 1 } })).toThrow("Staged resource exceeds"); + test("rejects control characters and invalid or oversized metadata", () => { + expect(() => normalizeStagedResourceMetadataInput({ memoId: "memo-1", name: "bad\u0000name", size: 1 })).toThrow("Invalid staged resource name"); + expect(() => normalizeStagedResourceMetadataInput({ memoId: "memo-1", name: "large.bin", size: MAX_STAGED_RESOURCE_BYTES + 1 })).toThrow("1 GiB"); + }); + + test("bounds every staged write to one multipart-sized chunk", () => { + expect(normalizeStagedResourcePart(new ArrayBuffer(3))).toBeInstanceOf(Uint8Array); + expect(() => normalizeStagedResourcePart(new Uint8Array())).toThrow("Invalid staged resource part"); + expect(() => normalizeStagedResourcePart(new Uint8Array(STAGED_RESOURCE_PART_BYTES + 1))).toThrow("Invalid staged resource part"); }); test("remaps a staged image from its temporary memo id to the server id", () => { diff --git a/apps/desktop/src/preload/index.cjs b/apps/desktop/src/preload/index.cjs index 7716d3e7..333e6812 100644 --- a/apps/desktop/src/preload/index.cjs +++ b/apps/desktop/src/preload/index.cjs @@ -28,10 +28,14 @@ contextBridge.exposeInMainWorld("edgeeverDesktop", Object.freeze({ return () => ipcRenderer.removeListener("desktop:update-status-changed", listener); }, sidecarRequest: (method, params = {}) => ipcRenderer.invoke("desktop:sidecar-request", method, params), - stageResource: (input) => ipcRenderer.invoke("desktop:stage-resource", input), + beginStagedResource: (input) => ipcRenderer.invoke("desktop:stage-resource-begin", input), + appendStagedResource: (id, bytes) => ipcRenderer.invoke("desktop:stage-resource-append", id, bytes), + completeStagedResource: (id) => ipcRenderer.invoke("desktop:stage-resource-complete", id), + abortStagedResource: (id) => ipcRenderer.invoke("desktop:stage-resource-abort", id), listStagedResources: () => ipcRenderer.invoke("desktop:list-staged-resources"), remapStagedResourceMemoIds: (mappings) => ipcRenderer.invoke("desktop:remap-staged-resource-memo-ids", mappings), readStagedResource: (id) => ipcRenderer.invoke("desktop:read-staged-resource", id), + readStagedResourcePart: (id, start, length) => ipcRenderer.invoke("desktop:read-staged-resource-part", id, start, length), readResource: (id) => ipcRenderer.invoke("desktop:read-resource", id), removeStagedResource: (id) => ipcRenderer.invoke("desktop:remove-staged-resource", id), onCommand: (callback) => { diff --git a/apps/ios/EdgeEver/Data/Network/APIClient.swift b/apps/ios/EdgeEver/Data/Network/APIClient.swift index a77f51b1..121f9405 100644 --- a/apps/ios/EdgeEver/Data/Network/APIClient.swift +++ b/apps/ios/EdgeEver/Data/Network/APIClient.swift @@ -291,22 +291,107 @@ actor APIClient { // MARK: - Resources / tokens / tags func uploadMemoResource(memoId: String, filename: String, mimeType: String, data: Data) async throws -> Resource { - let boundary = "EdgeEverBoundary\(UUID().uuidString.replacingOccurrences(of: "-", with: ""))" - var body = Data() - body.append("--\(boundary)\r\n".data(using: .utf8)!) - body.append("Content-Disposition: form-data; name=\"file\"; filename=\"\(filename)\"\r\n".data(using: .utf8)!) - body.append("Content-Type: \(mimeType)\r\n\r\n".data(using: .utf8)!) - body.append(data) - body.append("\r\n--\(boundary)--\r\n".data(using: .utf8)!) + try await uploadMemoResourceParts( + memoId: memoId, + filename: filename, + mimeType: mimeType, + byteSize: data.count, + readPart: { range in data.subdata(in: range) } + ) + } - var request = URLRequest(url: makeURL(path: "/api/v1/memos/\(memoId)/resources")) - request.httpMethod = "POST" - request.setValue("multipart/form-data; boundary=\(boundary)", forHTTPHeaderField: "Content-Type") - if let token { - request.setValue("Bearer \(token)", forHTTPHeaderField: "Authorization") + /// Upload a file without retaining the complete attachment in process memory. + /// The file handle reads only the server-advertised multipart chunk for each request. + func uploadMemoResource(memoId: String, filename: String, mimeType: String, fileURL: URL) async throws -> Resource { + let values = try fileURL.resourceValues(forKeys: [.fileSizeKey, .isRegularFileKey]) + guard values.isRegularFile == true, let byteSize = values.fileSize, byteSize > 0 else { + throw APIError(status: 0, code: nil, message: "Attachment file is unavailable") + } + let handle = try FileHandle(forReadingFrom: fileURL) + defer { try? handle.close() } + return try await uploadMemoResourceParts( + memoId: memoId, + filename: filename, + mimeType: mimeType, + byteSize: byteSize, + readPart: { range in + try handle.seek(toOffset: UInt64(range.lowerBound)) + let data = try handle.read(upToCount: range.count) ?? Data() + guard data.count == range.count else { + throw APIError(status: 0, code: nil, message: "Attachment file changed while uploading") + } + return data + } + ) + } + + private func uploadMemoResourceParts( + memoId: String, + filename: String, + mimeType: String, + byteSize: Int, + readPart: (Range) throws -> Data + ) async throws -> Resource { + struct StartBody: Encodable { + var filename: String + var mimeType: String + var byteSize: Int + } + struct Upload: Decodable { + var id: String + var partSize: Int + var partCount: Int + } + struct StartResponse: Decodable { var upload: Upload } + + let started: StartResponse = try await request( + path: "/api/v1/memos/\(memoId)/resource-uploads", + method: "POST", + body: StartBody(filename: filename, mimeType: mimeType, byteSize: byteSize) + ) + + do { + for partNumber in 1 ... started.upload.partCount { + let start = (partNumber - 1) * started.upload.partSize + let end = min(start + started.upload.partSize, byteSize) + let chunk = try readPart(start ..< end) + var attempt = 0 + while true { + do { + var partRequest = URLRequest( + url: makeURL(path: "/api/v1/resource-uploads/\(started.upload.id)/parts/\(partNumber)") + ) + partRequest.httpMethod = "PUT" + partRequest.setValue("application/octet-stream", forHTTPHeaderField: "Content-Type") + partRequest.setValue(String(chunk.count), forHTTPHeaderField: "Content-Length") + if let token { + partRequest.setValue("Bearer \(token)", forHTTPHeaderField: "Authorization") + } + partRequest.httpBody = chunk + _ = try await performRaw(partRequest) + break + } catch { + attempt += 1 + let retryable = !(error is APIError) + || ((error as? APIError).map { $0.status == 408 || $0.status == 429 || $0.status >= 500 } ?? false) + if !retryable || attempt >= 3 { throw error } + try await Task.sleep(for: .milliseconds(attempt * 250)) + } + } + } + let completed: ResourceResponse = try await request( + path: "/api/v1/resource-uploads/\(started.upload.id)/complete", + method: "POST", + body: EmptyBody() + ) + return completed.resource + } catch { + let _: OkResponse? = try? await request( + path: "/api/v1/resource-uploads/\(started.upload.id)", + method: "DELETE" + ) + throw error } - request.httpBody = body - return try await perform(request).resource } /// Fetch a resource path (usually `/api/v1/resources/:id/blob`) with session auth. @@ -333,6 +418,45 @@ actor APIClient { return (data, mimeType) } + /// Download a protected resource directly to a temporary file without retaining + /// the complete response body in process memory. + func downloadResourceFile(path: String, suggestedFilename: String) async throws -> URL { + var request = URLRequest(url: makeURL(path: path)) + request.httpMethod = "GET" + request.setValue("*/*", forHTTPHeaderField: "Accept") + if let token { + request.setValue("Bearer \(token)", forHTTPHeaderField: "Authorization") + } + let (temporaryURL, response) = try await session.download(for: request) + guard let http = response as? HTTPURLResponse else { + throw APIError(status: -1, code: nil, message: "Invalid response") + } + if http.statusCode == 401 { + onUnauthorized?() + } + guard (200 ..< 300).contains(http.statusCode) else { + throw APIError( + status: http.statusCode, + code: nil, + message: HTTPURLResponse.localizedString(forStatusCode: http.statusCode) + ) + } + + let safeName = suggestedFilename + .replacingOccurrences(of: "/", with: "_") + .trimmingCharacters(in: .whitespacesAndNewlines) + let filename = safeName.isEmpty ? "resource.bin" : safeName + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("edgeever-share", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + let destination = directory.appendingPathComponent(filename) + if FileManager.default.fileExists(atPath: destination.path) { + try FileManager.default.removeItem(at: destination) + } + try FileManager.default.moveItem(at: temporaryURL, to: destination) + return destination + } + /// Fetch an absolute public URL and return bytes + mime (for file:// WebView display). func getPublicURLData(_ absoluteURL: URL) async throws -> (data: Data, mimeType: String) { var request = URLRequest(url: absoluteURL) diff --git a/apps/ios/EdgeEver/Editor/ResourceActions.swift b/apps/ios/EdgeEver/Editor/ResourceActions.swift index 0e907ba0..9f8c8679 100644 --- a/apps/ios/EdgeEver/Editor/ResourceActions.swift +++ b/apps/ios/EdgeEver/Editor/ResourceActions.swift @@ -494,21 +494,11 @@ struct ResourceActionSheet: View { let token = env.session.session?.token let path = "/api/v1/resources/\(target.resourceId)/blob" let client = APIClient(baseURL: base, token: token) - let result = try await client.getResourceData(path: path) - guard !result.data.isEmpty else { throw ResourceActionError.emptyFile } let safeName = target.filename .replacingOccurrences(of: "/", with: "_") .trimmingCharacters(in: .whitespacesAndNewlines) let name = safeName.isEmpty ? "\(target.resourceId).bin" : safeName - let dir = FileManager.default.temporaryDirectory.appendingPathComponent("edgeever-share", isDirectory: true) - try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) - let fileURL = dir.appendingPathComponent(name) - if FileManager.default.fileExists(atPath: fileURL.path) { - try FileManager.default.removeItem(at: fileURL) - } - try result.data.write(to: fileURL, options: .atomic) - // Document picker / share sheet need a file URL that remains readable while presented. - return fileURL + return try await client.downloadResourceFile(path: path, suggestedFilename: name) } } diff --git a/apps/mobile/src/hooks/useMobileEditorResourceActions.ts b/apps/mobile/src/hooks/useMobileEditorResourceActions.ts index 4fd120bb..0484226b 100644 --- a/apps/mobile/src/hooks/useMobileEditorResourceActions.ts +++ b/apps/mobile/src/hooks/useMobileEditorResourceActions.ts @@ -54,19 +54,19 @@ export const useMobileEditorResourceActions = ({ const downloadResource = useCallback(async (target: MobileResourceTarget) => { if (!client) throw unavailable("The attachment client is unavailable.", "当前无法读取附件。"); - await openMobileResource(client, target); - }, [client, unavailable]); + await openMobileResource(client, target, { baseUrl: sessionBaseUrl ?? baseUrl, token }); + }, [baseUrl, client, sessionBaseUrl, token, unavailable]); const saveResourceAs = useCallback(async (target: MobileResourceTarget) => { if (!client) throw unavailable("The resource client is unavailable.", "当前无法读取资源。"); - const result = await saveMobileResourceAs(client, target); + const result = await saveMobileResourceAs(client, target, { baseUrl: sessionBaseUrl ?? baseUrl, token }); if (result.kind === "saf") { Alert.alert( resolvedLocale === "en-US" ? "Downloaded" : "下载成功", resolvedLocale === "en-US" ? `Saved ${result.filename}` : `已保存:${result.filename}`, ); } - }, [client, resolvedLocale, unavailable]); + }, [baseUrl, client, resolvedLocale, sessionBaseUrl, token, unavailable]); const requireMutableClient = useCallback(() => { if (!client || !canMutate()) { diff --git a/apps/mobile/src/lib/mobile-attachments.test.mjs b/apps/mobile/src/lib/mobile-attachments.test.mjs index f83d42d3..229fb096 100644 --- a/apps/mobile/src/lib/mobile-attachments.test.mjs +++ b/apps/mobile/src/lib/mobile-attachments.test.mjs @@ -1,5 +1,6 @@ import { describe, expect, test } from "bun:test"; import { + buildMobileResourceDownloadRequest, deleteMobileAttachmentFromDoc, deleteMobileResourceFromDoc, getMobileAttachmentTarget, @@ -29,6 +30,16 @@ const doc = { }; describe("mobile attachments", () => { + test("builds a native disk download request with protected-resource authentication", () => { + expect(buildMobileResourceDownloadRequest("res/a", { + baseUrl: "https://notes.example///", + token: "secret", + })).toEqual({ + url: "https://notes.example/api/v1/resources/res%2Fa/blob", + headers: { Authorization: "Bearer secret" }, + }); + }); + test("recognizes resource links and attachment paragraphs", () => { expect(getMobileAttachmentTarget(href, "📄 报告:report.pdf")).toEqual(attachmentTarget); expect(getParagraphAttachmentTarget({ diff --git a/apps/mobile/src/lib/mobile-attachments.ts b/apps/mobile/src/lib/mobile-attachments.ts index 56ccd93e..92d4e48d 100644 --- a/apps/mobile/src/lib/mobile-attachments.ts +++ b/apps/mobile/src/lib/mobile-attachments.ts @@ -13,6 +13,19 @@ export type MobileImageTarget = MobileResourceTarget & { kind: "image" }; type AttachmentClient = Pick, "getResourceBlob">; +export type MobileResourceDownloadOptions = { + baseUrl: string; + token?: string | null; +}; + +export const buildMobileResourceDownloadRequest = ( + resourceId: string, + options: MobileResourceDownloadOptions, +) => ({ + url: `${options.baseUrl.replace(/\/+$/, "")}/api/v1/resources/${encodeURIComponent(resourceId)}/blob`, + headers: options.token ? { Authorization: `Bearer ${options.token}` } : undefined, +}); + /** * Convert a Blob to bytes. React Native's Blob polyfill has no arrayBuffer(), * so calling blob.arrayBuffer() throws "undefined is not a function". @@ -249,21 +262,23 @@ export class MobileResourceCancelledError extends Error { } } -const cacheMobileResource = async (client: AttachmentClient, target: MobileResourceTarget) => { +const cacheMobileResource = async ( + _client: AttachmentClient, + target: MobileResourceTarget, + options: MobileResourceDownloadOptions, +) => { const { Directory, File, Paths } = await import("expo-file-system"); - // The DOM editor resolves relative links against the instance URL for display. - // The client itself prefixes the instance base URL, so always give it the - // canonical relative resource path here. - const blob = await client.getResourceBlob(`/api/v1/resources/${encodeURIComponent(target.resourceId)}/blob`); - const bytes = await readBlobAsUint8Array(blob); const directory = new Directory(Paths.cache, "edgeever-attachments"); if (!directory.exists) directory.create({ idempotent: true, intermediates: true }); const file = new File(directory, `${target.resourceId}-${safeCacheFilename(target.filename)}`); if (file.exists) file.delete(); - file.create({ overwrite: true, intermediates: true }); - file.write(bytes); + const request = buildMobileResourceDownloadRequest(target.resourceId, options); + await File.downloadFileAsync(request.url, file, { + headers: request.headers, + idempotent: true, + }); - return { blob, file }; + return { file }; }; const shareCachedResource = async ( @@ -281,9 +296,13 @@ const shareCachedResource = async ( return fileUri; }; -export const openMobileResource = async (client: AttachmentClient, target: MobileResourceTarget) => { - const { blob, file } = await cacheMobileResource(client, target); - const mimeType = resolveResourceMimeType(target.filename, blob.type); +export const openMobileResource = async ( + client: AttachmentClient, + target: MobileResourceTarget, + options: MobileResourceDownloadOptions, +) => { + const { file } = await cacheMobileResource(client, target, options); + const mimeType = resolveResourceMimeType(target.filename); return shareCachedResource(file.uri, { dialogTitle: target.filename, mimeType, @@ -295,10 +314,15 @@ export const openMobileResource = async (client: AttachmentClient, target: Mobil * Android: the action sheet Modal must be closed first — SAF needs a free activity * result channel; otherwise the folder picker never appears and it looks like a no-op. */ -export const saveMobileResourceAs = async (client: AttachmentClient, target: MobileResourceTarget) => { +export const saveMobileResourceAs = async ( + client: AttachmentClient, + target: MobileResourceTarget, + options: MobileResourceDownloadOptions, +) => { const { Platform } = await import("react-native"); - const { blob, file } = await cacheMobileResource(client, target); - const mimeType = resolveResourceMimeType(target.filename, blob.type); + const { File } = await import("expo-file-system"); + const { file } = await cacheMobileResource(client, target, options); + const mimeType = resolveResourceMimeType(target.filename); const exportName = resolveExportFilename(target.filename, mimeType); if (Platform.OS === "android") { @@ -314,10 +338,7 @@ export const saveMobileResourceAs = async (client: AttachmentClient, target: Mob exportName, mimeType ); - const base64 = await file.base64(); - await FileSystem.StorageAccessFramework.writeAsStringAsync(destination, base64, { - encoding: FileSystem.EncodingType.Base64, - }); + await file.copy(new File(destination), { overwrite: true }); return { kind: "saf" as const, uri: destination, filename: exportName }; } catch (error) { if (error instanceof MobileResourceCancelledError) { diff --git a/apps/mobile/src/screens/WorkspaceEditors.tsx b/apps/mobile/src/screens/WorkspaceEditors.tsx index ad2c7245..2b00ad41 100644 --- a/apps/mobile/src/screens/WorkspaceEditors.tsx +++ b/apps/mobile/src/screens/WorkspaceEditors.tsx @@ -533,9 +533,7 @@ export const CreateMemoModal = ({ const memo = await materializeMemoForImage(); setImageOperation("uploading"); const uploadAsset = await prepareUploadAsset(asset, imageCompressionEnabled); - const form = new FormData(); - form.append("file", new ExpoFile(uploadAsset.uri)); - const { resource } = await client!.uploadMemoResource(memo.id, form); + const { resource } = await client!.uploadMemoResource(memo.id, new ExpoFile(uploadAsset.uri)); applyMobileEditorUpload(editorRef, resource, uploadId, uploadAsset.name || (resource.kind === "image" ? "图片" : "附件")); } catch (error) { cancelMobileEditorUpload(editorRef, uploadId); @@ -1004,9 +1002,7 @@ export const RichEditorModal = ({ safeDomCall(() => editorRef.current?.beginImageUpload(uploadId, previewDataUrl)); } const uploadAsset = await prepareUploadAsset(asset, imageCompressionEnabled); - const form = new FormData(); - form.append("file", new ExpoFile(uploadAsset.uri)); - const { resource } = await client.uploadMemoResource(memo.id, form); + const { resource } = await client.uploadMemoResource(memo.id, new ExpoFile(uploadAsset.uri)); applyMobileEditorUpload(editorRef, resource, uploadId, uploadAsset.name || (resource.kind === "image" ? "图片" : "附件")); } catch (uploadError) { cancelMobileEditorUpload(editorRef, uploadId); diff --git a/apps/mobile/src/screens/WorkspaceMemoDetail.tsx b/apps/mobile/src/screens/WorkspaceMemoDetail.tsx index b5a0456a..a2c156ff 100644 --- a/apps/mobile/src/screens/WorkspaceMemoDetail.tsx +++ b/apps/mobile/src/screens/WorkspaceMemoDetail.tsx @@ -504,7 +504,7 @@ export const MemoDetailModal = ({ const downloadResource = useCallback(async (target: MobileResourceTarget) => { if (!client) throw new Error(resolvedLocale === "en-US" ? "The resource client is unavailable." : "当前无法读取资源。"); try { - await openMobileResource(client, target); + await openMobileResource(client, target, { baseUrl, token: session?.token }); } catch (error) { Alert.alert( resolvedLocale === "en-US" ? "Unable to open resource" : "无法打开资源", @@ -512,17 +512,17 @@ export const MemoDetailModal = ({ ); throw error; } - }, [client, resolvedLocale]); + }, [baseUrl, client, resolvedLocale, session?.token]); const saveResourceAs = useCallback(async (target: MobileResourceTarget) => { if (!client) throw new Error(resolvedLocale === "en-US" ? "The resource client is unavailable." : "当前无法读取资源。"); - const result = await saveMobileResourceAs(client, target); + const result = await saveMobileResourceAs(client, target, { baseUrl, token: session?.token }); if (result.kind === "saf") { Alert.alert( resolvedLocale === "en-US" ? "Downloaded" : "下载成功", resolvedLocale === "en-US" ? `Saved ${result.filename}` : `已保存:${result.filename}` ); } - }, [client, resolvedLocale]); + }, [baseUrl, client, resolvedLocale, session?.token]); const loadViewerResource = useCallback((source: string) => { if (!client) { diff --git a/apps/site/src/deployment-prompts.ts b/apps/site/src/deployment-prompts.ts index 842580d1..5d09d5e2 100644 --- a/apps/site/src/deployment-prompts.ts +++ b/apps/site/src/deployment-prompts.ts @@ -1,70 +1,84 @@ export const deploymentPrompts = { "en-US": `Deploy EdgeEver online: 1. Fork https://github.com/tianma-if/edgeever. -2. Import the Fork into Cloudflare Workers & Pages. -3. Create D1 \`edgeever\` and R2 \`edgeever-resources\`, set the - \`EDGE_EVER_AUTH_PASSWORD\` Worker Secret, and configure the production \`main\` build. -4. Start the first build and verify \`/api/health\`, \`/api/openapi.json\`, and login. -5. Enable and manually run the GitHub Actions workflow named \`Update deployed EdgeEver\` +2. Create D1 \`edgeever\` and R2 \`edgeever-resources\` in Cloudflare. +3. Import the Fork into Cloudflare Workers & Pages and use \`main\` as the production + branch. +4. Add a Worker Secret named \`EDGE_EVER_AUTH_PASSWORD\`, using a password chosen by the + user as its value. Prefer a strong password of at least 32 characters that is unique + to this instance. +5. Start the first build, verify \`/api/health\` and \`/api/openapi.json\`, then verify login + with username \`admin\` and the configured password. +6. Enable and manually run the GitHub Actions workflow named \`Update deployed EdgeEver\` once so the Fork can automatically receive the latest EdgeEver features and fixes.`, "zh-CN": `请在线完成 EdgeEver 部署: 1. Fork https://github.com/tianma-if/edgeever。 -2. 将这个 Fork 导入 Cloudflare Workers & Pages。 -3. 创建 D1 \`edgeever\` 与 R2 \`edgeever-resources\`,设置 - \`EDGE_EVER_AUTH_PASSWORD\` Worker Secret,并配置生产环境 \`main\` 构建。 -4. 启动首次构建,验证 \`/api/health\`、\`/api/openapi.json\` 和登录。 -5. 启用并手动运行一次名为 \`Update deployed EdgeEver\` 的 GitHub Actions 工作流, +2. 在 Cloudflare 中创建 D1 \`edgeever\` 与 R2 \`edgeever-resources\`。 +3. 将这个 Fork 导入 Cloudflare Workers & Pages,并将 \`main\` 设为生产分支。 +4. 添加一个名为 \`EDGE_EVER_AUTH_PASSWORD\` 的 Worker Secret,值为用户自行设置的 + 管理员登录密码,建议使用至少 32 个字符且仅用于此实例的强密码。 +5. 启动首次构建,验证 \`/api/health\`、\`/api/openapi.json\`,并使用用户名 \`admin\` + 和配置的密码验证登录。 +6. 启用并手动运行一次名为 \`Update deployed EdgeEver\` 的 GitHub Actions 工作流, 以便后续自动同步更新,持续获得 EdgeEver 最新的产品特性和问题修复。`, } as const; export const manualDeploymentCopy = { "en-US": { - intro: "Complete setup in 5 simple web steps:", + intro: "Complete setup in 6 web steps:", steps: [ { title: "Fork the Repository", body: "Click Fork at the top right of GitHub to fork EdgeEver into your personal account.", }, { - title: "Enable Actions", - body: "Open the Fork's Actions tab and click I understand my workflows, go ahead and enable them so the GitHub Actions workflow named Update deployed EdgeEver can run automatically, keeping you up to date with the latest EdgeEver features and fixes.", + title: "Create Cloudflare Resources", + body: "Create D1 edgeever and R2 edgeever-resources.", }, { - title: "Import into Cloudflare", - body: "Log into the Cloudflare Dashboard, navigate to Workers & Pages, and choose to import your Fork repository.", + title: "Import & Configure the Project", + body: "Import the Fork into Cloudflare Workers & Pages and use main as the production branch. The deploy command creates the bindings; do not edit Fork files.", }, { - title: "Create Resources & Credentials", - body: "Create D1 edgeever and R2 edgeever-resources, then set the Worker Secret EDGE_EVER_AUTH_PASSWORD as your admin password. The deploy command creates the bindings; do not edit Fork files.", + title: "Set the Administrator Password", + body: "Add a Worker Secret named EDGE_EVER_AUTH_PASSWORD and set its value to your chosen administrator login password. Prefer a strong password of at least 32 characters that is unique to this instance.", }, { title: "Build & Verify", - body: "Start the first build after importing the repository. Once complete, visit /api/health to verify a 200 response before logging in.", + body: "Start the initial build. Once deployed, confirm /api/health returns 200, then verify login with username admin and the configured password.", + }, + { + title: "Enable Automatic Updates", + body: "Open the Fork's Actions tab, click I understand my workflows, go ahead and enable them, then manually run Update deployed EdgeEver once so the Fork can automatically receive future EdgeEver features and fixes.", }, ], }, "zh-CN": { - intro: "仅需在网页端完成 5 步极简配置:", + intro: "仅需在网页端完成 6 步配置:", steps: [ { title: "Fork 仓库", body: "在 GitHub 点击右上角 Fork,将项目 Fork 到您的个人账户下。", }, { - title: "启用 Actions", - body: "进入 Fork 的 Actions 标签页,点击 I understand my workflows, go ahead and enable them,确保名为 Update deployed EdgeEver 的 GitHub Actions 工作流能够自动运行,从而持续获得 EdgeEver 最新的产品特性和问题修复。", + title: "创建 Cloudflare 资源", + body: "创建 D1 edgeever 与 R2 edgeever-resources。", }, { - title: "导入 Cloudflare", - body: "登录 Cloudflare 控制台,进入 Workers & Pages,选择导入该 Fork 仓库。", + title: "导入并配置项目", + body: "在 Cloudflare Workers & Pages 中导入该 Fork,并将 main 设为生产分支。binding 由部署命令生成,不要修改 Fork 中的文件。", }, { - title: "创建资源与登录凭据", - body: "创建 D1 edgeever 与 R2 edgeever-resources,并添加 Worker Secret EDGE_EVER_AUTH_PASSWORD 作为管理员登录密码。binding 由部署命令生成,不要修改 Fork 中的文件。", + title: "设置管理员密码", + body: "添加一个名为 EDGE_EVER_AUTH_PASSWORD 的 Worker Secret,并将其值设为您要使用的管理员登录密码。建议使用至少 32 个字符且仅用于此实例的强密码。", }, { - title: "启动构建与验证", - body: "导入仓库后直接启动首次构建。部署完成后访问 /api/health,确认返回 200 即可开始使用。", + title: "首次构建与验证", + body: "启动首次构建。部署完成后访问 /api/health,确认返回 200,并使用用户名 admin 和配置的密码验证登录。", + }, + { + title: "启用自动更新", + body: "进入 Fork 的 Actions 标签页,点击 I understand my workflows, go ahead and enable them,然后手动运行一次 Update deployed EdgeEver,确保后续能够自动获得 EdgeEver 的最新功能与修复。", }, ], }, diff --git a/apps/web/src/components/EditorPane.tsx b/apps/web/src/components/EditorPane.tsx index 64e2d2b7..82ec7cd7 100644 --- a/apps/web/src/components/EditorPane.tsx +++ b/apps/web/src/components/EditorPane.tsx @@ -3056,35 +3056,26 @@ const RichEditorPane = ({ const getResourceActionFailure = useCallback((target: ResourceMenuTarget) => target.kind === "image" ? t("editor.imageActions.failed") : t("editor.attachmentActions.failed"), [t]); - const fetchResourceBlob = useCallback(async (target: ResourceMenuTarget) => { - try { - return await api.getResourceBlob(target.url); - } catch (error) { - if (target.resourceId) throw error; - const response = await fetch(target.url); - if (!response.ok) throw new Error(response.statusText || getResourceActionFailure(target)); - return response.blob(); - } + const fetchResourceResponse = useCallback(async (target: ResourceMenuTarget) => { + const response = target.url.startsWith("edgeever-resource:") || target.url.startsWith("edgeever-staged:") + ? await fetch(target.url) + : await api.getResourceResponse(target.url); + if (!response.ok) throw new Error(response.statusText || getResourceActionFailure(target)); + return response; }, [getResourceActionFailure]); - const downloadBlob = useCallback((blob: Blob, filename: string) => { - const objectUrl = URL.createObjectURL(blob); + const downloadResourceDirectly = useCallback((target: ResourceMenuTarget) => { const anchor = document.createElement("a"); - anchor.href = objectUrl; - anchor.download = filename; + anchor.href = target.url; + anchor.download = target.filename; anchor.click(); - window.setTimeout(() => URL.revokeObjectURL(objectUrl), 0); }, []); const handleResourceDownload = useCallback(async (target: ResourceMenuTarget) => { hideResourceMenu(); clearResourceActionError(); - try { - downloadBlob(await fetchResourceBlob(target), target.filename); - } catch (error) { - failResourceAction(error instanceof Error ? error.message : getResourceActionFailure(target)); - } - }, [clearResourceActionError, downloadBlob, failResourceAction, fetchResourceBlob, getResourceActionFailure, hideResourceMenu]); + downloadResourceDirectly(target); + }, [clearResourceActionError, downloadResourceDirectly, hideResourceMenu]); const handleResourceSaveAs = useCallback(async (target: ResourceMenuTarget) => { hideResourceMenu(); @@ -3092,25 +3083,25 @@ const RichEditorPane = ({ try { const savePicker = (window as Window & { showSaveFilePicker?: (options: { suggestedName: string }) => Promise<{ - createWritable: () => Promise<{ write: (data: Blob) => Promise; close: () => Promise }>; + createWritable: () => Promise>; }>; }).showSaveFilePicker; if (!savePicker) { - downloadBlob(await fetchResourceBlob(target), target.filename); + downloadResourceDirectly(target); return; } const handle = await savePicker.call(window, { suggestedName: target.filename }); - const blob = await fetchResourceBlob(target); + const response = await fetchResourceResponse(target); + if (!response.body) throw new Error(getResourceActionFailure(target)); const writable = await handle.createWritable(); - await writable.write(blob); - await writable.close(); + await response.body.pipeTo(writable); } catch (error) { if (error instanceof DOMException && error.name === "AbortError") return; failResourceAction(error instanceof Error ? error.message : getResourceActionFailure(target)); } - }, [clearResourceActionError, downloadBlob, failResourceAction, fetchResourceBlob, getResourceActionFailure, hideResourceMenu]); + }, [clearResourceActionError, downloadResourceDirectly, failResourceAction, fetchResourceResponse, getResourceActionFailure, hideResourceMenu]); const openResourceDialog = useCallback((action: ResourceDialogState["action"], target: ResourceMenuTarget) => { openResourceActionDialog(action, target); diff --git a/apps/web/src/components/pdf/PdfViewer.tsx b/apps/web/src/components/pdf/PdfViewer.tsx index c17a8059..231a165e 100644 --- a/apps/web/src/components/pdf/PdfViewer.tsx +++ b/apps/web/src/components/pdf/PdfViewer.tsx @@ -23,7 +23,7 @@ import { ButtonTooltip } from "@/components/ui/button-tooltip"; import { isDesktopResourceRuntime, toApiResourceUrl } from "@/lib/desktop-resources"; import { cn } from "@/lib/utils"; import { loadPdfJs } from "./pdfjs-loader"; -import { loadPdfDocumentSource } from "./pdf-document-source"; +import { canPreviewPdfInline, loadPdfDocumentSource } from "./pdf-document-source"; const MIN_ZOOM = 0.5; const MAX_ZOOM = 2.5; @@ -193,7 +193,8 @@ export const PdfViewer = ({ const [failed, setFailed] = useState(false); const [internalFullscreen, setInternalFullscreen] = useState(false); const isFullscreen = fullscreen || internalFullscreen; - const expanded = isFullscreen || (controlledExpanded ?? uncontrolledExpanded); + const previewAllowed = canPreviewPdfInline(byteSize); + const expanded = previewAllowed && (isFullscreen || (controlledExpanded ?? uncontrolledExpanded)); const metadata = formatAttachmentMetadata("application/pdf", filename || label, byteSize); useEffect(() => { @@ -237,6 +238,7 @@ export const PdfViewer = ({ const handleError = useCallback(() => setFailed(true), []); const closeFullscreen = () => fullscreen ? onRequestClose?.() : setInternalFullscreen(false); const setExpanded = (nextExpanded: boolean) => { + if (!previewAllowed) return; if (controlledExpanded === undefined) setUncontrolledExpanded(nextExpanded); onExpandedChange?.(nextExpanded); }; @@ -275,13 +277,15 @@ export const PdfViewer = ({ type="button" className="edgeever-attachment-link flex min-w-0 flex-1 cursor-pointer items-center gap-2 text-left disabled:cursor-default" aria-expanded={expanded} - disabled={isFullscreen} + disabled={isFullscreen || !previewAllowed} onClick={() => setExpanded(!expanded)} > {label} - {metadata} + + {previewAllowed ? metadata : `${metadata} · ${t("pdfViewer.previewTooLarge")}`} + {expanded && !failed ? ( diff --git a/apps/web/src/components/pdf/pdf-document-source.test.mjs b/apps/web/src/components/pdf/pdf-document-source.test.mjs index 2732afea..20e49f47 100644 --- a/apps/web/src/components/pdf/pdf-document-source.test.mjs +++ b/apps/web/src/components/pdf/pdf-document-source.test.mjs @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { loadPdfDocumentSource } from "./pdf-document-source.ts"; +import { canPreviewPdfInline, loadPdfDocumentSource, MAX_INLINE_PDF_BYTES } from "./pdf-document-source.ts"; const bridge = { readResource: async (id) => ({ type: "application/pdf", bytes: new Uint8Array([1, id.length]) }), @@ -7,6 +7,12 @@ const bridge = { }; describe("PDF document sources", () => { + test("keeps oversized PDFs compact instead of materializing them for inline preview", () => { + expect(canPreviewPdfInline(MAX_INLINE_PDF_BYTES)).toBe(true); + expect(canPreviewPdfInline(MAX_INLINE_PDF_BYTES + 1)).toBe(false); + expect(canPreviewPdfInline(undefined)).toBe(true); + }); + test("keeps normal web URLs for PDF.js network loading", async () => { await expect(loadPdfDocumentSource("https://example.com/report.pdf", bridge)).resolves.toEqual({ url: "https://example.com/report.pdf", diff --git a/apps/web/src/components/pdf/pdf-document-source.ts b/apps/web/src/components/pdf/pdf-document-source.ts index cad34792..cc1578f1 100644 --- a/apps/web/src/components/pdf/pdf-document-source.ts +++ b/apps/web/src/components/pdf/pdf-document-source.ts @@ -4,6 +4,13 @@ type PdfResourceBridge = Pick + typeof byteSize !== "number" + || !Number.isFinite(byteSize) + || byteSize <= MAX_INLINE_PDF_BYTES; + const parseDesktopResourceId = (url: string, scheme: "edgeever-resource:" | "edgeever-staged:") => { try { const parsed = new URL(url); diff --git a/apps/web/src/components/settings/DataExportCard.tsx b/apps/web/src/components/settings/DataExportCard.tsx index a4e83cca..e12c0aed 100644 --- a/apps/web/src/components/settings/DataExportCard.tsx +++ b/apps/web/src/components/settings/DataExportCard.tsx @@ -13,11 +13,11 @@ import { } from "@/components/ui/dialog"; import { api, ApiRequestError } from "@/lib/api"; import { - createEdgeEverZip, - downloadEdgeEverZip, EdgeEverZipImportError, + EdgeEverZipMemoryLimitError, parseEdgeEverZip, restoreEdgeEverZipAndRefresh, + saveEdgeEverZip, type EdgeEverZipProgress, type ParsedEdgeEverZip, } from "@/lib/json-backup"; @@ -43,6 +43,7 @@ export const DataExportCard = ({ refreshWorkspaceAfterImport }: DataExportCardPr const fileInputRef = useRef(null); const [state, setState] = useState("idle"); const [operation, setOperation] = useState("export"); + const [scanningImport, setScanningImport] = useState(false); const [progress, setProgress] = useState({ completed: 0, total: 0 }); const [pendingImport, setPendingImport] = useState(null); const [errorMessage, setErrorMessage] = useState(null); @@ -76,18 +77,24 @@ export const DataExportCard = ({ refreshWorkspaceAfterImport }: DataExportCardPr setOperation("export"); setState("working"); setProgress({ completed: 0, total: 0 }); + setScanningImport(false); setErrorMessage(null); try { - const blob = await createEdgeEverZip( - { listNotebooks: api.listNotebooks, listPrompts: api.listAiPrompts, getPage: api.getJsonBackupPage, getResourceBlob: api.getResourceBlob }, + await saveEdgeEverZip( + { listNotebooks: api.listNotebooks, listPrompts: api.listAiPrompts, getPage: api.getJsonBackupPage, getResourceResponse: api.getResourceResponse }, { edgeeverVersion: __EDGEEVER_APP_VERSION__, buildId: __EDGEEVER_BUILD_ID__ }, setProgress ); - downloadEdgeEverZip(blob); setState("complete"); } catch (error) { + if (error instanceof DOMException && error.name === "AbortError") { + setState("idle"); + return; + } console.error("Failed to export EdgeEver ZIP", error); - setErrorMessage(t("dataExport.error")); + setErrorMessage(error instanceof EdgeEverZipMemoryLimitError + ? t("dataExport.largeBackupRequiresStreaming") + : t("dataExport.error")); setState("error"); } }; @@ -97,9 +104,12 @@ export const DataExportCard = ({ refreshWorkspaceAfterImport }: DataExportCardPr setOperation("import"); setState("working"); setProgress({ completed: 0, total: 0 }); + setScanningImport(true); setErrorMessage(null); try { - const parsed = await parseEdgeEverZip(file); + const parsed = await parseEdgeEverZip(file, (percentage) => { + setProgress({ completed: percentage, total: 100 }); + }); setPendingImport(parsed); setState("idle"); } catch (error) { @@ -107,6 +117,7 @@ export const DataExportCard = ({ refreshWorkspaceAfterImport }: DataExportCardPr setErrorMessage(describeImportError(error)); setState("error"); } finally { + setScanningImport(false); if (fileInputRef.current) fileInputRef.current.value = ""; } }; @@ -118,6 +129,7 @@ export const DataExportCard = ({ refreshWorkspaceAfterImport }: DataExportCardPr setOperation("import"); setState("working"); setProgress({ completed: 0, total: 0 }); + setScanningImport(false); setErrorMessage(null); try { await restoreEdgeEverZipAndRefresh( @@ -126,7 +138,7 @@ export const DataExportCard = ({ refreshWorkspaceAfterImport }: DataExportCardPr restoreNotebooks: api.restoreJsonNotebooks, restoreMemos: api.restoreJsonMemos, restorePrompts: api.restoreJsonAiPrompts, - restoreResource: api.restoreJsonResource, + createResourceRestoreSink: api.createJsonResourceRestoreSink, }, refreshWorkspaceAfterImport, setProgress @@ -171,8 +183,10 @@ export const DataExportCard = ({ refreshWorkspaceAfterImport }: DataExportCardPr {busy ? (
- {operation === "import" ? t("dataExport.importing") : t("dataExport.working")} - {t("dataExport.progress", { completed: progress.completed, total: progress.total })} + {operation === "import" ? t(scanningImport ? "dataExport.scanning" : "dataExport.importing") : t("dataExport.working")} + {scanningImport + ? t("dataExport.scanProgress", { percentage: progress.completed }) + : t("dataExport.progress", { completed: progress.completed, total: progress.total })}
diff --git a/apps/web/src/components/settings/WebDavAutoBackup.tsx b/apps/web/src/components/settings/WebDavAutoBackup.tsx index 211dffe9..88ceccd9 100644 --- a/apps/web/src/components/settings/WebDavAutoBackup.tsx +++ b/apps/web/src/components/settings/WebDavAutoBackup.tsx @@ -1,6 +1,6 @@ import { useEffect } from "react"; import { api } from "@/lib/api"; -import { createEdgeEverZip } from "@/lib/json-backup"; +import { createEdgeEverZipTemporaryFile } from "@/lib/json-backup"; import { isWebDavBackupDue, loadWebDavBackupConfig, @@ -31,11 +31,11 @@ export const WebDavAutoBackup = () => { const attemptAt = new Date().toISOString(); saveWebDavBackupSchedule({ ...schedule, lastAttemptAt: attemptAt }); try { - const archive = await createEdgeEverZip( - { listNotebooks: api.listNotebooks, listPrompts: api.listAiPrompts, getPage: api.getJsonBackupPage, getResourceBlob: api.getResourceBlob }, + const temporary = await createEdgeEverZipTemporaryFile( + { listNotebooks: api.listNotebooks, listPrompts: api.listAiPrompts, getPage: api.getJsonBackupPage, getResourceResponse: api.getResourceResponse }, { edgeeverVersion: __EDGEEVER_APP_VERSION__, buildId: __EDGEEVER_BUILD_ID__ } ); - await uploadWebDavBackup(config, password, archive); + await uploadWebDavBackup(config, password, temporary.file).finally(temporary.cleanup); saveWebDavBackupSchedule({ ...schedule, lastAttemptAt: attemptAt, lastSuccessAt: new Date().toISOString() }); } catch (error) { console.error("Automatic WebDAV backup failed", error); diff --git a/apps/web/src/components/settings/WebDavBackupCard.tsx b/apps/web/src/components/settings/WebDavBackupCard.tsx index 98fb4601..e9c25907 100644 --- a/apps/web/src/components/settings/WebDavBackupCard.tsx +++ b/apps/web/src/components/settings/WebDavBackupCard.tsx @@ -5,7 +5,7 @@ import { Button } from "@/components/ui/button"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { Input } from "@/components/ui/input"; import { api } from "@/lib/api"; -import { createEdgeEverZip, type EdgeEverZipProgress } from "@/lib/json-backup"; +import { createEdgeEverZipTemporaryFile, EdgeEverZipMemoryLimitError, type EdgeEverZipProgress } from "@/lib/json-backup"; import { loadWebDavBackupConfig, loadWebDavBackupPassword, @@ -52,6 +52,7 @@ export const WebDavBackupCard = () => { }; const describeError = (error: unknown) => { + if (error instanceof EdgeEverZipMemoryLimitError) return t("dataExport.webdavErrors.largeBackupRequiresStreaming"); if (error instanceof TypeError) return t("dataExport.webdavErrors.network"); if (error instanceof Error && error.message) return error.message; return t("dataExport.webdavErrors.unknown"); @@ -79,12 +80,13 @@ export const WebDavBackupCard = () => { setMessage(null); try { const normalized = persistSettings(); - const archive = await createEdgeEverZip( - { listNotebooks: api.listNotebooks, listPrompts: api.listAiPrompts, getPage: api.getJsonBackupPage, getResourceBlob: api.getResourceBlob }, + const temporary = await createEdgeEverZipTemporaryFile( + { listNotebooks: api.listNotebooks, listPrompts: api.listAiPrompts, getPage: api.getJsonBackupPage, getResourceResponse: api.getResourceResponse }, { edgeeverVersion: __EDGEEVER_APP_VERSION__, buildId: __EDGEEVER_BUILD_ID__ }, setProgress ); - const result = await uploadWebDavBackup(normalized, password, archive); + const result = await uploadWebDavBackup(normalized, password, temporary.file) + .finally(temporary.cleanup); const nextSchedule = { ...schedule, lastSuccessAt: new Date().toISOString() }; saveWebDavBackupSchedule(nextSchedule); setSchedule(nextSchedule); diff --git a/apps/web/src/lib/desktop-resources.test.mjs b/apps/web/src/lib/desktop-resources.test.mjs index b340d2ef..2b5db994 100644 --- a/apps/web/src/lib/desktop-resources.test.mjs +++ b/apps/web/src/lib/desktop-resources.test.mjs @@ -19,6 +19,7 @@ afterAll(() => { const { mapTiptapResourceUrls, createStagedResourceListItem, + stageDesktopResource, toApiResourceUrl, toDesktopResourceUrl, } = await import("./desktop-resources.ts"); @@ -57,4 +58,26 @@ describe("desktop resource URLs", () => { createdAt: "2026-01-01T00:00:00.000Z", }); }); + + test("stages offline attachments in bounded chunks", async () => { + const parts = []; + window.edgeeverDesktop.beginStagedResource = async (metadata) => { + expect(metadata).toMatchObject({ memoId: "memo-1", name: "archive.bin", size: 18 }); + return { id: "stage-streamed", partSize: 8 }; + }; + window.edgeeverDesktop.appendStagedResource = async (_id, bytes) => { + parts.push(Array.from(new Uint8Array(bytes))); + return { receivedBytes: parts.reduce((total, part) => total + part.length, 0) }; + }; + window.edgeeverDesktop.completeStagedResource = async (id) => ({ id }); + window.edgeeverDesktop.abortStagedResource = async () => {}; + + const staged = await stageDesktopResource( + "memo-1", + new File(["abcdefghijklmnopqr"], "archive.bin", { type: "application/octet-stream" }), + ); + + expect(staged).toEqual({ id: "stage-streamed" }); + expect(parts.map((part) => part.length)).toEqual([8, 8, 2]); + }); }); diff --git a/apps/web/src/lib/desktop-resources.ts b/apps/web/src/lib/desktop-resources.ts index 1d68563e..1e8b5a58 100644 --- a/apps/web/src/lib/desktop-resources.ts +++ b/apps/web/src/lib/desktop-resources.ts @@ -53,7 +53,23 @@ export const mapMarkdownResourceUrls = (markdown: string | undefined, mapper: (u export const stageDesktopResource = async (memoId: string, file: File) => { if (!isDesktopResourceRuntime()) return null; - return window.edgeeverDesktop!.stageResource({ memoId, name: file.name, type: file.type, bytes: await file.arrayBuffer() }); + const bridge = window.edgeeverDesktop!; + const staged = await bridge.beginStagedResource({ + memoId, + name: file.name, + type: file.type, + size: file.size, + }); + try { + for (let start = 0; start < file.size; start += staged.partSize) { + const bytes = await file.slice(start, Math.min(start + staged.partSize, file.size)).arrayBuffer(); + await bridge.appendStagedResource(staged.id, bytes); + } + return await bridge.completeStagedResource(staged.id); + } catch (error) { + await bridge.abortStagedResource(staged.id).catch(() => undefined); + throw error; + } }; export const createStagedResourceListItem = ( diff --git a/apps/web/src/lib/desktop-sync.ts b/apps/web/src/lib/desktop-sync.ts index 165fc903..ff5f681f 100644 --- a/apps/web/src/lib/desktop-sync.ts +++ b/apps/web/src/lib/desktop-sync.ts @@ -67,9 +67,15 @@ const syncStagedResources = async (memoIdMappings: Map) => { // that the note content containing the placeholder has been saved. if (!isStagedResourceReferenced(pendingPayloads, item.id)) continue; try { - const stored = await window.edgeeverDesktop!.readStagedResource(item.id); const memoId = memoIdMappings.get(item.memoId) ?? item.memoId; - const uploaded = await api.uploadMemoResource(memoId, new File([stored.bytes as unknown as ArrayBuffer], stored.name, { type: stored.type })); + const uploaded = await api.uploadMemoResourceParts(memoId, { + filename: item.name, + mimeType: item.type, + byteSize: item.size, + readPart: async (start, end) => new Blob([ + await window.edgeeverDesktop!.readStagedResourcePart(item.id, start, end - start), + ], { type: item.type }), + }); await window.edgeeverDesktop!.sidecarRequest("resource.cache", { resource: uploaded.resource }); rewrites.push({ memoId, placeholder: `edgeever-staged://${item.id}`, url: uploaded.resource.url }); stagedIds.push(item.id); diff --git a/apps/web/src/lib/json-backup.ts b/apps/web/src/lib/json-backup.ts index 717440f5..ed7bf725 100644 --- a/apps/web/src/lib/json-backup.ts +++ b/apps/web/src/lib/json-backup.ts @@ -15,7 +15,16 @@ import { type Notebook, type Resource, } from "@edgeever/shared"; -import { strFromU8, strToU8, unzip, Zip, ZipDeflate, ZipPassThrough } from "fflate"; +import { + strFromU8, + strToU8, + Unzip, + UnzipInflate, + Zip, + ZipDeflate, + ZipPassThrough, + type UnzipFile, +} from "fflate"; import { buildMarkdownFrontMatter, buildNotebookExportPaths, @@ -62,14 +71,24 @@ type JsonBackupPage = { type JsonBackupSource = { listNotebooks: () => Promise<{ notebooks: Notebook[] }>; getPage: (offset: number, limit: number) => Promise; - getResourceBlob: (resourceUrl: string) => Promise; + getResourceResponse?: (resourceUrl: string) => Promise; + getResourceBlob?: (resourceUrl: string) => Promise; listPrompts: () => Promise<{ prompts: JsonBackupAiPrompt[] }>; }; +type JsonResourceRestoreSink = { + write: (chunk: Uint8Array) => Promise; + close: () => Promise; + abort: () => Promise; +}; + type JsonRestoreTarget = { restoreNotebooks: (notebooks: JsonBackupNotebook[]) => Promise; restoreMemos: (memos: JsonBackupMemo[]) => Promise; - restoreResource: (resourceId: string, metadata: JsonBackupResource, file: Blob) => Promise; + createResourceRestoreSink: ( + resourceId: string, + metadata: JsonBackupResource, + ) => Promise; restorePrompts: (prompts: JsonBackupAiPrompt[]) => Promise; }; @@ -78,11 +97,12 @@ export type ParsedEdgeEverZip = { notebooks: JsonBackupNotebook[]; memos: JsonBackupMemo[]; prompts: JsonBackupAiPrompt[]; - files: Record; + archive: Blob; }; const PAGE_SIZE = 25; const ZIP_MIME_TYPE = "application/zip"; +const MAX_BUFFERED_ZIP_BYTES = 128 * 1024 * 1024; const jsonBytes = (value: unknown) => strToU8(`${JSON.stringify(value, null, 2)}\n`); const addJsonFile = (zip: Zip, path: string, value: unknown) => { @@ -91,6 +111,37 @@ const addJsonFile = (zip: Zip, path: string, value: unknown) => { file.push(jsonBytes(value), true); }; +const streamResourceIntoZip = async ( + source: JsonBackupSource, + resourceUrl: string, + file: ZipPassThrough, + waitForOutputCapacity: () => Promise, +) => { + if (!source.getResourceResponse && !source.getResourceBlob) { + throw new Error("A resource response source is required"); + } + const response = source.getResourceResponse + ? await source.getResourceResponse(resourceUrl) + : new Response(await source.getResourceBlob!(resourceUrl)); + if (!response.ok) throw new Error(`Resource download failed (${response.status})`); + if (!response.body) { + file.push(new Uint8Array(), true); + return; + } + const reader = response.body.getReader(); + try { + while (true) { + const next = await reader.read(); + if (next.done) break; + file.push(next.value, false); + await waitForOutputCapacity(); + } + file.push(new Uint8Array(), true); + } finally { + reader.releaseLock(); + } +}; + const toBackupNotebook = (notebook: Notebook): JsonBackupNotebook => ({ id: notebook.id, parentId: notebook.parentId, @@ -103,7 +154,7 @@ const toBackupNotebook = (notebook: Notebook): JsonBackupNotebook => ({ updatedAt: notebook.updatedAt, }); -export const createEdgeEverZip = async ( +export const createEdgeEverZipStream = async ( source: JsonBackupSource, version: { edgeeverVersion: string; buildId: string }, onProgress?: (progress: EdgeEverZipProgress) => void @@ -114,25 +165,31 @@ export const createEdgeEverZip = async ( ]); const notebookPaths = buildNotebookExportPaths(notebooks); const memoNamesByNotebook = new Map>(); - const chunks: ArrayBuffer[] = []; + let zip: Zip | null = null; + let releaseBackpressure: (() => void) | null = null; - return new Promise((resolve, reject) => { - const zip = new Zip((error, data, final) => { - if (error) { - reject(error); - return; - } - chunks.push(new Uint8Array(data).buffer); - if (final) { - resolve(new Blob(chunks, { type: ZIP_MIME_TYPE })); - } - }); + return new ReadableStream({ + start(controller) { + const waitForOutputCapacity = () => { + if ((controller.desiredSize ?? 1) > 0) return Promise.resolve(); + return new Promise((resolve) => { + releaseBackpressure = resolve; + }); + }; + zip = new Zip((error, data, final) => { + if (error) { + controller.error(error); + return; + } + if (data.byteLength > 0) controller.enqueue(data); + if (final) controller.close(); + }); - void (async () => { - try { + void (async () => { + try { const backupNotebooks = notebooks.map(toBackupNotebook); - addJsonFile(zip, "notebooks.json", backupNotebooks); - addJsonFile(zip, "prompts.json", prompts); + addJsonFile(zip!, "notebooks.json", backupNotebooks); + addJsonFile(zip!, "prompts.json", prompts); let offset = 0; let completed = 0; let total = 0; @@ -178,10 +235,9 @@ export const createEdgeEverZip = async ( ); const relativePath = `${assetDirectory}/${filename}`; const archivePath = `${markdownDirectory}/${relativePath}`; - const blob = await source.getResourceBlob(resource.url); const file = new ZipPassThrough(archivePath); - zip.add(file); - file.push(new Uint8Array(await blob.arrayBuffer()), true); + zip!.add(file); + await streamResourceIntoZip(source, resource.url, file, waitForOutputCapacity); backupResources.push({ id: resource.id, memoId: resource.memoId, @@ -202,9 +258,9 @@ export const createEdgeEverZip = async ( } const revisions = revisionsByMemo.get(memo.id) ?? []; - addJsonFile(zip, `memos/${memo.id}.json`, { memo, revisions, resources: backupResources }); + addJsonFile(zip!, `memos/${memo.id}.json`, { memo, revisions, resources: backupResources }); const markdownFile = new ZipDeflate(`${markdownDirectory}/${memoStem}.md`, { level: 6 }); - zip.add(markdownFile); + zip!.add(markdownFile); markdownFile.push(strToU8(`${buildMarkdownFrontMatter(memo, notebookPath)}${markdown}`), true); revisionCount += revisions.length; completed += 1; @@ -233,25 +289,243 @@ export const createEdgeEverZip = async ( prompts: prompts.length, }, }; - addJsonFile(zip, "manifest.json", manifest); - zip.end(); - } catch (error) { - zip.terminate(); - reject(error); - } - })(); + addJsonFile(zip!, "manifest.json", manifest); + zip!.end(); + } catch (error) { + zip?.terminate(); + controller.error(error); + } + })(); + }, + pull() { + releaseBackpressure?.(); + releaseBackpressure = null; + }, + cancel() { + releaseBackpressure?.(); + releaseBackpressure = null; + zip?.terminate(); + }, }); }; -const parseJsonFile = ( - files: Record, - path: string, - missingCode: EdgeEverZipImportErrorCode = "missingData" -) => { - const data = files[path]; - if (!data) { - throw new EdgeEverZipImportError(missingCode); +export const createEdgeEverZip = async ( + source: JsonBackupSource, + version: { edgeeverVersion: string; buildId: string }, + onProgress?: (progress: EdgeEverZipProgress) => void, +) => new Response(await createEdgeEverZipStream(source, version, onProgress), { + headers: { "Content-Type": ZIP_MIME_TYPE }, +}).blob(); + +export class EdgeEverZipMemoryLimitError extends Error { + constructor() { + super("This backup is too large for an in-memory export. Use a browser that supports streaming file saves."); + this.name = "EdgeEverZipMemoryLimitError"; } +} + +const bufferedZipBlob = async (stream: ReadableStream) => { + const reader = stream.getReader(); + const chunks: ArrayBuffer[] = []; + let total = 0; + try { + while (true) { + const next = await reader.read(); + if (next.done) break; + total += next.value.byteLength; + if (total > MAX_BUFFERED_ZIP_BYTES) { + await reader.cancel(); + throw new EdgeEverZipMemoryLimitError(); + } + const copy = new Uint8Array(next.value.byteLength); + copy.set(next.value); + chunks.push(copy.buffer); + } + } finally { + reader.releaseLock(); + } + return new Blob(chunks, { type: ZIP_MIME_TYPE }); +}; + +type StreamingFileHandle = { + createWritable: () => Promise>; + getFile?: () => Promise; +}; + +export const saveEdgeEverZip = async ( + source: JsonBackupSource, + version: { edgeeverVersion: string; buildId: string }, + onProgress?: (progress: EdgeEverZipProgress) => void, +) => { + const filename = `edgeever-export-${new Date().toISOString().slice(0, 10)}.zip`; + const savePicker = (window as Window & { + showSaveFilePicker?: (options: { suggestedName: string }) => Promise; + }).showSaveFilePicker; + if (savePicker) { + const handle = await savePicker.call(window, { suggestedName: filename }); + const stream = await createEdgeEverZipStream(source, version, onProgress); + await stream.pipeTo(await handle.createWritable()); + return { filename, streamed: true as const }; + } + + const stream = await createEdgeEverZipStream(source, version, onProgress); + downloadEdgeEverZip(await bufferedZipBlob(stream), filename); + return { filename, streamed: false as const }; +}; + +export const createEdgeEverZipTemporaryFile = async ( + source: JsonBackupSource, + version: { edgeeverVersion: string; buildId: string }, + onProgress?: (progress: EdgeEverZipProgress) => void, +) => { + const storage = navigator.storage as StorageManager & { + getDirectory?: () => Promise<{ + getFileHandle: (name: string, options: { create: true }) => Promise; + removeEntry: (name: string) => Promise; + }>; + }; + if (!storage.getDirectory) { + const stream = await createEdgeEverZipStream(source, version, onProgress); + return { file: await bufferedZipBlob(stream), cleanup: async () => {} }; + } + + const root = await storage.getDirectory(); + const name = `edgeever-backup-${crypto.randomUUID()}.zip`; + const handle = await root.getFileHandle(name, { create: true }); + const stream = await createEdgeEverZipStream(source, version, onProgress); + try { + await stream.pipeTo(await handle.createWritable()); + if (!handle.getFile) throw new Error("The temporary backup file is unavailable"); + return { + file: await handle.getFile(), + cleanup: () => root.removeEntry(name).catch(() => {}), + }; + } catch (error) { + await root.removeEntry(name).catch(() => {}); + throw error; + } +}; + +type ZipEntryConsumer = { + write?: (chunk: Uint8Array) => void | Promise; + close?: () => void | Promise; +}; + +const streamZipEntries = async ( + blob: Blob, + onEntry: (file: UnzipFile) => ZipEntryConsumer, + onInputProgress?: (completedBytes: number) => void, +) => { + let processing = Promise.resolve(); + const unzipper = new Unzip((file) => { + const consumer = onEntry(file); + file.ondata = (error, data, final) => { + processing = processing.then(async () => { + if (error) throw new EdgeEverZipImportError("invalidZip", error); + if (data?.byteLength) await consumer.write?.(data); + if (final) await consumer.close?.(); + }); + }; + file.start(); + }); + unzipper.register(UnzipInflate); + + const reader = blob.stream().getReader(); + let completedBytes = 0; + try { + while (true) { + const next = await reader.read(); + completedBytes += next.value?.byteLength ?? 0; + onInputProgress?.(completedBytes); + try { + unzipper.push(next.value ?? new Uint8Array(), next.done); + } catch (error) { + if (error instanceof EdgeEverZipImportError) throw error; + throw new EdgeEverZipImportError("invalidZip", error); + } + await processing; + if (next.done) break; + } + } finally { + reader.releaseLock(); + } +}; + +const MAX_ZIP_METADATA_ENTRY_BYTES = 32 * 1024 * 1024; +const MAX_ZIP_ENTRY_BYTES = 1024 * 1024 * 1024; +const MAX_RESTORED_IMAGE_BYTES = 100 * 1024 * 1024; + +const isStructuredZipEntry = (path: string) => path === "manifest.json" + || path === "notebooks.json" + || path === "prompts.json" + || /^memos\/[^/]+\.json$/.test(path); + +const concatZipChunks = (chunks: Uint8Array[], total: number) => { + const bytes = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + return bytes; +}; + +const scanEdgeEverZip = async (blob: Blob, onProgress?: (percentage: number) => void) => { + const signature = new Uint8Array(await blob.slice(0, 4).arrayBuffer()); + const signatureValue = signature.length === 4 + ? signature[0] | (signature[1] << 8) | (signature[2] << 16) | (signature[3] << 24) + : 0; + if (signatureValue !== 0x04034b50 && signatureValue !== 0x06054b50) { + throw new EdgeEverZipImportError("invalidZip"); + } + const entrySizes = new Map(); + const structuredEntries = new Map(); + let lastPercentage = -1; + + await streamZipEntries(blob, (file) => { + if (entrySizes.has(file.name)) { + throw new EdgeEverZipImportError("invalidZip", new Error(`Duplicate ZIP entry: ${file.name}`)); + } + entrySizes.set(file.name, -1); + const structured = isStructuredZipEntry(file.name); + const chunks: Uint8Array[] = []; + let byteSize = 0; + return { + write(chunk) { + byteSize += chunk.byteLength; + const limit = structured ? MAX_ZIP_METADATA_ENTRY_BYTES : MAX_ZIP_ENTRY_BYTES; + if (byteSize > limit) { + throw new EdgeEverZipImportError(structured ? "invalidData" : "incompleteResources"); + } + if (structured) { + const copy = new Uint8Array(chunk.byteLength); + copy.set(chunk); + chunks.push(copy); + } + }, + close() { + entrySizes.set(file.name, byteSize); + if (structured) structuredEntries.set(file.name, concatZipChunks(chunks, byteSize)); + }, + }; + }, (completedBytes) => { + const percentage = blob.size > 0 ? Math.min(100, Math.round((completedBytes / blob.size) * 100)) : 100; + if (percentage !== lastPercentage) { + lastPercentage = percentage; + onProgress?.(percentage); + } + }); + + return { entrySizes, structuredEntries }; +}; + +const parseJsonEntry = ( + entries: Map, + path: string, + missingCode: EdgeEverZipImportErrorCode = "missingData", +) => { + const data = entries.get(path); + if (!data) throw new EdgeEverZipImportError(missingCode); try { return JSON.parse(strFromU8(data)) as unknown; } catch (error) { @@ -259,13 +533,6 @@ const parseJsonFile = ( } }; -const unzipBlob = async (blob: Blob) => { - const bytes = new Uint8Array(await blob.arrayBuffer()); - return new Promise>((resolve, reject) => { - unzip(bytes, (error, files) => error ? reject(new EdgeEverZipImportError("invalidZip", error)) : resolve(files)); - }); -}; - const sortNotebooksForRestore = (notebooks: JsonBackupNotebook[]) => { const byId = new Map(notebooks.map((notebook) => [notebook.id, notebook])); const depth = (notebook: JsonBackupNotebook, seen = new Set()): number => { @@ -278,9 +545,12 @@ const sortNotebooksForRestore = (notebooks: JsonBackupNotebook[]) => { return [...notebooks].sort((left, right) => depth(left) - depth(right)); }; -export const parseEdgeEverZip = async (blob: Blob): Promise => { - const files = await unzipBlob(blob); - const manifestValue = parseJsonFile(files, "manifest.json", "missingManifest"); +export const parseEdgeEverZip = async ( + blob: Blob, + onProgress?: (percentage: number) => void, +): Promise => { + const { entrySizes, structuredEntries } = await scanEdgeEverZip(blob, onProgress); + const manifestValue = parseJsonEntry(structuredEntries, "manifest.json", "missingManifest"); if (!manifestValue || typeof manifestValue !== "object") { throw new EdgeEverZipImportError("invalidManifest"); } @@ -296,7 +566,7 @@ export const parseEdgeEverZip = async (blob: Blob): Promise = throw new EdgeEverZipImportError("invalidManifest", manifestResult.error); } const manifest = manifestResult.data; - const notebooksValue = parseJsonFile(files, "notebooks.json"); + const notebooksValue = parseJsonEntry(structuredEntries, "notebooks.json"); if (!Array.isArray(notebooksValue)) { throw new EdgeEverZipImportError("invalidData"); } @@ -306,23 +576,23 @@ export const parseEdgeEverZip = async (blob: Blob): Promise = } const notebooks = sortNotebooksForRestore(notebooksResult.data); const promptsValue = manifest.formatVersion >= 2 - ? parseJsonFile(files, "prompts.json") + ? parseJsonEntry(structuredEntries, "prompts.json") : []; const promptsResult = JsonBackupAiPromptSchema.array().safeParse(promptsValue); if (!promptsResult.success) { throw new EdgeEverZipImportError("invalidData", promptsResult.error); } const prompts = promptsResult.data as JsonBackupAiPrompt[]; - const memoPaths = Object.keys(files).filter((path) => /^memos\/[^/]+\.json$/.test(path)).sort(); + const memoPaths = [...structuredEntries.keys()].filter((path) => /^memos\/[^/]+\.json$/.test(path)).sort(); const memos: JsonBackupMemo[] = []; for (const path of memoPaths) { - const memoResult = JsonBackupMemoSchema.safeParse(parseJsonFile(files, path)); + const memoResult = JsonBackupMemoSchema.safeParse(parseJsonEntry(structuredEntries, path)); if (!memoResult.success) { throw new EdgeEverZipImportError("invalidData", memoResult.error); } memos.push(memoResult.data as JsonBackupMemo); } - const markdownCount = Object.keys(files).filter((path) => /^notes\/.+\.md$/.test(path)).length; + const markdownCount = [...entrySizes.keys()].filter((path) => /^notes\/.+\.md$/.test(path)).length; if ( manifest.counts.notebooks !== notebooks.length @@ -334,18 +604,25 @@ export const parseEdgeEverZip = async (blob: Blob): Promise = } const resources = memos.flatMap((memo) => memo.resources); + const resourcePaths = new Set(resources.map((resource) => resource.archivePath)); const revisionCount = memos.reduce((count, memo) => count + memo.revisions.length, 0); if (manifest.counts.revisions !== revisionCount) { throw new EdgeEverZipImportError("incompleteData"); } if ( manifest.counts.resources !== resources.length - || resources.some((resource) => !files[resource.archivePath] || files[resource.archivePath].byteLength !== resource.byteSize) + || resourcePaths.size !== resources.length + || resources.some((resource) => ( + resource.byteSize <= 0 + || resource.byteSize > (resource.kind === "image" ? MAX_RESTORED_IMAGE_BYTES : MAX_ZIP_ENTRY_BYTES) + || isStructuredZipEntry(resource.archivePath) + || entrySizes.get(resource.archivePath) !== resource.byteSize + )) ) { throw new EdgeEverZipImportError("incompleteResources"); } - return { manifest, notebooks, memos, prompts, files }; + return { manifest, notebooks, memos, prompts, archive: blob }; }; export const restoreEdgeEverZip = async ( @@ -381,18 +658,51 @@ export const restoreEdgeEverZip = async ( onProgress?.({ completed, total }); } - for (const memo of backup.memos) { - for (const resource of memo.resources) { - await target.restoreResource( - resource.id, - resource, - new Blob([new Uint8Array(backup.files[resource.archivePath])], { - type: resource.mimeType || "application/octet-stream", - }) - ); - completed += 1; - onProgress?.({ completed, total }); - } + const resourcesByPath = new Map( + backup.memos.flatMap((memo) => memo.resources).map((resource) => [resource.archivePath, resource]), + ); + const restoredPaths = new Set(); + const activeSinks = new Set(); + try { + await streamZipEntries(backup.archive, (file) => { + const resource = resourcesByPath.get(file.name); + if (!resource) return {}; + const sinkPromise = target.createResourceRestoreSink(resource.id, resource); + let byteSize = 0; + return { + async write(chunk) { + byteSize += chunk.byteLength; + if (byteSize > resource.byteSize) { + const sink = await sinkPromise; + await sink.abort(); + throw new EdgeEverZipImportError("incompleteResources"); + } + const sink = await sinkPromise; + activeSinks.add(sink); + await sink.write(chunk); + }, + async close() { + const sink = await sinkPromise; + activeSinks.add(sink); + if (byteSize !== resource.byteSize) { + await sink.abort(); + activeSinks.delete(sink); + throw new EdgeEverZipImportError("incompleteResources"); + } + await sink.close(); + activeSinks.delete(sink); + restoredPaths.add(file.name); + completed += 1; + onProgress?.({ completed, total }); + }, + }; + }); + } catch (error) { + await Promise.all([...activeSinks].map((sink) => sink.abort().catch(() => undefined))); + throw error; + } + if (restoredPaths.size !== resourcesByPath.size) { + throw new EdgeEverZipImportError("incompleteResources"); } }; @@ -406,12 +716,14 @@ export const restoreEdgeEverZipAndRefresh = async ( await refreshWorkspace(); }; -export const downloadEdgeEverZip = (blob: Blob) => { - const date = new Date().toISOString().slice(0, 10); +export const downloadEdgeEverZip = ( + blob: Blob, + filename = `edgeever-export-${new Date().toISOString().slice(0, 10)}.zip`, +) => { const url = URL.createObjectURL(blob); const anchor = document.createElement("a"); anchor.href = url; - anchor.download = `edgeever-export-${date}.zip`; + anchor.download = filename; document.body.appendChild(anchor); anchor.click(); anchor.remove(); diff --git a/apps/web/src/lib/mobile-editor-standalone.ts b/apps/web/src/lib/mobile-editor-standalone.ts index 4111f742..8dc26e70 100644 --- a/apps/web/src/lib/mobile-editor-standalone.ts +++ b/apps/web/src/lib/mobile-editor-standalone.ts @@ -1,5 +1,5 @@ import { emptyDoc, resolveMemoContentDoc, type MemoDetail, type Resource, type TiptapDoc } from "@edgeever/shared"; -import { ApiRequestError } from "@/lib/api"; +import { ApiRequestError, api } from "@/lib/api"; export const MOBILE_EDITOR_LEAVE_SAVE_TIMEOUT_MS = 1600; export const MOBILE_EDITOR_INITIAL_FOCUS_DELAY_MS = 160; @@ -114,13 +114,7 @@ export const requestMobileEditorJson = async (path: string, init?: RequestIn }; export const uploadMobileEditorResource = async (memoId: string, file: File) => { - const form = new FormData(); - form.append("file", file); - - return requestMobileEditorJson(`/api/v1/memos/${encodeURIComponent(memoId)}/resources`, { - method: "POST", - body: form, - }); + return api.uploadMemoResource(memoId, file); }; export const normalizeMobileEditorDoc = (memo: MemoDetail): TiptapDoc => { diff --git a/apps/web/src/lib/webdav-backup.ts b/apps/web/src/lib/webdav-backup.ts index 5b52a6aa..aac80d8a 100644 --- a/apps/web/src/lib/webdav-backup.ts +++ b/apps/web/src/lib/webdav-backup.ts @@ -147,7 +147,10 @@ export const uploadWebDavBackup = async ( const filename = `edgeever-${timestamp.toISOString().replace(/[:.]/g, "-")}.zip`; const remotePath = `${normalized.remotePath}/${filename}`; - const uploaded = await client.putFileContents(remotePath, await archive.arrayBuffer(), { + // The fetch-backed browser client accepts Blob/File bodies even though its + // published TypeScript signature only lists ArrayBuffer-like payloads. + const uploaded = await client.putFileContents(remotePath, archive as never, { + contentLength: archive.size, overwrite: false, }); diff --git a/apps/web/src/vite-env.d.ts b/apps/web/src/vite-env.d.ts index 6279c66e..2d24419b 100644 --- a/apps/web/src/vite-env.d.ts +++ b/apps/web/src/vite-env.d.ts @@ -49,10 +49,14 @@ interface EdgeEverDesktopBridge { installUpdate(): Promise; onUpdateStatus(callback: (status: DesktopUpdateStatus) => void): () => void; sidecarRequest(method: string, params?: Record): Promise; - stageResource(input: { memoId: string; name: string; type: string; bytes: ArrayBuffer }): Promise<{ id: string }>; + beginStagedResource(input: { memoId: string; name: string; type: string; size: number }): Promise<{ id: string; partSize: number }>; + appendStagedResource(id: string, bytes: ArrayBuffer): Promise<{ receivedBytes: number }>; + completeStagedResource(id: string): Promise<{ id: string }>; + abortStagedResource(id: string): Promise; listStagedResources(): Promise>; remapStagedResourceMemoIds?(mappings: Array<[string, string]>): Promise<{ updated: number }>; readStagedResource(id: string): Promise<{ name: string; type: string; bytes: Uint8Array }>; + readStagedResourcePart(id: string, start: number, length: number): Promise; readResource(id: string): Promise<{ type: string; bytes: Uint8Array }>; removeStagedResource(id: string): Promise; onCommand(callback: (command: string) => void): () => void; diff --git a/docs/openapi.json b/docs/openapi.json index d509e106..f02fc16b 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -1299,7 +1299,7 @@ ], "post": { "summary": "Upload an image or attachment resource for a memo", - "description": "Uploads an image or attachment resource for a memo. Images are stored as provided; server-side compression is disabled.", + "description": "Legacy single-request upload for files up to 100 MiB. New clients should use the resumable multipart upload endpoints. Images are stored as provided; server-side compression is disabled.", "requestBody": { "required": true, "content": { @@ -1337,6 +1337,103 @@ } } }, + "/api/v1/memos/{id}/resource-uploads": { + "parameters": [ + { + "$ref": "#/components/parameters/MemoId" + } + ], + "post": { + "summary": "Initialize a resumable resource upload", + "description": "Creates an 8 MiB-part upload session. Attachments support up to 1 GiB; images support up to 100 MiB to keep decoding and previews bounded. The session expires after 24 hours.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "filename": { "type": "string" }, + "mimeType": { "type": "string" }, + "byteSize": { "type": "integer", "minimum": 1, "maximum": 1073741824 } + }, + "required": ["filename", "mimeType", "byteSize"] + } + } + } + }, + "responses": { + "201": { + "description": "Initialized upload", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "upload": { "$ref": "#/components/schemas/ResourceUpload" } + }, + "required": ["upload"] + } + } + } + } + } + } + }, + "/api/v1/resource-uploads/{uploadId}/parts/{partNumber}": { + "parameters": [ + { "name": "uploadId", "in": "path", "required": true, "schema": { "type": "string" } }, + { "name": "partNumber", "in": "path", "required": true, "schema": { "type": "integer", "minimum": 1 } } + ], + "put": { + "summary": "Upload or replace one resource part", + "description": "Streams one exact-sized part to object storage. Repeating a part number safely replaces that part.", + "requestBody": { + "required": true, + "content": { + "application/octet-stream": { + "schema": { "type": "string", "format": "binary" } + } + } + }, + "responses": { + "200": { "description": "Part stored" } + } + } + }, + "/api/v1/resource-uploads/{uploadId}/complete": { + "parameters": [ + { "name": "uploadId", "in": "path", "required": true, "schema": { "type": "string" } } + ], + "post": { + "summary": "Complete a resource upload", + "responses": { + "201": { + "description": "Completed resource", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { "resource": { "$ref": "#/components/schemas/Resource" } }, + "required": ["resource"] + } + } + } + } + } + } + }, + "/api/v1/resource-uploads/{uploadId}": { + "parameters": [ + { "name": "uploadId", "in": "path", "required": true, "schema": { "type": "string" } } + ], + "delete": { + "summary": "Abort a resource upload", + "responses": { + "200": { "description": "Upload aborted" } + } + } + }, "/api/v1/exports/markdown": { "get": { "summary": "Read a page of Markdown export data", @@ -1465,13 +1562,44 @@ "responses": { "200": { "$ref": "#/components/responses/Ok" } } } }, + "/api/v1/restores/json/resources/{id}/uploads": { + "parameters": [ + { "name": "id", "in": "path", "required": true, "schema": { "type": "string" } } + ], + "post": { + "summary": "Initialize a resumable resource restore", + "description": "Interactive-user-only restore session that preserves the resource ID from an EdgeEver ZIP. The client streams decompressed bytes through the shared 8 MiB multipart endpoints, supporting attachments up to 1 GiB without buffering the resource or archive in memory.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { "type": "object", "additionalProperties": true } + } + } + }, + "responses": { + "201": { + "description": "Initialized restore upload", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { "upload": { "$ref": "#/components/schemas/ResourceUpload" } }, + "required": ["upload"] + } + } + } + } + } + } + }, "/api/v1/restores/json/resources/{id}": { "parameters": [ { "name": "id", "in": "path", "required": true, "schema": { "type": "string" } } ], "put": { - "summary": "Import one resource from an EdgeEver ZIP", - "description": "Interactive-user-only resource import. The multipart metadata field contains the structured EdgeEver ZIP resource record.", + "summary": "Import one small resource from an EdgeEver ZIP (legacy)", + "description": "Interactive-user-only compatibility endpoint for resources up to 100 MiB. Larger restores use the resumable restore session so request parsing remains memory-bounded.", "requestBody": { "required": true, "content": { @@ -2619,6 +2747,18 @@ }, "required": ["id", "memoId", "originalMemoId", "kind", "mimeType", "filename", "byteSize", "sha256", "width", "height", "createdAt", "updatedAt", "url"] }, + "ResourceUpload": { + "type": "object", + "properties": { + "id": { "type": "string" }, + "resourceId": { "type": "string" }, + "partSize": { "type": "integer", "const": 8388608 }, + "partCount": { "type": "integer", "minimum": 1, "maximum": 50 }, + "byteSize": { "type": "integer", "minimum": 1, "maximum": 1073741824 }, + "expiresAt": { "type": "string", "format": "date-time" } + }, + "required": ["id", "resourceId", "partSize", "partCount", "byteSize", "expiresAt"] + }, "ResourceListItem": { "allOf": [ { diff --git a/migrations/0036_resource_multipart_uploads.sql b/migrations/0036_resource_multipart_uploads.sql new file mode 100644 index 00000000..19b997d6 --- /dev/null +++ b/migrations/0036_resource_multipart_uploads.sql @@ -0,0 +1,35 @@ +CREATE TABLE resource_uploads ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + memo_id TEXT NOT NULL, + resource_id TEXT NOT NULL UNIQUE, + storage_config_id TEXT NOT NULL, + bucket_name TEXT NOT NULL, + object_key TEXT NOT NULL, + provider_upload_id TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('image', 'attachment')), + mime_type TEXT NOT NULL, + filename TEXT NOT NULL, + byte_size INTEGER NOT NULL CHECK (byte_size > 0), + part_size INTEGER NOT NULL CHECK (part_size > 0), + part_count INTEGER NOT NULL CHECK (part_count > 0), + restore_metadata_json TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + expires_at TEXT NOT NULL, + FOREIGN KEY (workspace_id) REFERENCES workspaces(id) ON DELETE CASCADE, + FOREIGN KEY (memo_id) REFERENCES memos(id) ON DELETE CASCADE +); + +CREATE INDEX idx_resource_uploads_workspace + ON resource_uploads(workspace_id, expires_at); + +CREATE TABLE resource_upload_parts ( + upload_id TEXT NOT NULL, + part_number INTEGER NOT NULL CHECK (part_number > 0), + etag TEXT NOT NULL, + byte_size INTEGER NOT NULL CHECK (byte_size > 0), + created_at TEXT NOT NULL, + PRIMARY KEY (upload_id, part_number), + FOREIGN KEY (upload_id) REFERENCES resource_uploads(id) ON DELETE CASCADE +); diff --git a/packages/client/src/index.test.mjs b/packages/client/src/index.test.mjs index c5c063b9..953a3bf9 100644 --- a/packages/client/src/index.test.mjs +++ b/packages/client/src/index.test.mjs @@ -106,12 +106,52 @@ describe("EdgeEver client HTTP contract", () => { expect(call.headers.get("Content-Type")).toBeNull(); }); - test("keeps multipart restore bodies free of a synthetic JSON content type", async () => { - let headers; + test("exposes an authenticated resource response without forcing Blob buffering", async () => { + let authorization; const client = createEdgeEverClient({ + baseUrl: "https://notes.example", + token: "secret", fetch: async (_input, init) => { - headers = new Headers(init.headers); - return jsonResponse({ ok: true }); + authorization = new Headers(init.headers).get("Authorization"); + return new Response(new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array([1, 2])); + controller.enqueue(new Uint8Array([3])); + controller.close(); + }, + }), { headers: { "Content-Length": "3" } }); + }, + }); + + const response = await client.getResourceResponse("/api/v1/resources/res/blob"); + expect(authorization).toBe("Bearer secret"); + expect(response).toBeInstanceOf(Response); + expect(response.headers.get("Content-Length")).toBe("3"); + expect(Array.from(new Uint8Array(await response.arrayBuffer()))).toEqual([1, 2, 3]); + }); + + test("streams restored resources through bounded multipart requests", async () => { + const calls = []; + const client = createEdgeEverClient({ + fetch: async (input, init) => { + const path = String(input); + calls.push({ path, headers: new Headers(init.headers), body: init.body }); + if (path.endsWith("/uploads")) { + return jsonResponse({ + upload: { + id: "restore_upload", + resourceId: "resource/id", + partSize: 4, + partCount: 2, + byteSize: 5, + expiresAt: "2026-09-01T00:00:00.000Z", + }, + }, { status: 201 }); + } + if (path.includes("/parts/")) { + return jsonResponse({ part: { partNumber: Number(path.split("/").at(-1)), byteSize: init.body.size } }); + } + return jsonResponse({ resource: { id: "resource/id" } }, { status: 201 }); }, }); @@ -123,14 +163,113 @@ describe("EdgeEver client HTTP contract", () => { filename: "file.bin", mimeType: "application/octet-stream", kind: "attachment", - size: 1, - url: "/blob", - createdAt: "", - updatedAt: "", + byteSize: 5, + sha256: null, + width: null, + height: null, + originalMemoId: null, + archivePath: "resources/file.bin", + createdAt: "2026-08-31T00:00:00.000Z", + updatedAt: "2026-08-31T00:00:00.000Z", }, - new Blob(["x"]), + new Blob(["abcde"]), ); - expect(headers.has("Content-Type")).toBe(false); + expect(calls.map((call) => call.path)).toEqual([ + "/api/v1/restores/json/resources/resource%2Fid/uploads", + "/api/v1/resource-uploads/restore_upload/parts/1", + "/api/v1/resource-uploads/restore_upload/parts/2", + "/api/v1/resource-uploads/restore_upload/complete", + ]); + expect(await calls[1].body.text()).toBe("abcd"); + expect(await calls[2].body.text()).toBe("e"); + expect(calls[1].headers.get("Content-Type")).toBe("application/octet-stream"); + }); + + test("uploads memo resources as ordered resumable parts", async () => { + const calls = []; + const file = new File(["abcdefghij"], "archive.bin", { type: "application/octet-stream" }); + const client = createEdgeEverClient({ + fetch: async (input, init) => { + const path = String(input); + calls.push({ path, method: init.method, body: init.body }); + if (path.endsWith("/resource-uploads")) { + expect(JSON.parse(init.body)).toEqual({ + filename: "archive.bin", + mimeType: "application/octet-stream", + byteSize: 10, + }); + return jsonResponse({ + upload: { + id: "upload_1", + resourceId: "res_1", + partSize: 4, + partCount: 3, + byteSize: 10, + expiresAt: "2026-09-01T00:00:00.000Z", + }, + }, { status: 201 }); + } + if (path.includes("/parts/")) { + const partNumber = Number(path.split("/").at(-1)); + return jsonResponse({ part: { partNumber, byteSize: init.body.size } }); + } + return jsonResponse({ resource: { id: "res_1", url: "/api/v1/resources/res_1/blob" } }, { status: 201 }); + }, + }); + + const result = await client.uploadMemoResource("memo/1", file); + + expect(result.resource.id).toBe("res_1"); + expect(calls.map(({ path }) => path)).toEqual([ + "/api/v1/memos/memo%2F1/resource-uploads", + "/api/v1/resource-uploads/upload_1/parts/1", + "/api/v1/resource-uploads/upload_1/parts/2", + "/api/v1/resource-uploads/upload_1/parts/3", + "/api/v1/resource-uploads/upload_1/complete", + ]); + expect(await calls[1].body.text()).toBe("abcd"); + expect(await calls[2].body.text()).toBe("efgh"); + expect(await calls[3].body.text()).toBe("ij"); + }); + + test("uploads a file-backed source without requesting more than one part at a time", async () => { + const ranges = []; + const bodies = []; + const client = createEdgeEverClient({ + fetch: async (input, init) => { + const path = String(input); + if (path.endsWith("/resource-uploads")) { + return jsonResponse({ + upload: { + id: "upload_staged", + resourceId: "res_staged", + partSize: 4, + partCount: 3, + byteSize: 9, + expiresAt: "2026-09-01T00:00:00.000Z", + }, + }, { status: 201 }); + } + if (path.includes("/parts/")) { + bodies.push(await init.body.text()); + return jsonResponse({ part: { partNumber: Number(path.split("/").at(-1)), byteSize: init.body.size } }); + } + return jsonResponse({ resource: { id: "res_staged" } }, { status: 201 }); + }, + }); + + await client.uploadMemoResourceParts("memo_1", { + filename: "offline.bin", + mimeType: "application/octet-stream", + byteSize: 9, + readPart: async (start, end) => { + ranges.push([start, end]); + return new Blob(["abcdefghi".slice(start, end)]); + }, + }); + + expect(ranges).toEqual([[0, 4], [4, 8], [8, 9]]); + expect(bodies).toEqual(["abcd", "efgh", "i"]); }); test("sends unsaved note content to the AI tag suggestion endpoint", async () => { diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index 77bd0fdc..4c84782b 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -53,6 +53,13 @@ export type EdgeEverClientOptions = { onUnauthorized?: (context: EdgeEverClientRequestContext) => void | Promise; }; +export type MultipartResourceUploadSource = { + filename: string; + mimeType: string; + byteSize: number; + readPart: (start: number, end: number) => Promise; +}; + export type InstanceHealth = { ok: true; name: string; @@ -149,6 +156,17 @@ export type ResourceResponse = { resource: Resource; }; +export type ResourceUploadResponse = { + upload: { + id: string; + resourceId: string; + partSize: number; + partCount: number; + byteSize: number; + expiresAt: string; + }; +}; + export type MarkdownExportPage = { memos: MemoDetail[]; resources: Resource[]; @@ -302,22 +320,182 @@ export const createEdgeEverClient = (options: EdgeEverClientOptions = {}) => { return response.json() as Promise; }; - const requestBlob = async (path: string) => { + const requestResourceResponse = async (path: string, init?: RequestInit) => { const isAbsolute = isAbsoluteHttpUrl(path); - const { context, response } = await send(path, undefined, { + const { context, response } = await send(path, init, { attachToken: !isAbsolute, setJsonContentType: false, }); if (!response.ok) await throwRequestError(context, response, "Resource download failed"); - return response.blob(); + return response; }; + const requestBlob = async (path: string) => (await requestResourceResponse(path)).blob(); + const requestArrayBuffer = async (path: string) => { const { context, response } = await send(path, undefined, { setJsonContentType: false }); if (!response.ok) await throwRequestError(context, response, "Binary download failed"); return response.arrayBuffer(); }; + const uploadMemoResourceParts = async (memoId: string, source: MultipartResourceUploadSource) => { + const filename = source.filename.trim() || "attachment"; + const mimeType = source.mimeType || "application/octet-stream"; + const { upload } = await request( + `/api/v1/memos/${encodeURIComponent(memoId)}/resource-uploads`, + { + method: "POST", + body: JSON.stringify({ filename, mimeType, byteSize: source.byteSize }), + }, + ); + + try { + for (let partNumber = 1; partNumber <= upload.partCount; partNumber += 1) { + const start = (partNumber - 1) * upload.partSize; + const chunk = await source.readPart(start, Math.min(start + upload.partSize, source.byteSize)); + let attempt = 0; + while (true) { + try { + await request<{ part: { partNumber: number; byteSize: number } }>( + `/api/v1/resource-uploads/${encodeURIComponent(upload.id)}/parts/${partNumber}`, + { + method: "PUT", + headers: { "Content-Type": "application/octet-stream" }, + body: chunk, + }, + ); + break; + } catch (error) { + attempt += 1; + const retryable = !(error instanceof ApiRequestError) + || error.status === 408 + || error.status === 429 + || error.status >= 500; + if (!retryable || attempt >= 3) throw error; + await new Promise((resolve) => setTimeout(resolve, attempt * 250)); + } + } + } + return await request( + `/api/v1/resource-uploads/${encodeURIComponent(upload.id)}/complete`, + { method: "POST", body: JSON.stringify({}) }, + ); + } catch (error) { + await request<{ ok: true }>(`/api/v1/resource-uploads/${encodeURIComponent(upload.id)}`, { + method: "DELETE", + }).catch(() => undefined); + throw error; + } + }; + + const uploadMemoResourceMultipart = (memoId: string, file: Blob) => + uploadMemoResourceParts(memoId, { + filename: "name" in file && typeof file.name === "string" && file.name.trim() + ? file.name + : "attachment", + mimeType: file.type || "application/octet-stream", + byteSize: file.size, + readPart: async (start, end) => file.slice(start, end), + }); + + const createResourceMultipartSink = (upload: ResourceUploadResponse["upload"]) => { + let partNumber = 1; + let bufferedBytes = 0; + let receivedBytes = 0; + let chunks: ArrayBuffer[] = []; + let closed = false; + + const abort = async () => { + if (closed) return; + closed = true; + chunks = []; + bufferedBytes = 0; + await request<{ ok: true }>(`/api/v1/resource-uploads/${encodeURIComponent(upload.id)}`, { + method: "DELETE", + }).catch(() => undefined); + }; + + const sendPart = async () => { + if (bufferedBytes === 0) return; + const body = new Blob(chunks, { type: "application/octet-stream" }); + const currentPart = partNumber; + chunks = []; + bufferedBytes = 0; + let attempt = 0; + while (true) { + try { + await request<{ part: { partNumber: number; byteSize: number } }>( + `/api/v1/resource-uploads/${encodeURIComponent(upload.id)}/parts/${currentPart}`, + { + method: "PUT", + headers: { "Content-Type": "application/octet-stream" }, + body, + }, + ); + partNumber += 1; + return; + } catch (error) { + attempt += 1; + const retryable = !(error instanceof ApiRequestError) + || error.status === 408 + || error.status === 429 + || error.status >= 500; + if (!retryable || attempt >= 3) throw error; + await new Promise((resolve) => setTimeout(resolve, attempt * 250)); + } + } + }; + + return { + write: async (chunk: Uint8Array) => { + if (closed) throw new Error("Resource restore upload is closed"); + let offset = 0; + while (offset < chunk.byteLength) { + const available = upload.partSize - bufferedBytes; + const length = Math.min(available, chunk.byteLength - offset); + const copy = new Uint8Array(length); + copy.set(chunk.subarray(offset, offset + length)); + chunks.push(copy.buffer); + bufferedBytes += length; + receivedBytes += length; + offset += length; + if (bufferedBytes === upload.partSize) await sendPart(); + } + }, + close: async () => { + if (closed) throw new Error("Resource restore upload is closed"); + if (receivedBytes !== upload.byteSize) { + await abort(); + throw new Error(`Resource restore size mismatch: expected ${upload.byteSize}, received ${receivedBytes}`); + } + try { + await sendPart(); + const result = await request( + `/api/v1/resource-uploads/${encodeURIComponent(upload.id)}/complete`, + { method: "POST", body: JSON.stringify({}) }, + ); + closed = true; + return result; + } catch (error) { + await abort(); + throw error; + } + }, + abort, + }; + }; + + const createJsonResourceRestoreSink = async ( + resourceId: string, + metadata: JsonBackupMemo["resources"][number], + ) => { + const { upload } = await request( + `/api/v1/restores/json/resources/${encodeURIComponent(resourceId)}/uploads`, + { method: "POST", body: JSON.stringify(metadata) }, + ); + return createResourceMultipartSink(upload); + }; + return { getInstanceHealth: () => request("/api/health"), @@ -807,16 +985,30 @@ export const createEdgeEverClient = (options: EdgeEverClientOptions = {}) => { body: JSON.stringify({ prompts }), }), - restoreJsonResource: (resourceId: string, metadata: JsonBackupMemo["resources"][number], file: Blob) => { - const form = new FormData(); - form.append("metadata", JSON.stringify(metadata)); - form.append("file", file, metadata.filename || metadata.id); - return request<{ ok: true }>(`/api/v1/restores/json/resources/${encodeURIComponent(resourceId)}`, { - method: "PUT", - body: form, - }); + createJsonResourceRestoreSink, + + restoreJsonResource: async (resourceId: string, metadata: JsonBackupMemo["resources"][number], file: Blob) => { + const sink = await createJsonResourceRestoreSink(resourceId, metadata); + const reader = file.stream().getReader(); + try { + while (true) { + const next = await reader.read(); + if (next.done) break; + await sink.write(next.value); + } + await sink.close(); + return { ok: true as const }; + } catch (error) { + await sink.abort(); + throw error; + } finally { + reader.releaseLock(); + } }, + getResourceResponse: (resourceUrl: string, init?: RequestInit) => + requestResourceResponse(resourceUrl, init), + getResourceBlob: (resourceUrl: string) => requestBlob(resourceUrl), downloadGithubPluginAsset: ( @@ -829,6 +1021,9 @@ export const createEdgeEverClient = (options: EdgeEverClientOptions = {}) => { ), uploadMemoResource: (memoId: string, file: Blob | FormData) => { + if (!(file instanceof FormData)) { + return uploadMemoResourceMultipart(memoId, file); + } const form = file instanceof FormData ? file : new FormData(); if (!(file instanceof FormData)) form.append("file", file); return request(`/api/v1/memos/${memoId}/resources`, { @@ -837,6 +1032,8 @@ export const createEdgeEverClient = (options: EdgeEverClientOptions = {}) => { }); }, + uploadMemoResourceParts, + updateMemo: ( memoId: string, payload: { diff --git a/packages/shared/src/i18n/en-US.ts b/packages/shared/src/i18n/en-US.ts index 7be428f7..d0a349f4 100644 --- a/packages/shared/src/i18n/en-US.ts +++ b/packages/shared/src/i18n/en-US.ts @@ -738,8 +738,11 @@ export const enUS = { working: "Preparing notes and attachments…", importing: "Importing notes and attachments…", progress: "{{completed}} / {{total}}", + scanning: "Checking ZIP structure…", + scanProgress: "{{percentage}}%", complete: "Export complete. Your ZIP download has started.", error: "Export failed. Check your connection and try again.", + largeBackupRequiresStreaming: "This backup is too large for an in-memory export. Use Chrome, Edge, or the desktop app to save it as a stream.", desktopLocalTitle: "Desktop local backups", desktopLocalDescription: "The sidecar keeps the five most recent backups in the local SQLite data directory. A protective backup is created before every restore.", desktopLocalBackup: "Back up now", @@ -771,6 +774,7 @@ export const enUS = { webdavComplete: "Backup uploaded: {{filename}}", webdavErrors: { network: "Could not connect to the WebDAV service. Check the URL, CORS, and network connection.", + largeBackupRequiresStreaming: "This backup is too large for this browser. Use Chrome, Edge, or the desktop app for disk-backed WebDAV backup.", unknown: "The WebDAV operation failed. Check your settings and try again.", }, importComplete: "EdgeEver ZIP imported successfully.", @@ -1053,6 +1057,7 @@ export const enUS = { pdfViewer: { loading: "Loading PDF…", unavailable: "This PDF cannot be previewed. You can still download it or open it externally.", + previewTooLarge: "Preview disabled over 50 MiB", fullscreenLabel: "PDF viewer: {{filename}}", zoomIn: "Zoom in", zoomOut: "Zoom out", diff --git a/packages/shared/src/i18n/zh-CN.ts b/packages/shared/src/i18n/zh-CN.ts index 890e1235..37d4561e 100644 --- a/packages/shared/src/i18n/zh-CN.ts +++ b/packages/shared/src/i18n/zh-CN.ts @@ -738,8 +738,11 @@ export const zhCN = { working: "正在整理笔记和附件…", importing: "正在导入笔记和附件…", progress: "{{completed}} / {{total}}", + scanning: "正在检查 ZIP 结构…", + scanProgress: "{{percentage}}%", complete: "导出完成,ZIP 文件已开始下载。", error: "导出失败,请检查网络连接后重试。", + largeBackupRequiresStreaming: "备份体积过大,不能在内存中生成。请使用 Chrome、Edge 或桌面应用流式保存。", desktopLocalTitle: "桌面端本地备份", desktopLocalDescription: "sidecar 会在本机 SQLite 数据目录中保留最近 5 份备份。恢复前会自动创建保护性备份。", desktopLocalBackup: "立即备份", @@ -771,6 +774,7 @@ export const zhCN = { webdavComplete: "备份已上传:{{filename}}", webdavErrors: { network: "无法连接 WebDAV 服务。请检查地址、CORS 和网络连接。", + largeBackupRequiresStreaming: "备份体积超过当前浏览器的安全内存范围,请使用 Chrome、Edge 或桌面应用进行磁盘暂存备份。", unknown: "WebDAV 操作失败,请检查配置后重试。", }, importComplete: "EdgeEver ZIP 导入完成。", @@ -1053,6 +1057,7 @@ export const zhCN = { pdfViewer: { loading: "正在加载 PDF…", unavailable: "无法预览此 PDF,你仍可下载或在外部打开。", + previewTooLarge: "超过 50 MiB,不自动预览", fullscreenLabel: "PDF 查看器:{{filename}}", zoomIn: "放大", zoomOut: "缩小", diff --git a/tests/e2e/large-zip-streaming-restore.spec.ts b/tests/e2e/large-zip-streaming-restore.spec.ts new file mode 100644 index 00000000..712405fe --- /dev/null +++ b/tests/e2e/large-zip-streaming-restore.spec.ts @@ -0,0 +1,221 @@ +import { expect, test, type APIRequestContext } from "@playwright/test"; +import { createWriteStream } from "node:fs"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { once } from "node:events"; +import { Zip, ZipDeflate, ZipPassThrough, strToU8 } from "fflate"; + +const E2E_USERNAME = process.env.EDGE_EVER_E2E_USERNAME || "admin"; +const E2E_PASSWORD = process.env.EDGE_EVER_E2E_PASSWORD || "admin123"; +const MIB = 1024 * 1024; +const RESOURCE_SPECS = [ + { suffix: "a", byteSize: 600 * MIB + 3, fill: 0x5a, tail: new Uint8Array([0x11, 0x22, 0x33]) }, +] as const; + +const login = async (request: APIRequestContext) => { + const response = await request.post("/api/v1/auth/login", { + data: { username: E2E_USERNAME, password: E2E_PASSWORD }, + }); + expect(response.ok(), `login failed: ${response.status()} ${await response.text()}`).toBe(true); +}; + +const addJsonFile = (zip: Zip, path: string, value: unknown) => { + const entry = new ZipDeflate(path, { level: 6 }); + zip.add(entry); + entry.push(strToU8(`${JSON.stringify(value)}\n`), true); +}; + +const createLargeBackup = async () => { + const directory = await mkdtemp(join(tmpdir(), "edgeever-large-zip-e2e-")); + const archivePath = join(directory, "edgeever-600-mib-attachment.zip"); + const output = createWriteStream(archivePath, { highWaterMark: MIB }); + let needsDrain = false; + const finished = once(output, "finish"); + const zip = new Zip((error, data, final) => { + if (error) { + output.destroy(error); + return; + } + if (data.byteLength > 0 && !output.write(data)) needsDrain = true; + if (final) output.end(); + }); + const drain = async () => { + if (!needsDrain) return; + needsDrain = false; + await once(output, "drain"); + }; + + const marker = `${Date.now()}-${Math.random().toString(16).slice(2)}`; + const notebookId = `nb_e2e_large_zip_${marker}`; + const memoId = `memo_e2e_large_zip_${marker}`; + const title = `E2E 超大 ZIP 恢复 ${marker}`; + const timestamp = "2026-08-31T00:00:00.000Z"; + const resources = RESOURCE_SPECS.map((spec) => ({ + id: `res_e2e_large_zip_${spec.suffix}_${marker}`, + memoId, + originalMemoId: null, + kind: "attachment" as const, + mimeType: "application/octet-stream", + filename: `large-${spec.suffix}.bin`, + byteSize: spec.byteSize, + sha256: null, + width: null, + height: null, + createdAt: timestamp, + updatedAt: timestamp, + archivePath: `notes/E2E/${title}.assets/large-${spec.suffix}.bin`, + })); + const memo = { + id: memoId, + notebookId, + title, + excerpt: "Streaming restore fixture", + tags: ["e2e-large-zip"], + isPinned: false, + isArchived: false, + isDeleted: false, + revision: 1, + createdAt: timestamp, + updatedAt: timestamp, + deletedAt: null, + contentJson: { + type: "doc", + content: [{ type: "paragraph", content: [{ type: "text", text: "Streaming restore fixture" }] }], + }, + contentMarkdown: resources + .map((resource) => `[${resource.filename}](/api/v1/resources/${resource.id}/blob)`) + .join("\n\n"), + contentText: "Streaming restore fixture", + contentHash: `e2e-large-zip-${marker}`, + sourceMemoIds: [], + mergeSourceCount: 0, + mergedIntoMemoId: null, + }; + + try { + addJsonFile(zip, "notebooks.json", [{ + id: notebookId, + parentId: null, + name: `E2E Large ZIP ${marker}`, + slug: null, + icon: null, + color: null, + sortOrder: 0, + createdAt: timestamp, + updatedAt: timestamp, + }]); + addJsonFile(zip, "prompts.json", []); + await drain(); + + for (const [index, resource] of resources.entries()) { + const spec = RESOURCE_SPECS[index]; + const entry = new ZipPassThrough(resource.archivePath); + zip.add(entry); + const fullChunk = new Uint8Array(MIB); + fullChunk.fill(spec.fill); + const fullChunkCount = Math.floor(spec.byteSize / MIB); + for (let chunkIndex = 0; chunkIndex < fullChunkCount; chunkIndex += 1) { + entry.push(fullChunk, false); + await drain(); + } + const remaining = spec.byteSize - fullChunkCount * MIB; + if (remaining > 0) { + const finalChunk = new Uint8Array(remaining); + finalChunk.fill(spec.fill); + if (spec.tail.byteLength > 0) finalChunk.set(spec.tail, finalChunk.byteLength - spec.tail.byteLength); + entry.push(finalChunk, true); + } else { + entry.push(new Uint8Array(), true); + } + await drain(); + } + + addJsonFile(zip, `memos/${memoId}.json`, { memo, revisions: [], resources }); + const markdown = new ZipDeflate(`notes/E2E/${title}.md`, { level: 6 }); + zip.add(markdown); + markdown.push(strToU8(`# ${title}\n\n${memo.contentMarkdown}\n`), true); + addJsonFile(zip, "manifest.json", { + format: "edgeever-zip", + formatVersion: 2, + schemaVersion: 1, + edgeeverVersion: "e2e", + buildId: `e2e-${marker}`, + exportedAt: timestamp, + includesTrash: false, + counts: { notebooks: 1, memos: 1, revisions: 0, resources: resources.length, prompts: 0 }, + }); + zip.end(); + await finished; + return { archivePath, directory, memoId, notebookId, resources, title }; + } catch (error) { + output.destroy(); + await rm(directory, { recursive: true, force: true }); + throw error; + } +}; + +test("imports a 600 MiB attachment from an EdgeEver ZIP through streaming multipart restore", async ({ page }) => { + test.setTimeout(300_000); + const fixture = await createLargeBackup(); + const restoredPartNumbers: number[] = []; + + try { + await login(page.request); + page.on("response", (response) => { + const match = new URL(response.url()).pathname.match(/^\/api\/v1\/resource-uploads\/[^/]+\/parts\/(\d+)$/); + if (match && response.request().method() === "PUT" && response.ok()) { + restoredPartNumbers.push(Number(match[1])); + } + }); + + await page.goto("/"); + await page.getByRole("button", { name: "个人中心", exact: true }).click(); + await expect(page.getByRole("heading", { name: "我的", exact: true })).toBeVisible(); + await page.getByRole("button", { name: "导入导出", exact: true }).click(); + await expect(page.getByText("导入与导出", { exact: true })).toBeVisible(); + + const chooserPromise = page.waitForEvent("filechooser"); + await page.getByRole("button", { name: "导入 EdgeEver ZIP", exact: true }).click(); + const chooser = await chooserPromise; + await chooser.setFiles(fixture.archivePath); + + const confirmation = page.getByRole("dialog", { name: "导入 EdgeEver ZIP" }); + await expect(confirmation).toBeVisible({ timeout: 120_000 }); + await expect(confirmation).toContainText("1 篇笔记和 1 个资源"); + await confirmation.getByRole("button", { name: "确认导入", exact: true }).click(); + await expect(page.getByText("EdgeEver ZIP 导入完成。", { exact: true })).toBeVisible({ timeout: 180_000 }); + + expect(restoredPartNumbers).toHaveLength(76); + expect(restoredPartNumbers.filter((part) => part === 1)).toHaveLength(1); + expect(Math.max(...restoredPartNumbers)).toBe(76); + + const memoResponse = await page.request.get(`/api/v1/memos/${fixture.memoId}`); + expect(memoResponse.ok(), await memoResponse.text()).toBe(true); + const restoredMemo = await memoResponse.json() as { memo: { title: string; contentMarkdown: string } }; + expect(restoredMemo.memo.title).toBe(fixture.title); + + for (const [index, resource] of fixture.resources.entries()) { + const first = await page.request.get(`/api/v1/resources/${resource.id}/blob`, { + headers: { Range: "bytes=0-2" }, + }); + expect(first.status()).toBe(206); + expect(Array.from(await first.body())).toEqual([RESOURCE_SPECS[index].fill, RESOURCE_SPECS[index].fill, RESOURCE_SPECS[index].fill]); + + const last = await page.request.get(`/api/v1/resources/${resource.id}/blob`, { + headers: { Range: `bytes=${resource.byteSize - 3}-${resource.byteSize - 1}` }, + }); + expect(last.status()).toBe(206); + const expectedTail = RESOURCE_SPECS[index].tail.byteLength > 0 + ? Array.from(RESOURCE_SPECS[index].tail) + : [RESOURCE_SPECS[index].fill, RESOURCE_SPECS[index].fill, RESOURCE_SPECS[index].fill]; + expect(Array.from(await last.body())).toEqual(expectedTail); + } + } finally { + await login(page.request).catch(() => undefined); + await page.request.delete(`/api/v1/memos/${fixture.memoId}`).catch(() => undefined); + await page.request.delete(`/api/v1/memos/${fixture.memoId}?permanent=1`).catch(() => undefined); + await page.request.delete(`/api/v1/notebooks/${fixture.notebookId}`).catch(() => undefined); + await rm(fixture.directory, { recursive: true, force: true }); + } +}); diff --git a/tests/json-backup.test.ts b/tests/json-backup.test.ts index 35803cbc..075a9e53 100644 --- a/tests/json-backup.test.ts +++ b/tests/json-backup.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from "bun:test"; import type { AiPromptTemplate, JsonBackupMemo, JsonBackupNotebook, MemoDetail, Notebook, Resource } from "@edgeever/shared"; -import { strFromU8, strToU8, zipSync } from "fflate"; +import { strFromU8, strToU8, unzipSync, zipSync } from "fflate"; import { createEdgeEverZip, parseEdgeEverZip, @@ -107,6 +107,7 @@ describe("EdgeEver ZIP", () => { }); test("combines readable Markdown with versioned recovery data", async () => { + const resourceChunks: number[] = []; const blob = await createEdgeEverZip( { listNotebooks: async () => ({ notebooks: [notebook("nb_child", "Child", "nb_root"), notebook("nb_root", "Root")] }), @@ -130,7 +131,15 @@ describe("EdgeEver ZIP", () => { totalCount: 1, nextOffset: null, }), - getResourceBlob: async () => new Blob([new Uint8Array([1, 2, 3])], { type: "image/png" }), + getResourceResponse: async () => new Response(new ReadableStream({ + start(controller) { + for (const chunk of [new Uint8Array([1]), new Uint8Array([2, 3])]) { + resourceChunks.push(chunk.byteLength); + controller.enqueue(chunk); + } + controller.close(); + }, + }), { headers: { "Content-Type": "image/png" } }), }, { edgeeverVersion: "0.1.13", buildId: "test-build" } ); @@ -142,8 +151,11 @@ describe("EdgeEver ZIP", () => { expect(backup.prompts).toEqual([prompt]); expect(backup.notebooks.map((item) => item.id)).toEqual(["nb_root", "nb_child"]); expect(backup.memos[0].memo.contentMarkdown).toBe(memo.contentMarkdown); - expect(Array.from(backup.files[backup.memos[0].resources[0].archivePath])).toEqual([1, 2, 3]); - const markdown = strFromU8(backup.files["notes/Root/Child/Backup note.md"]); + expect(backup.archive).toBe(blob); + expect(resourceChunks).toEqual([1, 2]); + const files = unzipSync(new Uint8Array(await blob.arrayBuffer())); + expect(Array.from(files[backup.memos[0].resources[0].archivePath])).toEqual([1, 2, 3]); + const markdown = strFromU8(files["notes/Root/Child/Backup note.md"]); expect(markdown).toContain("edgeever_id: \"memo_backup\""); expect(markdown).toContain("![asset](Backup%20note.assets/asset.png)"); expect(backup.memos[0].resources[0].archivePath).toBe("notes/Root/Child/Backup note.assets/asset.png"); @@ -169,6 +181,40 @@ describe("EdgeEver ZIP", () => { expect(backup.prompts).toEqual([]); }); + test("inspects multi-gigabyte archives through their stream instead of buffering the ZIP", async () => { + const manifest = { + format: "edgeever-zip", + formatVersion: 1, + schemaVersion: 1, + edgeeverVersion: "1.13.3", + buildId: "large-stream-test", + exportedAt: "2026-08-31T00:00:00.000Z", + includesTrash: false, + counts: { notebooks: 0, memos: 0, revisions: 0, resources: 0 }, + }; + const actualArchive = new Blob([zipSync({ + "manifest.json": strToU8(JSON.stringify(manifest)), + "notebooks.json": strToU8("[]"), + })]); + let wholeArchiveBuffered = false; + const largeStreamingArchive = { + size: 2 * 1024 * 1024 * 1024, + type: "application/zip", + slice: actualArchive.slice.bind(actualArchive), + stream: actualArchive.stream.bind(actualArchive), + arrayBuffer: async () => { + wholeArchiveBuffered = true; + throw new Error("The whole archive must not be buffered"); + }, + } as Blob; + + const backup = await parseEdgeEverZip(largeStreamingArchive); + + expect(backup.manifest.buildId).toBe("large-stream-test"); + expect(backup.archive).toBe(largeStreamingArchive); + expect(wholeArchiveBuffered).toBe(false); + }); + test("restores notebooks before memos and binary resources", async () => { const calls: string[] = []; const { url: _url, ...resourceMetadata } = resource; @@ -201,15 +247,20 @@ describe("EdgeEver ZIP", () => { }] satisfies JsonBackupNotebook[], memos: [backupMemo], prompts: [prompt], - files: { "resources/res_backup/asset.png": new Uint8Array([1, 2, 3]) }, + archive: new Blob([zipSync({ "resources/res_backup/asset.png": new Uint8Array([1, 2, 3]) })]), }; await restoreEdgeEverZip(backup, { restoreNotebooks: async () => { calls.push("notebooks"); }, restoreMemos: async () => { calls.push("memos"); }, restorePrompts: async () => { calls.push("prompts"); }, - restoreResource: async (_id, _metadata, file) => { - calls.push(`resource:${file.size}`); + createResourceRestoreSink: async () => { + let byteSize = 0; + return { + write: async (chunk) => { byteSize += chunk.byteLength; }, + close: async () => { calls.push(`resource:${byteSize}`); }, + abort: async () => {}, + }; }, }); @@ -242,7 +293,7 @@ describe("EdgeEver ZIP", () => { }] satisfies JsonBackupNotebook[], memos: [{ memo, revisions: [], resources: [] }] satisfies JsonBackupMemo[], prompts: [], - files: {}, + archive: new Blob([zipSync({})]), }; await restoreEdgeEverZipAndRefresh( @@ -251,7 +302,11 @@ describe("EdgeEver ZIP", () => { restoreNotebooks: async () => { calls.push("notebooks"); }, restoreMemos: async () => { calls.push("memos"); }, restorePrompts: async () => { calls.push("prompts"); }, - restoreResource: async () => { calls.push("resource"); }, + createResourceRestoreSink: async () => ({ + write: async () => {}, + close: async () => { calls.push("resource"); }, + abort: async () => {}, + }), }, async () => { calls.push("refresh"); }, ); diff --git a/tests/storage-adapter.test.ts b/tests/storage-adapter.test.ts index 0a44107a..e634d300 100644 --- a/tests/storage-adapter.test.ts +++ b/tests/storage-adapter.test.ts @@ -87,6 +87,58 @@ describe("storage adapter", () => { } }); + test("streams full and ranged filesystem downloads in bounded chunks", async () => { + const directory = await mkdtemp(`${tmpdir()}/edgeever-storage-`); + const sqlite = { + query: () => ({ all: () => [], get: () => null, run: () => undefined }), + transaction: (callback: () => void) => () => callback(), + }; + const payload = new Uint8Array(2 * 1024 * 1024); + payload.fill(7); + + try { + const store = createSelfHostedStorageAdapter(sqlite, directory).resources; + await store.put("workspace/memo/large.bin", payload); + const object = await store.get("workspace/memo/large.bin"); + const reader = object!.body.getReader(); + const first = await reader.read(); + expect(first.done).toBe(false); + expect(first.value!.byteLength).toBeLessThan(payload.byteLength); + await reader.cancel(); + + const ranged = await store.get("workspace/memo/large.bin", { + range: { offset: 1_000_000, length: 100_000 }, + }); + expect(ranged?.range).toEqual({ offset: 1_000_000, length: 100_000 }); + expect((await new Response(ranged!.body).arrayBuffer()).byteLength).toBe(100_000); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); + + test("assembles filesystem multipart uploads without exposing staging parts", async () => { + const directory = await mkdtemp(`${tmpdir()}/edgeever-storage-`); + const sqlite = { + query: () => ({ all: () => [], get: () => null, run: () => undefined }), + transaction: (callback: () => void) => () => callback(), + }; + + try { + const store = createSelfHostedStorageAdapter(sqlite, directory).resources; + const upload = await store.createMultipartUpload("workspace/memo/archive.bin"); + const second = await upload.uploadPart(2, new Uint8Array([4, 5])); + const first = await upload.uploadPart(1, new Uint8Array([1, 2, 3])); + await upload.complete([first, second]); + + expect(await readFile(`${directory}/workspace/memo/archive.bin`)).toEqual( + new Uint8Array([1, 2, 3, 4, 5]), + ); + expect(await store.get(`.uploads/${upload.uploadId}/1.part`)).toBeNull(); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); + test("rejects attachment path traversal", async () => { const directory = await mkdtemp(`${tmpdir()}/edgeever-storage-`); const sqlite = { @@ -141,4 +193,36 @@ describe("storage adapter", () => { "DeleteObjectsCommand:", ]); }); + + test("maps multipart lifecycle operations to an S3-compatible client", async () => { + const commands: string[] = []; + const client = { + send: async (command: { constructor: { name: string }; input: { PartNumber?: number } }) => { + commands.push(`${command.constructor.name}:${command.input.PartNumber ?? ""}`); + if (command.constructor.name === "CreateMultipartUploadCommand") return { UploadId: "provider-upload" }; + if (command.constructor.name === "UploadPartCommand") return { ETag: `etag-${command.input.PartNumber}` }; + return {}; + }, + }; + const sqlite = { + query: () => ({ all: () => [], get: () => null, run: () => undefined }), + transaction: (callback: () => void) => () => callback(), + }; + const store = createS3CompatibleStorageAdapter( + sqlite, + { bucket: "edgeever", endpoint: "http://minio:9000" }, + client as never, + ).resources; + + const upload = await store.createMultipartUpload("memo/archive.bin"); + const part = await upload.uploadPart(1, new Uint8Array([1, 2, 3])); + await upload.complete([part]); + + expect(part).toEqual({ partNumber: 1, etag: "etag-1" }); + expect(commands).toEqual([ + "CreateMultipartUploadCommand:", + "UploadPartCommand:1", + "CompleteMultipartUploadCommand:", + ]); + }); });