From 9ebc0a1460b38c616d31ff7aa43f6e8f215b64bd Mon Sep 17 00:00:00 2001 From: lyonbot Date: Wed, 22 Jan 2025 19:24:48 +0800 Subject: [PATCH] refactor: new upload api format --- src/database.ts | 1 + src/index.ts | 186 +++++++++++++++++++++++++++++++++++++++--------- web/src/App.tsx | 9 ++- 3 files changed, 163 insertions(+), 33 deletions(-) diff --git a/src/database.ts b/src/database.ts index f34a4db..3d6d8e3 100644 --- a/src/database.ts +++ b/src/database.ts @@ -29,6 +29,7 @@ export interface RecordFileItem { size: number; path: string; thumbnail?: string; + type?: string; } function fromDB(data: any): UploadRecord { diff --git a/src/index.ts b/src/index.ts index 8b78e60..df26096 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,5 @@ import { Hono } from "hono"; +// import { parseMultipartRequest } from '@mjackson/multipart-parser'; import { createUploadRecord, deleteRecord, @@ -6,6 +7,7 @@ import { getUploadRecordBySlug, migrateTables, purgeRecordsBeforeId, + RecordFileItem, } from "./database"; import { H } from "hono/types"; import { createSeekableTarball } from "./stream-tarball"; @@ -45,45 +47,165 @@ app.get("/api/list", authWithPassword, async (c) => { // return c.json(r) }); -app.post("/api/upload", authWithPassword, async (c) => { +const timingMiddleware: H = async (c, next) => { + const start = Date.now(); + const resp = await next(); + console.log(`[${c.req.method}] ${c.req.url} ${Date.now() - start}ms`); + return resp; +}; + +app.post("/api/upload", timingMiddleware, authWithPassword, async (c) => { + const reqContentType = String(c.req.header('Content-Type')); const uploader = c.req.header("x-uploader") || "unknown"; - const body = await c.req.formData(); + const filePathPrefix = `cf_drop/${Date.now()}`; - const files = body.getAll("files").filter((file) => file instanceof File); - const thumbnails = body.getAll("thumbnails") as string[]; - const message = String(body.get("message") || ""); + let message = ''; + let fileInfos: RecordFileItem[] = []; - if (!files.length && !message) { - return c.json({ error: "No files or message" }); + const rollbacks: (() => void)[] = []; + const filePutPromises: Promise[] = []; + const errors: { error: Error, fileInfo: RecordFileItem }[] = []; + + try { + if (false && reqContentType.startsWith('multipart/form-data')) { + // stream upload -- kinda slower than native formData() parser + + /* + const fields = parseMultipartRequest(c.req.raw); + for await (const field of fields) { + switch (field.name) { + case 'message': + message = await field.text(); + break; + case 'fileInfos': + initializeFileInfos(JSON.parse(await field.text())); + break; + case 'files': { + const info = fileInfos[filePutPromises.length]; + if (info) { + let stopped = false; + let readLength = 0; + let reader = field.body.getReader(); + const stream = new ReadableStream({ + expectedLength: info.size, + cancel(reason) { + stopped = true; + reader.cancel(reason); + }, + async start(controller) { + while (!stopped) { + const { done, value } = await reader.read(); + if (done) { + controller.close(); + break; + } + + readLength += value.length; + const remaining = info.size - readLength; + if (remaining <= 0) { + controller.enqueue(remaining !== 0 ? value.subarray(0, remaining) : value); + controller.close(); + reader.cancel(); + stopped = true; + break; + } + controller.enqueue(value); + } + } + }) + queueNextFileToUpload(stream); + } + break; + } + } + } + */ + } else { + // native formData() parser + const body = await c.req.formData(); + message = String(body.get("message") || ""); + initializeFileInfos(JSON.parse(body.get("fileInfos")!.toString())); + body.getAll("files").forEach((file) => { + if (file instanceof File) queueNextFileToUpload(file); + }); + } + + if (!fileInfos.length && !message) { + return c.json({ error: "No files or message" }); + } + + // create record + await Promise.all(filePutPromises); + + if (filePutPromises.length !== fileInfos.length) { + throw new Error('Invalid fileInfos'); + } + + if (errors.length) { + rollbacks.forEach((fn) => fn()); + c.status(500); + return c.json({ + message: 'Failed to upload files', + errors: errors.map(it => ({ + error: String(it.error), + file: it.fileInfo.name, + })), + }); + } + + const record = await createUploadRecord(c.env.DB, { + uploader, + size: fileInfos.reduce((acc, file) => acc + file.size, 0), + files: fileInfos, + message, + }); + + return c.json({ record }); + } catch (err) { + rollbacks.forEach((fn) => fn()); + c.status(500); + return c.json({ + message: 'Failed to upload files', + errors: [{ error: String(err), file: '' }], + }); } // upload files to bucket - const filePathPrefix = `cf_drop/${Date.now()}`; - const uploadedFiles = await Promise.all( - files.map(async (file, index) => { - const fileName = file.name; - const filePath = `${filePathPrefix}/${fileName}`; - const r = await c.env.MY_BUCKET.put(filePath, file, { - httpMetadata: { contentType: file.type }, + function queueNextFileToUpload(body: ReadableStream | Blob) { + const index = filePutPromises.length; + const item = fileInfos[index]; + if (!item) throw new Error('Invalid index of file'); + + const { path, type } = item; + const rollback = () => c.env.MY_BUCKET.delete(path).catch(() => { }); + rollbacks.push(rollback); + + filePutPromises.push((async () => { + try { + const r = await c.env.MY_BUCKET.put(path, body, { + httpMetadata: { contentType: type }, + }) + item.size = r.size; + } catch (e) { + errors.push({ error: e as Error, fileInfo: item }); + } + })()); + } + + function initializeFileInfos(tmp: any) { + if (!Array.isArray(tmp)) throw new Error('Invalid fileInfos'); + + fileInfos.length = 0; + for (const file of tmp) { + fileInfos.push({ + name: String(file.name), + size: +file.size, + thumbnail: String(file.thumbnail || ''), + path: `${filePathPrefix}/${fileInfos.length}_${file.name}`, + type: String(file.type || 'application/octet-stream'), }); - return { - name: fileName, - path: filePath, - size: r.size, - thumbnail: thumbnails[index] || "", - }; - }) - ); - - // create record - const record = await createUploadRecord(c.env.DB, { - uploader, - size: uploadedFiles.reduce((acc, file) => acc + file.size, 0), - files: uploadedFiles, - message, - }); - - return c.json({ record }); + } + } }); app.get("/api/download/:slug/message", async (c) => { diff --git a/web/src/App.tsx b/web/src/App.tsx index cb1e346..9f4c41a 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -15,10 +15,17 @@ const App = () => { function startUpload() { return new Promise((resolve, reject) => { + const fileInfos = files.map(f => ({ + thumbnail: f.thumbnail, + size: f.blob.size, + type: f.blob.type, + name: f.name, + })) + const body = new FormData(); body.append('message', text); + body.append('fileInfos', JSON.stringify(fileInfos)); files.forEach((file) => body.append('files', file.blob)); - files.forEach((file) => body.append('thumbnails', file.thumbnail || '')); const xhr = new XMLHttpRequest(); xhr.open('POST', '/api/upload');