feat: support chunk split max than 25m file

Author: oustn

src/files/fileCreate.ts

@@ -74,7 +74,7 @@ export class FileCreate extends Endpoint {
     let duration: string = c.env.SHARE_DURATION
     let isEphemeral = false
     let isEncrypted = false
-    let objectId = ''
+    let objectId: string | Array<{ objectId: string }> = ''
     let hash = ''
     const contentType = c.req.header('Content-Type')
     if (
@@ -85,7 +85,7 @@ export class FileCreate extends Endpoint {
       const file = formData.get('file') as File
 
       const fileInfo = this.getFormDataField<null | {
-        objectId: string
+        objectId: string | Array<{ objectId: string }>
         name: string
         type?: string
         size: number
@@ -116,7 +116,10 @@ export class FileCreate extends Endpoint {
       size = blob.size
     }
 
-    if ((!data || data.byteLength === 0) && !objectId) {
+    if (
+      (!data || data.byteLength === 0) &&
+      (!objectId || (Array.isArray(objectId) && !objectId.length))
+    ) {
       return this.error('分享内容为空')
     }
 
@@ -128,15 +131,22 @@ export class FileCreate extends Endpoint {
     }
 
     const kv = this.getKV(c)
-    const key = objectId || createId()
+    const key = objectId && !Array.isArray(objectId) ? objectId : createId()
+    // 直接上传
     if (data && data.byteLength) {
       await kv.put(key, data)
       hash = await sha256(data)
-    } else {
+      // 单个
+    } else if (typeof objectId === 'string') {
       const cacheFile = await kv.get(objectId, 'stream')
       if (!cacheFile) {
         return this.error('分片上传的文件不存在')
       }
+      // 分片存储
+    } else if (Array.isArray(objectId) && objectId.length) {
+      await kv.put(key, 'chunks', {
+        metadata: objectId,
+      })
     }
 
     const db = this.getDB(c)

src/files/fileFetch.ts

@@ -50,9 +50,18 @@ export class FileFetch extends Endpoint {
     }
     const objectId = record.objectId
 
-    const file = await kv.get(objectId, 'arrayBuffer')
+    const {
+      value: file,
+      metadata,
+    }: {
+      value: null | ArrayBuffer
+      metadata: null | Array<{
+        objectId: string
+        chunkId: number
+      }>
+    } = await kv.getWithMetadata(objectId, 'arrayBuffer')
 
-    if (!file) {
+    if (!file && !metadata) {
       return new Response('Not found', {
         status: 404,
         headers: {
@@ -61,6 +70,31 @@ export class FileFetch extends Endpoint {
       })
     }
 
+    if (metadata) {
+      // 大于 25M 不考虑文件类型
+      const { readable, writable } = new TransformStream()
+      const writer = writable.getWriter()
+      ;(async () => {
+        for (let i = 0; i < metadata.length; i++) {
+          const chunk = await kv.get(metadata[i].objectId, 'arrayBuffer')
+          if (!chunk) {
+            await writer.close()
+            throw new Error('文件 Chunk 不完整')
+          }
+          await writer.write(new Uint8Array(chunk))
+        }
+        await writer.close()
+      })().then()
+
+      return new Response(readable, {
+        status: 200,
+        headers: {
+          'Content-Type': record.type ?? 'application/octet-stream',
+          'Content-Disposition': `attachment; filename="${record.filename}"`,
+        },
+      })
+    }
+
     const isText = record.type === 'plain/string'
 
     return new Response(file, {

src/scheduled.ts

@@ -2,7 +2,7 @@ import { drizzle } from 'drizzle-orm/d1'
 import { files } from '../data/schemas'
 import { lte } from 'drizzle-orm'
 
-export async function scheduled(event: ScheduledEvent, env: Env) {
+export async function scheduled(_event: ScheduledEvent, env: Env) {
   const db = drizzle(env.DB)
   const kv = env.file_drops
 
@@ -16,7 +16,19 @@ export async function scheduled(event: ScheduledEvent, env: Env) {
     })
 
   if (records.length) {
-    await Promise.all(records.map((d) => kv.delete(d.objectId)))
+    await Promise.all(
+      records.map(async (d) => {
+        const {
+          value: _,
+          metadata,
+        }: { value: unknown; metadata: Array<{ objectId: string }> | null } =
+          await kv.getWithMetadata(d.objectId, 'stream')
+        if (Array.isArray(metadata) && metadata.length) {
+          await Promise.all(metadata.map((d) => kv.delete(d.objectId)))
+        }
+        return kv.delete(d.objectId)
+      }),
+    )
   }
 
   console.log(`clear before ${now}`)

web/api/uploader.ts

@@ -55,8 +55,23 @@ export class Uploader {
         const chunk = file.slice(start, end)
         tasks.push(this.uploadWithChunk(chunk))
       }
-      const data = await Promise.all(tasks)
-      console.log(data)
+      const chunkInfo = (await Promise.all(tasks)).map((d, i) => ({
+        ...d,
+        chunkId: i,
+      }))
+      formData.delete('file') // 移除 file
+      formData.append(
+        'fileInfo',
+        JSON.stringify({
+          objectId: chunkInfo,
+          name: file.name,
+          type: file.type,
+          size: file.size,
+          sha: await this.getSHA(file),
+        }),
+      )
+      const { data } = await axios.put('/files', formData)
+      return data as ApiResponseType<FileUploadedType>
     }
     throw new Error('建议使用 R2')
   }