support for background tasks

packages/server/services/files/routes/upload/chunk/post.js

@@ -1,23 +1,31 @@
 import path from "path"
 import fs from "fs"
+import RemoteUpload from "@services/remoteUpload"
 
 import ChunkFileUpload from "@shared-classes/ChunkFileUpload"
 
-import RemoteUpload from "@services/remoteUpload"
-
 const availableProviders = ["b2", "standard"]
 
 export default {
 	useContext: ["cache", "limits"],
-    middlewares: [
-        "withAuthentication",
-    ],
+	middlewares: ["withAuthentication"],
 	fn: async (req, res) => {
-        const tmpPath = path.resolve(this.default.contexts.cache.constructor.cachePath, req.auth.session.user_id)
+		const uploadId = `${req.headers["uploader-file-id"]}_${Date.now()}`
+
+		const tmpPath = path.resolve(
+			this.default.contexts.cache.constructor.cachePath,
+			req.auth.session.user_id,
+		)
 
 		const limits = {
-            maxFileSize: parseInt(this.default.contexts.limits.maxFileSizeInMB) * 1024 * 1024,
-            maxChunkSize: parseInt(this.default.contexts.limits.maxChunkSizeInMB) * 1024 * 1024,
+			maxFileSize:
+				parseInt(this.default.contexts.limits.maxFileSizeInMB) *
+				1024 *
+				1024,
+			maxChunkSize:
+				parseInt(this.default.contexts.limits.maxChunkSizeInMB) *
+				1024 *
+				1024,
 			useCompression: true,
 			useProvider: "standard",
 		}
@@ -50,9 +58,11 @@ export default {
 			try {
 				build = await build()
 
-                const result = await RemoteUpload({
+				if (req.headers["transmux"] || limits.useCompression === true) {
+					// add a background task
+					const job = await global.queues.createJob("remote_upload", {
+						filePath: build.filePath,
 						parentDir: req.auth.session.user_id,
-                    source: build.filePath,
 						service: limits.useProvider,
 						useCompression: limits.useCompression,
 						transmux: req.headers["transmux"] ?? false,
@@ -60,17 +70,35 @@ export default {
 						cachePath: tmpPath,
 					})
 
-                await fs.promises.rm(tmpPath, { recursive: true, force: true }).catch(() => {
-                    return false
+					const sseChannelId = job.sseChannelId
+
+					return {
+						uploadId: uploadId,
+						sseChannelId: sseChannelId,
+						eventChannelURL: `https://${req.get("host")}/upload/sse_events/${sseChannelId}`,
+					}
+				} else {
+					const result = await RemoteUpload({
+						source: build.filePath,
+						parentDir: req.auth.session.user_id,
+						service: limits.useProvider,
+						useCompression: limits.useCompression,
+						cachePath: tmpPath,
 					})
 
 					return result
+				}
 			} catch (error) {
-                await fs.promises.rm(tmpPath, { recursive: true, force: true }).catch(() => {
+				await fs.promises
+					.rm(tmpPath, { recursive: true, force: true })
+					.catch(() => {
 						return false
 					})
 
-                throw new OperationError(error.code ?? 500, error.message ?? "Failed to upload file")
+				throw new OperationError(
+					error.code ?? 500,
+					error.message ?? "Failed to upload file",
+				)
 			}
 		}
 
@@ -78,5 +106,5 @@ export default {
 			ok: 1,
 			chunkNumber: req.headers["uploader-chunk-number"],
 		}
-    }
+	},
 }

packages/server/services/files/routes/upload/sse_events/[sse_channel_id]/get.js

@@ -0,0 +1,5 @@
+export default async (req, res) => {
+    const { sse_channel_id } = req.params
+
+    global.sse.connectToChannelStream(sse_channel_id, req, res)
+}