mirror of
https://github.com/ragestudio/comty.git
synced 2025-06-09 18:44:16 +00:00
use new ChunckedUpload api
This commit is contained in:
parent
9b79f87db8
commit
ea746d5f76
@ -1,4 +1,169 @@
|
||||
import Core from "evite/src/core"
|
||||
import EventBus from "evite/src/internals/eventBus"
|
||||
import SessionModel from "models/session"
|
||||
|
||||
class ChunkedUpload {
|
||||
constructor(params) {
|
||||
this.endpoint = params.endpoint
|
||||
this.file = params.file
|
||||
this.headers = params.headers || {}
|
||||
this.postParams = params.postParams
|
||||
this.chunkSize = params.chunkSize || 1000000
|
||||
this.retries = params.retries || 5
|
||||
this.delayBeforeRetry = params.delayBeforeRetry || 5
|
||||
|
||||
this.start = 0
|
||||
this.chunk = null
|
||||
this.chunkCount = 0
|
||||
this.totalChunks = Math.ceil(this.file.size / this.chunkSize)
|
||||
this.retriesCount = 0
|
||||
this.offline = false
|
||||
this.paused = false
|
||||
|
||||
this.headers["Authorization"] = SessionModel.token
|
||||
this.headers["uploader-original-name"] = encodeURIComponent(this.file.name)
|
||||
this.headers["uploader-file-id"] = this.uniqid(this.file)
|
||||
this.headers["uploader-chunks-total"] = this.totalChunks
|
||||
|
||||
this._reader = new FileReader()
|
||||
this.eventBus = new EventBus()
|
||||
|
||||
this.validateParams()
|
||||
this.sendChunks()
|
||||
|
||||
// restart sync when back online
|
||||
// trigger events when offline/back online
|
||||
window.addEventListener("online", () => {
|
||||
if (!this.offline) return
|
||||
|
||||
this.offline = false
|
||||
this.eventBus.emit("online")
|
||||
this.sendChunks()
|
||||
})
|
||||
|
||||
window.addEventListener("offline", () => {
|
||||
this.offline = true
|
||||
this.eventBus.emit("offline")
|
||||
})
|
||||
}
|
||||
|
||||
on(event, fn) {
|
||||
this.eventBus.on(event, fn)
|
||||
}
|
||||
|
||||
validateParams() {
|
||||
if (!this.endpoint || !this.endpoint.length) throw new TypeError("endpoint must be defined")
|
||||
if (this.file instanceof File === false) throw new TypeError("file must be a File object")
|
||||
if (this.headers && typeof this.headers !== "object") throw new TypeError("headers must be null or an object")
|
||||
if (this.postParams && typeof this.postParams !== "object") throw new TypeError("postParams must be null or an object")
|
||||
if (this.chunkSize && (typeof this.chunkSize !== "number" || this.chunkSize === 0)) throw new TypeError("chunkSize must be a positive number")
|
||||
if (this.retries && (typeof this.retries !== "number" || this.retries === 0)) throw new TypeError("retries must be a positive number")
|
||||
if (this.delayBeforeRetry && (typeof this.delayBeforeRetry !== "number")) throw new TypeError("delayBeforeRetry must be a positive number")
|
||||
}
|
||||
|
||||
uniqid(file) {
|
||||
return Math.floor(Math.random() * 100000000) + Date.now() + this.file.size + "_tmp"
|
||||
}
|
||||
|
||||
getChunk() {
|
||||
return new Promise((resolve) => {
|
||||
const length = this.totalChunks === 1 ? this.file.size : this.chunkSize * 1000 * 1000
|
||||
const start = length * this.chunkCount
|
||||
|
||||
this._reader.onload = () => {
|
||||
this.chunk = new Blob([this._reader.result], { type: "application/octet-stream" })
|
||||
resolve()
|
||||
}
|
||||
|
||||
this._reader.readAsArrayBuffer(this.file.slice(start, start + length))
|
||||
})
|
||||
}
|
||||
|
||||
sendChunk() {
|
||||
const form = new FormData()
|
||||
|
||||
// send post fields on last request
|
||||
if (this.chunkCount + 1 === this.totalChunks && this.postParams) Object.keys(this.postParams).forEach(key => form.append(key, this.postParams[key]))
|
||||
|
||||
form.append("file", this.chunk)
|
||||
|
||||
this.headers["uploader-chunk-number"] = this.chunkCount
|
||||
|
||||
return fetch(this.endpoint, { method: "POST", headers: this.headers, body: form })
|
||||
}
|
||||
|
||||
manageRetries() {
|
||||
if (this.retriesCount++ < this.retries) {
|
||||
setTimeout(() => this.sendChunks(), this.delayBeforeRetry * 1000)
|
||||
|
||||
this.eventBus.emit("fileRetry", {
|
||||
message: `An error occured uploading chunk ${this.chunkCount}. ${this.retries - this.retriesCount} retries left`,
|
||||
chunk: this.chunkCount,
|
||||
retriesLeft: this.retries - this.retriesCount
|
||||
})
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
this.eventBus.emit("error", {
|
||||
message: `An error occured uploading chunk ${this.chunkCount}. No more retries, stopping upload`
|
||||
})
|
||||
}
|
||||
|
||||
sendChunks() {
|
||||
if (this.paused || this.offline) return
|
||||
|
||||
this.getChunk()
|
||||
.then(() => this.sendChunk())
|
||||
.then((res) => {
|
||||
if (res.status === 200 || res.status === 201 || res.status === 204) {
|
||||
if (++this.chunkCount < this.totalChunks) this.sendChunks()
|
||||
else {
|
||||
res.json().then((body) => {
|
||||
this.eventBus.emit("finish", body)
|
||||
})
|
||||
}
|
||||
|
||||
const percentProgress = Math.round((100 / this.totalChunks) * this.chunkCount)
|
||||
|
||||
this.eventBus.emit("progress", {
|
||||
percentProgress
|
||||
})
|
||||
}
|
||||
|
||||
// errors that might be temporary, wait a bit then retry
|
||||
else if ([408, 502, 503, 504].includes(res.status)) {
|
||||
if (this.paused || this.offline) return
|
||||
|
||||
this.manageRetries()
|
||||
}
|
||||
|
||||
else {
|
||||
if (this.paused || this.offline) return
|
||||
|
||||
this.eventBus.emit("error", {
|
||||
message: `An error occured uploading chunk ${this.chunkCount}. Server responded with ${res.status}`
|
||||
})
|
||||
}
|
||||
})
|
||||
.catch((err) => {
|
||||
if (this.paused || this.offline) return
|
||||
|
||||
console.error(err)
|
||||
|
||||
// this type of error can happen after network disconnection on CORS setup
|
||||
this.manageRetries()
|
||||
})
|
||||
}
|
||||
|
||||
togglePause() {
|
||||
this.paused = !this.paused
|
||||
|
||||
if (!this.paused) {
|
||||
this.sendChunks()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default class RemoteStorage extends Core {
|
||||
static namespace = "remoteStorage"
|
||||
@ -17,115 +182,55 @@ export default class RemoteStorage extends Core {
|
||||
return hashHex
|
||||
}
|
||||
|
||||
async uploadFile(file, callback, options = {}) {
|
||||
const CHUNK_SIZE = 5000000
|
||||
async uploadFile(
|
||||
file,
|
||||
{
|
||||
onProgress = () => { },
|
||||
onFinish = () => { },
|
||||
onError = () => { },
|
||||
} = {},
|
||||
) {
|
||||
const apiEndpoint = app.cores.api.instance().instances.files.getUri()
|
||||
|
||||
const fileHash = await this.getFileHash(file)
|
||||
const fileSize = file.size
|
||||
|
||||
const chunks = Math.ceil(fileSize / CHUNK_SIZE)
|
||||
|
||||
const uploadTasks = []
|
||||
|
||||
for (let i = 0; i < chunks; i++) {
|
||||
const start = i * CHUNK_SIZE
|
||||
const end = Math.min(start + CHUNK_SIZE, fileSize)
|
||||
|
||||
const chunkData = file.slice(start, end)
|
||||
|
||||
const uploadTask = async () => {
|
||||
const formData = new FormData()
|
||||
|
||||
formData.append("file", chunkData, file.name)
|
||||
|
||||
const response = await app.cores.api.customRequest({
|
||||
...options,
|
||||
url: "/files/upload_chunk",
|
||||
method: "POST",
|
||||
headers: {
|
||||
...options.headers ?? {},
|
||||
"file-size": fileSize,
|
||||
"file-hash": fileHash,
|
||||
"file-chunk-size": end - start,
|
||||
"file-chunk-number": i,
|
||||
"file-total-chunks": chunks,
|
||||
"Content-Range": `bytes ${start}-${end - 1}/${fileSize}`,
|
||||
"Content-Type": "multipart/form-data",
|
||||
},
|
||||
data: formData,
|
||||
})
|
||||
|
||||
console.debug(`[Chunk Upload](${file.name})(${i}) Response >`, response.data)
|
||||
|
||||
return response.data
|
||||
}
|
||||
|
||||
uploadTasks.push(uploadTask)
|
||||
}
|
||||
|
||||
const uploadChunksTask = async () => {
|
||||
try {
|
||||
let lastResponse = null
|
||||
|
||||
for await (const task of uploadTasks) {
|
||||
lastResponse = await task()
|
||||
}
|
||||
|
||||
if (typeof callback === "function") {
|
||||
callback(null, lastResponse)
|
||||
}
|
||||
|
||||
return lastResponse
|
||||
} catch (error) {
|
||||
if (typeof callback === "function") {
|
||||
callback(error, lastResponse)
|
||||
}
|
||||
|
||||
throw error
|
||||
}
|
||||
}
|
||||
// TODO: get value from settings
|
||||
const chunkSize = 2 * 1000 * 1000 // 10MB
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
app.cores.tasksQueue.appendToQueue(fileHash, async () => {
|
||||
try {
|
||||
console.log(`Starting upload of file ${file.name}`)
|
||||
console.log("fileHash", fileHash)
|
||||
console.log("fileSize", fileSize)
|
||||
console.log("chunks", chunks)
|
||||
app.cores.tasksQueue.appendToQueue(`upload_${file.name}`, async () => {
|
||||
const uploader = new ChunkedUpload({
|
||||
endpoint: `${apiEndpoint}/upload/chunk`,
|
||||
chunkSize: chunkSize,
|
||||
file: file,
|
||||
})
|
||||
|
||||
const result = await uploadChunksTask()
|
||||
uploader.on("error", ({ message }) => {
|
||||
console.error("[Uploader] Error", message)
|
||||
|
||||
return resolve(result)
|
||||
} catch (error) {
|
||||
return reject(error)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
async uploadFiles(files) {
|
||||
const results = []
|
||||
|
||||
const promises = files.map((file) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
const callback = (error, result) => {
|
||||
if (error) {
|
||||
reject(error)
|
||||
} else {
|
||||
results.push({
|
||||
name: file.name,
|
||||
size: file.size,
|
||||
result: result,
|
||||
})
|
||||
resolve()
|
||||
if (typeof onError === "function") {
|
||||
onError(message)
|
||||
}
|
||||
}
|
||||
app.cores.tasksQueue.appendToQueue(() => this.uploadFile(file, callback))
|
||||
|
||||
reject(message)
|
||||
})
|
||||
|
||||
uploader.on("progress", ({ percentProgress }) => {
|
||||
//console.debug(`[Uploader] Progress: ${percentProgress}%`)
|
||||
|
||||
if (typeof onProgress === "function") {
|
||||
onProgress(percentProgress)
|
||||
}
|
||||
})
|
||||
|
||||
uploader.on("finish", (data) => {
|
||||
console.debug("[Uploader] Finish", data)
|
||||
|
||||
if (typeof onFinish === "function") {
|
||||
onFinish(data)
|
||||
}
|
||||
|
||||
resolve(data)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
await Promise.all(promises)
|
||||
|
||||
return results
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user