RageStudio/comty
1
0
Fork 0
mirror of https://github.com/ragestudio/comty.git synced 2025-06-09 18:44:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

use bullmq

Browse Source
This commit is contained in:
SrGooglo 2025-02-25 23:07:52 +00:00
parent 35282dc1cd
commit cd0b029c0d
1 changed files with 53 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
packages/server/classes/TaskQueueManager/index.js
View File

@ -1,19 +1,19 @@
import fs from "node:fs" import fs from "node:fs"
import Queue from "bull" import { Queue, Worker } from "bullmq"
import { composeURL as composeRedisConnectionString } from "@shared-classes/RedisClient" import { composeURL as composeRedisConnectionString } from "@shared-classes/RedisClient"
process.env.DEBUG = "bull:*"
export default class TaskQueueManager { export default class TaskQueueManager {
constructor(params, ctx) { constructor(params, ctx) {
if (!params) { if (!params) {
throw new Error("Missing params") throw new Error("Missing params")
} }
this.params = params this.ctx = ctx
}
queues = new Object() this.params = params
this.queues = {}
this.workers = {}
}
async initialize(options = {}) { async initialize(options = {}) {
const queues = fs.readdirSync(this.params.workersPath) const queues = fs.readdirSync(this.params.workersPath)
@ -36,20 +36,29 @@ export default class TaskQueueManager {
} }
registerQueue = (queueObj, options) => { registerQueue = (queueObj, options) => {
let queue = new Queue(queueObj.id, { const connection = this.ctx.engine.ws.redis
redis: composeRedisConnectionString(options.redisOptions),
removeOnSuccess: true, const queue = new Queue(queueObj.id, {
connection,
defaultJobOptions: {
removeOnComplete: true,
},
}) })
queue = this.registerQueueEvents(queue) const worker = new Worker(queueObj.id, queueObj.process, {
connection,
concurrency: queueObj.maxJobs ?? 1,
})
queue.process(queueObj.maxJobs ?? 1, queueObj.process) this.registerQueueEvents(worker)
this.queues[queueObj.id] = queue
this.workers[queueObj.id] = worker
return queue return queue
} }
registerQueueEvents = (queue) => { registerQueueEvents = (worker) => {
queue.on("progress", (job, progress) => { worker.on("progress", (job, progress) => {
try { try {
console.log(`Job ${job.id} reported progress: ${progress}%`) console.log(`Job ${job.id} reported progress: ${progress}%`)
@ -61,11 +70,11 @@ export default class TaskQueueManager {
}) })
} }
} catch (error) { } catch (error) {
// sowy // manejar error
} }
}) })
queue.on("completed", (job, result) => { worker.on("completed", (job, result) => {
try { try {
console.log(`Job ${job.id} completed with result:`, result) console.log(`Job ${job.id} completed with result:`, result)
@ -78,7 +87,7 @@ export default class TaskQueueManager {
} catch (error) {} } catch (error) {}
}) })
queue.on("failed", (job, error) => { worker.on("failed", (job, error) => {
try { try {
console.error(`Job ${job.id} failed:`, error) console.error(`Job ${job.id} failed:`, error)
@ -90,39 +99,52 @@ export default class TaskQueueManager {
} }
} catch (error) {} } catch (error) {}
}) })
return queue
} }
createJob = async (queueId, data) => { createJob = async (queueId, data, { useSSE = false } = {}) => {
const queue = this.queues[queueId] const queue = this.queues[queueId]
if (!queue) { if (!queue) {
throw new Error("Queue not found") throw new Error("Queue not found")
} }
const sseChannelId = `${global.nanoid()}` let sseChannelId = null
// create job and create a sse channel id if (useSSE) {
const job = queue.add({ sseChannelId = `${global.nanoid()}`
}
const job = await queue.add("default", {
...data, ...data,
sseChannelId, sseChannelId,
}) })
// create the sse channel if (sseChannelId) {
await global.sse.createChannel(sseChannelId) await global.sse.createChannel(sseChannelId)
console.log(
`[JOB] Created new job with SSE channel [${sseChannelId}]`,
)
console.log(`[JOB] Created new job with SSE channel [${sseChannelId}]`) await global.sse.sendToChannel(sseChannelId, {
status: "progress",
events: "job_queued",
progress: 5,
})
}
await global.sse.sendToChannel(sseChannelId, { console.log(`[JOB] Created new job with ID [${job.id}]`)
status: "progress",
events: "job_queued",
progress: 5,
})
return { return {
...job, ...job,
sseChannelId: sseChannelId, sseChannelId,
} }
} }
// this function cleans up all queues, must be synchronous
cleanUp = () => {
const queues = Object.values(this.queues)
queues.forEach((queue) => queue.close())
console.log("All queues have been closed")
}
} }