From 8e466fedf45bf0b5b48ed549b088a68c3183d1b9 Mon Sep 17 00:00:00 2001 From: SrGooglo Date: Thu, 24 Apr 2025 10:17:34 +0000 Subject: [PATCH] Replace Backblaze-B2 with StorageClient in marketplace service --- .../classes/extension/methods/resolve.js | 37 ++++---- .../marketplace/marketplace.service.js | 60 +++++++------ .../server/services/marketplace/package.json | 5 +- .../routes/extensions/[user_id]/[pkg]/get.js | 12 +-- .../extensions/[user_id]/[pkg]/main/get.js | 12 --- .../marketplace/routes/extensions/list/get.js | 30 ++++++- .../routes/extensions/publish/put.js | 54 ++---------- .../marketplace/utils/b2Upload/index.js | 86 ------------------- .../utils/syncDirToRemote/index.js | 23 ----- 9 files changed, 97 insertions(+), 222 deletions(-) delete mode 100644 packages/server/services/marketplace/routes/extensions/[user_id]/[pkg]/main/get.js delete mode 100644 packages/server/services/marketplace/utils/b2Upload/index.js delete mode 100755 packages/server/services/marketplace/utils/syncDirToRemote/index.js diff --git a/packages/server/services/marketplace/classes/extension/methods/resolve.js b/packages/server/services/marketplace/classes/extension/methods/resolve.js index cc61dcc5..ee542f04 100644 --- a/packages/server/services/marketplace/classes/extension/methods/resolve.js +++ b/packages/server/services/marketplace/classes/extension/methods/resolve.js @@ -1,24 +1,27 @@ import { Extension } from "@db_models" export default async function resolve(payload) { - let { user_id, pkg } = payload + let { user_id, pkg } = payload - const [pkgName, pkgVersion] = pkg.split("@") + let [pkgName, pkgVersion] = pkg.split("@") - if (!pkgVersion) { - pkgVersion = "latest" - } + if (!pkgVersion) { + pkgVersion = "latest" + } - if (pkgVersion === "latest") { - return await Extension.findOne({ - user_id, - name: pkgName, - }).sort({ version: -1 }).limit(1).exec() - } + if (pkgVersion === "latest") { + return await Extension.findOne({ + user_id, + name: pkgName, + }) + .sort({ version: -1 }) + .limit(1) + .exec() + } - return await Extension.findOne({ - user_id, - name: pkgName, - version: pkgVersion, - }) -} \ No newline at end of file + return await Extension.findOne({ + user_id, + name: pkgName, + version: pkgVersion, + }) +} diff --git a/packages/server/services/marketplace/marketplace.service.js b/packages/server/services/marketplace/marketplace.service.js index a01cd9da..a9d7b338 100755 --- a/packages/server/services/marketplace/marketplace.service.js +++ b/packages/server/services/marketplace/marketplace.service.js @@ -1,41 +1,47 @@ import { Server } from "linebridge" -import B2 from "backblaze-b2" import DbManager from "@shared-classes/DbManager" import CacheService from "@shared-classes/CacheService" +import StorageClient from "@shared-classes/StorageClient" import SharedMiddlewares from "@shared-middlewares" class API extends Server { - static refName = "marketplace" - static wsRoutesPath = `${__dirname}/ws_routes` - static routesPath = `${__dirname}/routes` - static listen_port = process.env.HTTP_LISTEN_PORT ?? 3005 + static refName = "marketplace" + static useEngine = "hyper-express-ng" + static routesPath = `${__dirname}/routes` + static listen_port = process.env.HTTP_LISTEN_PORT ?? 3005 - middlewares = { - ...SharedMiddlewares - } + middlewares = { + ...SharedMiddlewares, + } - contexts = { - db: new DbManager(), - b2: new B2({ - applicationKeyId: process.env.B2_KEY_ID, - applicationKey: process.env.B2_APP_KEY, - }), - cache: new CacheService({ - fsram: false - }), - } + contexts = { + db: new DbManager(), + cache: new CacheService({ + fsram: false, + }), + storage: StorageClient({ + endPoint: process.env.B2_ENDPOINT, + cdnUrl: process.env.B2_CDN_ENDPOINT, + defaultBucket: process.env.B2_BUCKET, + accessKey: process.env.B2_KEY_ID, + secretKey: process.env.B2_APP_KEY, + port: 443, + useSSL: true, + setupBucket: false, + }), + } - async onInitialize() { - await this.contexts.db.initialize() - await this.contexts.b2.authorize() + async onInitialize() { + await this.contexts.db.initialize() + await this.contexts.storage.initialize() - global.cache = this.contexts.cache - global.b2 = this.contexts.b2 - } - - handleWsAuth = require("@shared-lib/handleWsAuth").default + global.cache = this.contexts.cache + global.storages = { + standard: this.contexts.storage, + } + } } -Boot(API) \ No newline at end of file +Boot(API) diff --git a/packages/server/services/marketplace/package.json b/packages/server/services/marketplace/package.json index cff27833..9f272677 100755 --- a/packages/server/services/marketplace/package.json +++ b/packages/server/services/marketplace/package.json @@ -1,9 +1,6 @@ { "name": "marketplace", "dependencies": { - "7zip-min": "^1.4.4", - "backblaze-b2": "^1.7.0", - "sucrase": "^3.32.0", - "uglify-js": "^3.17.4" + "7zip-min": "^1.4.4" } } diff --git a/packages/server/services/marketplace/routes/extensions/[user_id]/[pkg]/get.js b/packages/server/services/marketplace/routes/extensions/[user_id]/[pkg]/get.js index a24347b4..f02f43c7 100644 --- a/packages/server/services/marketplace/routes/extensions/[user_id]/[pkg]/get.js +++ b/packages/server/services/marketplace/routes/extensions/[user_id]/[pkg]/get.js @@ -1,10 +1,10 @@ import ExtensionClass from "@classes/extension" export default async (req) => { - const { user_id, pkg } = req.params + const { user_id, pkg } = req.params - return await ExtensionClass.resolveManifest({ - user_id, - pkg, - }) -} \ No newline at end of file + return await ExtensionClass.resolve({ + user_id, + pkg, + }) +} diff --git a/packages/server/services/marketplace/routes/extensions/[user_id]/[pkg]/main/get.js b/packages/server/services/marketplace/routes/extensions/[user_id]/[pkg]/main/get.js deleted file mode 100644 index 70a190ba..00000000 --- a/packages/server/services/marketplace/routes/extensions/[user_id]/[pkg]/main/get.js +++ /dev/null @@ -1,12 +0,0 @@ -import ExtensionClass from "@classes/extension" - -export default async (req, res) => { - const { user_id, pkg } = req.params - - const manifest = await ExtensionClass.resolve({ - user_id, - pkg, - }) - - return manifest -} \ No newline at end of file diff --git a/packages/server/services/marketplace/routes/extensions/list/get.js b/packages/server/services/marketplace/routes/extensions/list/get.js index 679325da..ae6c5cc9 100644 --- a/packages/server/services/marketplace/routes/extensions/list/get.js +++ b/packages/server/services/marketplace/routes/extensions/list/get.js @@ -1,7 +1,33 @@ import { Extension } from "@db_models" export default async (req) => { - const extensions = await Extension.find() + const { limit = 10, offset = 0 } = req.query - return extensions + const totalItems = await Extension.countDocuments() + + const extensions = await Extension.aggregate([ + { + $sort: { registryId: 1, version: -1 }, + }, + { + $group: { + _id: "$registryId", + doc: { $first: "$$ROOT" }, + }, + }, + { + $replaceRoot: { newRoot: "$doc" }, + }, + { + $skip: parseInt(offset), + }, + { + $limit: parseInt(limit), + }, + ]) + + return { + items: extensions, + total_items: totalItems, + } } diff --git a/packages/server/services/marketplace/routes/extensions/publish/put.js b/packages/server/services/marketplace/routes/extensions/publish/put.js index 18337084..e070e038 100644 --- a/packages/server/services/marketplace/routes/extensions/publish/put.js +++ b/packages/server/services/marketplace/routes/extensions/publish/put.js @@ -4,47 +4,7 @@ import fs from "node:fs" import path from "node:path" import sevenzip from "7zip-min" -async function uploadFolderToB2(bucketId, folderPath, b2Directory) { - try { - const uploadFiles = async (dir) => { - const files = fs.readdirSync(dir) - - for (const file of files) { - const fullPath = path.join(dir, file) - const stats = fs.statSync(fullPath) - - if (stats.isDirectory()) { - await uploadFiles(fullPath) - } else { - const fileData = fs.readFileSync(fullPath) - const b2FileName = path - .join(b2Directory, path.relative(folderPath, fullPath)) - .replace(/\\/g, "/") - - console.log(`Uploading ${b2FileName}...`) - - const uploadUrl = await b2.getUploadUrl({ - bucketId: bucketId, - }) - - await b2.uploadFile({ - uploadUrl: uploadUrl.data.uploadUrl, - uploadAuthToken: uploadUrl.data.authorizationToken, - fileName: b2FileName, - data: fileData, - }) - - console.log(`Uploaded ${b2FileName}`) - } - } - } - - await uploadFiles(folderPath) - console.log("All files uploaded successfully.") - } catch (error) { - console.error("Error uploading folder:", error) - } -} +import putObject from "@shared-classes/Upload/putObject" export default { middlewares: ["withAuthentication"], @@ -62,7 +22,7 @@ export default { pkg = JSON.parse(pkg) const { user_id } = req.auth.session - const registryId = `${user_id}/${pkg.name}@${pkg.version}` + const registryId = `${user_id}/${pkg.name}` const s3Path = `extensions/${pkg.name}/${pkg.version}` const workPath = path.resolve( @@ -84,7 +44,7 @@ export default { let extensionRegistry = await Extension.findOne({ user_id: user_id, - registryId: registryId, + name: pkg.name, version: pkg.version, }) @@ -116,16 +76,20 @@ export default { }) }) - await uploadFolderToB2(process.env.B2_BUCKET_ID, pkgPath, s3Path) + await putObject({ + filePath: pkgPath, + uploadPath: s3Path, + }) fs.promises.rm(workPath, { recursive: true, force: true }) - const assetsUrl = `https://${process.env.B2_CDN_ENDPOINT}/${process.env.B2_BUCKET}/${s3Path}` + const assetsUrl = `${process.env.B2_CDN_ENDPOINT}/${process.env.B2_BUCKET}/${s3Path}` extensionRegistry = await Extension.create({ user_id: user_id, name: pkg.name, version: pkg.version, + description: pkg.description, registryId: registryId, assetsUrl: assetsUrl, srcUrl: `${assetsUrl}/src`, diff --git a/packages/server/services/marketplace/utils/b2Upload/index.js b/packages/server/services/marketplace/utils/b2Upload/index.js deleted file mode 100644 index 82f80239..00000000 --- a/packages/server/services/marketplace/utils/b2Upload/index.js +++ /dev/null @@ -1,86 +0,0 @@ -import fs from "node:fs" -import path from "node:path" -import pMap from "p-map" - -export default async function b2Upload({ - source, - remotePath, - metadata = {}, - targetFilename, - isDirectory, - retryNumber = 0, -}) { - if (isDirectory) { - let files = await fs.promises.readdir(source) - - files = files.map((file) => { - const filePath = path.join(source, file) - - const isTargetDirectory = fs.lstatSync(filePath).isDirectory() - - return { - source: filePath, - remotePath: path.join(remotePath, file), - isDirectory: isTargetDirectory, - } - }) - - await pMap(files, b2Upload, { - concurrency: 5, - }) - - return { - id: remotePath, - url: `https://${process.env.B2_CDN_ENDPOINT}/${process.env.B2_BUCKET}/${remotePath}/${targetFilename}`, - metadata: metadata, - } - } - - try { - //await global.b2.authorize() - - if (!fs.existsSync(source)) { - throw new OperationError(500, "File not found") - } - - const uploadUrl = await global.b2.getUploadUrl({ - bucketId: process.env.B2_BUCKET_ID, - }) - - console.debug(`Uploading object to B2 Storage >`, { - source: source, - remote: remotePath, - }) - - const data = await fs.promises.readFile(source) - - await global.b2.uploadFile({ - uploadUrl: uploadUrl.data.uploadUrl, - uploadAuthToken: uploadUrl.data.authorizationToken, - fileName: remotePath, - data: data, - info: metadata, - }) - } catch (error) { - console.error(error) - - if (retryNumber < 5) { - return await b2Upload({ - source, - remotePath, - metadata, - targetFilename, - isDirectory, - retryNumber: retryNumber + 1, - }) - } - - throw new OperationError(500, "B2 upload failed") - } - - return { - id: remotePath, - url: `https://${process.env.B2_CDN_ENDPOINT}/${process.env.B2_BUCKET}/${remotePath}`, - metadata: metadata, - } -} diff --git a/packages/server/services/marketplace/utils/syncDirToRemote/index.js b/packages/server/services/marketplace/utils/syncDirToRemote/index.js deleted file mode 100755 index 7481c2a7..00000000 --- a/packages/server/services/marketplace/utils/syncDirToRemote/index.js +++ /dev/null @@ -1,23 +0,0 @@ -import fs from "fs" -import path from "path" - -async function syncFolder(dir, destPath) { - const files = await fs.promises.readdir(dir) - - for await (const file of files) { - const filePath = path.resolve(dir, file) - const desitinationFilePath = `${destPath}/${file}` - - const stat = fs.statSync(filePath) - - if (stat.isDirectory()) { - await syncFolder(filePath, desitinationFilePath) - } else { - const fileContent = await fs.promises.readFile(filePath) - - await global.storage.putObject(process.env.S3_BUCKET, desitinationFilePath, fileContent) - } - } -} - -export default syncFolder \ No newline at end of file