From bd719202c152e1e75b46db56749aedd0a79c4b60 Mon Sep 17 00:00:00 2001 From: SrGooglo Date: Sun, 30 Jun 2024 05:50:10 +0200 Subject: [PATCH] support for torrent download --- packages/core/package.json | 3 +- packages/core/src/generic_steps/torrent.js | 7 +- packages/core/src/handlers/install.js | 6 +- packages/core/src/helpers/downloadTorrent.js | 137 +++++++++++------- packages/core/src/index.js | 14 ++ .../core/src/manifest/libs/extract/index.js | 34 +++++ packages/core/src/manifest/libs/fs/index.js | 32 ++++ packages/core/src/manifest/libs/index.js | 4 +- packages/core/src/prerequisites.js | 21 +-- .../core/src/utils/resolveRemoteBinPath.js | 4 +- packages/core/src/vars.js | 6 +- 11 files changed, 197 insertions(+), 71 deletions(-) create mode 100644 packages/core/src/manifest/libs/extract/index.js diff --git a/packages/core/package.json b/packages/core/package.json index 60320a2..74f58ee 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -16,6 +16,7 @@ "dependencies": { "@foxify/events": "^2.1.0", "adm-zip": "^0.5.12", + "aria2": "^4.1.2", "axios": "^1.6.8", "checksum": "^1.0.0", "cli-color": "^2.0.4", @@ -41,4 +42,4 @@ "@swc/cli": "^0.3.12", "@swc/core": "^1.4.11" } -} \ No newline at end of file +} diff --git a/packages/core/src/generic_steps/torrent.js b/packages/core/src/generic_steps/torrent.js index 32666df..5c5edac 100644 --- a/packages/core/src/generic_steps/torrent.js +++ b/packages/core/src/generic_steps/torrent.js @@ -1,10 +1,13 @@ import path from "node:path" +import fs from "node:fs" import parseStringVars from "../utils/parseStringVars" -//import downloadTorrent from "../helpers/downloadTorrent" +import downloadTorrent from "../helpers/downloadTorrent" export default async (pkg, step, logger, abortController) => { - throw new Error("Not implemented") + if (!step.magnet) { + throw new Error(`Magnet is required for torrent step`) + } if (typeof step.path === "undefined") { step.path = `.` diff --git a/packages/core/src/handlers/install.js b/packages/core/src/handlers/install.js index 84efba6..854c7b6 100644 --- a/packages/core/src/handlers/install.js +++ b/packages/core/src/handlers/install.js @@ -10,7 +10,7 @@ import Apply from "../handlers/apply" const BaseLog = Logger.child({ service: "INSTALLER" }) -export default async function install(manifest) { +export default async function install(manifest, options = {}) { let id = null let abortController = new AbortController() @@ -111,8 +111,8 @@ export default async function install(manifest) { if (abortController.signal.aborted) { return false } - - if (Array.isArray(manifest.installSteps)) { + + if (Array.isArray(manifest.installSteps) && !options.noInstallSteps) { Log.info(`Executing generic install steps...`) global._relic_eventBus.emit(`pkg:update:state`, { diff --git a/packages/core/src/helpers/downloadTorrent.js b/packages/core/src/helpers/downloadTorrent.js index 6f10070..a64c4ef 100644 --- a/packages/core/src/helpers/downloadTorrent.js +++ b/packages/core/src/helpers/downloadTorrent.js @@ -2,6 +2,7 @@ import fs from "node:fs" import path from "node:path" import cliProgress from "cli-progress" import humanFormat from "human-format" +import aria2 from "aria2" function convertSize(size) { return `${humanFormat(size, { @@ -30,64 +31,102 @@ export default async function downloadTorrent( speedString: "0B/s", } - const client = new WebTorrent() + const client = new aria2({ + host: 'localhost', + port: 6800, + secure: false, + secret: '', + path: '/jsonrpc' + }) - await new Promise((resolve, reject) => { - client.add(magnet, (torrentInstance) => { - const progressBar = new cliProgress.SingleBar({ - format: "[{bar}] {percentage}% | {total_formatted} | {speed}/s | {eta_formatted}", - barCompleteChar: "\u2588", - barIncompleteChar: "\u2591", - hideCursor: true - }, cliProgress.Presets.shades_classic) + await client.open() - if (typeof onStart === "function") { - onStart(torrentInstance) + let downloadId = await client.call( + "addUri", + [magnet], + { + dir: destination, + }, + ) + + await new Promise(async (resolve, reject) => { + if (typeof onStart === "function") { + onStart() + } + + progressInterval = setInterval(async () => { + const data = await client.call("tellStatus", downloadId) + const isMetadata = data.totalLength === "0" && data.status === "active" + + console.log(data) + + if (data.status === "complete") { + if (Array.isArray(data.followedBy) && data.followedBy[0]) { + // replace downloadId + downloadId = data.followedBy[0] + } } - progressBar.start(tickProgress.total, 0, { - speed: "0B/s", - total_formatted: tickProgress.totalString, + tickProgress.total = parseInt(data.totalLength) + tickProgress.speed = parseInt(data.downloadSpeed) + tickProgress.transferred = parseInt(data.completedLength) + tickProgress.connections = data.connections + + tickProgress.transferredString = convertSize(tickProgress.transferred) + tickProgress.totalString = convertSize(tickProgress.total) + tickProgress.speedString = convertSize(tickProgress.speed) + + if (typeof onProgress === "function") { + onProgress(tickProgress) + } + }, 1000) + + client.on("onDownloadStart", async ([{ gid }]) => { + const data = await client.call("tellStatus", gid) + + console.log(data) + + if (typeof data.following !== "undefined") { + if (data.following === downloadId) { + downloadId = data.gid + } + } + }) + + client.on("onBtDownloadComplete", ([{ gid }]) => { + if (gid !== downloadId) { + return false + } + + clearInterval(progressInterval) + + if (typeof onDone === "function") { + onDone() + } + + resolve({ + downloadId, }) - torrentInstance.on("done", () => { - progressBar.stop() - clearInterval(progressInterval) + return null + }) - if (typeof onDone === "function") { - onDone(torrentInstance) - } + client.on("onDownloadError", ([{ gid }]) => { + if (gid !== downloadId) { + return false + } - resolve(torrentInstance) - }) + clearInterval(progressInterval) - torrentInstance.on("error", (error) => { - progressBar.stop() - clearInterval(progressInterval) - - if (typeof onError === "function") { - onError(error) - } else { - reject(error) - } - }) - - progressInterval = setInterval(() => { - tickProgress.speed = torrentInstance.downloadSpeed - tickProgress.transferred = torrentInstance.downloaded - - tickProgress.transferredString = convertSize(tickProgress.transferred) - tickProgress.totalString = convertSize(tickProgress.total) - tickProgress.speedString = convertSize(tickProgress.speed) - - if (typeof onProgress === "function") { - onProgress(tickProgress) - } - - progressBar.update(tickProgress.transferred, { - speed: tickProgress.speedString, - }) - }, 1000) + if (typeof onError === "function") { + onError() + } else { + reject() + } }) }) + + await client.call("remove", downloadId) + + return downloadId } \ No newline at end of file diff --git a/packages/core/src/index.js b/packages/core/src/index.js index 4f1dec3..bee9602 100644 --- a/packages/core/src/index.js +++ b/packages/core/src/index.js @@ -4,6 +4,7 @@ import { onExit } from "signal-exit" import open from "open" import SetupHelper from "./helpers/setup" +import { execa } from "./libraries/execa" import Logger from "./logger" import Settings from "./classes/Settings" @@ -49,6 +50,15 @@ export default class RelicCore { await Settings.set("packages_path", Vars.packages_path) } + this.aria2c_instance = execa( + Vars.aria2_bin, + ["--enable-rpc", "--rpc-listen-all=true", "--rpc-allow-origin-all", "--file-allocation=none"], + { + stdout: "inherit", + stderr: "inherit", + } + ) + onExit(this.onExit) } @@ -56,6 +66,10 @@ export default class RelicCore { if (fs.existsSync(Vars.cache_path)) { fs.rmSync(Vars.cache_path, { recursive: true, force: true }) } + + if (this.aria2c_instance) { + this.aria2c_instance.kill("SIGINT") + } } async setup() { diff --git a/packages/core/src/manifest/libs/extract/index.js b/packages/core/src/manifest/libs/extract/index.js new file mode 100644 index 0000000..c997b78 --- /dev/null +++ b/packages/core/src/manifest/libs/extract/index.js @@ -0,0 +1,34 @@ +import extractFile from "../../../utils/extractFile" +import { execa } from "../../../libraries/execa" +import Vars from "../../../vars" + +export default class Extract { + async extractFull(file, dest, { password } = {}) { + const args = [ + "x", + "-y", + ] + + if (password) { + args.push(`-p"${password}"`) + } + + args.push(`-o"${dest}"`) + + args.push(`"${file}"`) + + const cmd = `${Vars.sevenzip_bin} ${args.join(" ")}` + + console.log(cmd) + + await execa(cmd, { + shell: true, + stdout: "inherit", + stderr: "inherit", + }) + } + + async autoExtract(file, dest) { + return await extractFile(file, dest) + } +} \ No newline at end of file diff --git a/packages/core/src/manifest/libs/fs/index.js b/packages/core/src/manifest/libs/fs/index.js index d12eaee..5d4a9ea 100644 --- a/packages/core/src/manifest/libs/fs/index.js +++ b/packages/core/src/manifest/libs/fs/index.js @@ -36,4 +36,36 @@ export default class SecureFileSystem { existsSync(...args) { return fs.existsSync(...args) } + + async rename(from, to) { + this.checkOutsideJail(from) + this.checkOutsideJail(to) + + return await fs.promises.rename(from, to) + } + + async writeFile(path, data, options) { + this.checkOutsideJail(path) + return await fs.promises.writeFile(path, data, options) + } + + async readDir(path) { + this.checkOutsideJail(path) + return await fs.promises.readdir(path) + } + + async rm(path, options) { + this.checkOutsideJail(path) + return await fs.promises.rm(path, options) + } + + async mkdir(path, options) { + this.checkOutsideJail(path) + return await fs.promises.mkdir(path, options) + } + + async stat(path) { + this.checkOutsideJail(path) + return await fs.promises.stat(path) + } } \ No newline at end of file diff --git a/packages/core/src/manifest/libs/index.js b/packages/core/src/manifest/libs/index.js index 3a33e39..bfe1087 100644 --- a/packages/core/src/manifest/libs/index.js +++ b/packages/core/src/manifest/libs/index.js @@ -2,6 +2,7 @@ import Open from "./open" import Path from "./path" import Fs from "./fs" import Auth from "./auth" +import Extract from "./extract" // Third party libraries import Mcl from "./mcl" @@ -11,5 +12,6 @@ export default { path: Path, open: Open, auth: Auth, - mcl: Mcl + extract: Extract, + mcl: Mcl, } \ No newline at end of file diff --git a/packages/core/src/prerequisites.js b/packages/core/src/prerequisites.js index 9b7f442..77b8cff 100644 --- a/packages/core/src/prerequisites.js +++ b/packages/core/src/prerequisites.js @@ -9,9 +9,11 @@ export default [ { id: "7z-bin", finalBin: Vars.sevenzip_bin, - url: resolveRemoteBinPath(`${baseURL}/7zip-bin`, process.platform === "win32" ? "7za.exe" : "7za"), - destination: Vars.sevenzip_bin, + url: resolveRemoteBinPath(`${baseURL}/7z-full`, "7z.zip"), + destination: path.resolve(Vars.binaries_path, "7z.zip"), + extract: path.resolve(Vars.binaries_path, "7z-bin"), rewriteExecutionPermission: true, + deleteBeforeExtract: true, }, { id: "git-bin", @@ -24,14 +26,13 @@ export default [ deleteBeforeExtract: true, }, { - id: "rclone-bin", - finalBin: Vars.rclone_bin, - url: resolveRemoteBinPath(`${baseURL}/rclone`, "rclone-bin.zip"), - destination: path.resolve(Vars.binaries_path, "rclone-bin.zip"), - extract: path.resolve(Vars.binaries_path, "rclone-bin"), - requireOs: ["win32"], - rewriteExecutionPermission: true, - deleteBeforeExtract: true, + id: "aria2", + finalBin: Vars.aria2_bin, + url: async (os, arch) => { + return `https://storage.ragestudio.net/rstudio/binaries/aria2/${os}/${arch}/${os === "win32" ? "aria2c.exe" : "aria2c"}` + }, + destination: Vars.aria2_bin, + rewriteExecutionPermission: Vars.aria2_bin, }, { id: "java22_jre_bin", diff --git a/packages/core/src/utils/resolveRemoteBinPath.js b/packages/core/src/utils/resolveRemoteBinPath.js index acc8926..8c2b818 100644 --- a/packages/core/src/utils/resolveRemoteBinPath.js +++ b/packages/core/src/utils/resolveRemoteBinPath.js @@ -2,10 +2,10 @@ export default (pre, post) => { let url = null if (process.platform === "darwin") { - url = `${pre}/mac/${process.arch}/${post}` + url = `${pre}/darwin/${process.arch}/${post}` } else if (process.platform === "win32") { - url = `${pre}/win/${process.arch}/${post}` + url = `${pre}/win32/${process.arch}/${post}` } else { url = `${pre}/linux/${process.arch}/${post}` diff --git a/packages/core/src/vars.js b/packages/core/src/vars.js index 117a441..5e1bb65 100644 --- a/packages/core/src/vars.js +++ b/packages/core/src/vars.js @@ -2,7 +2,7 @@ import path from "node:path" import upath from "upath" import resolveUserDataPath from "./utils/resolveUserDataPath" -const isWin = process.platform.includes("win") +const isWin = process.platform.includes("win32") const isMac = process.platform.includes("darwin") const runtimeName = "rs-relic" @@ -15,9 +15,9 @@ const binaries_path = upath.normalizeSafe(path.resolve(runtime_path, "binaries") const db_path = upath.normalizeSafe(path.resolve(runtime_path, "db.json")) const binaries = { - sevenzip_bin: upath.normalizeSafe(path.resolve(binaries_path, "7z-bin", isWin ? "7za.exe" : "7za")), + sevenzip_bin: upath.normalizeSafe(path.resolve(binaries_path, "7z-bin", isWin ? "7z.exe" : "7zz")), git_bin: upath.normalizeSafe(path.resolve(binaries_path, "git-bin", "bin", isWin ? "git.exe" : "git")), - rclone_bin: upath.normalizeSafe(path.resolve(binaries_path, "rclone-bin", isWin ? "rclone.exe" : "rclone")), + aria2_bin: upath.normalizeSafe(path.resolve(binaries_path, "aria2", isWin ? "aria2c.exe" : "aria2c")), java22_jre_bin: upath.normalizeSafe(path.resolve(binaries_path, "java22_jre_bin", (isMac ? "Contents/Home/bin/java" : (isWin ? "bin/java.exe" : "bin/java")))), java17_jre_bin: upath.normalizeSafe(path.resolve(binaries_path, "java17_jre_bin", (isMac ? "Contents/Home/bin/java" : (isWin ? "bin/java.exe" : "bin/java")))), }