merge from local

This commit is contained in:
SrGooglo 2024-03-17 00:26:25 +00:00
parent 6d553830ab
commit 751ba7956b
16 changed files with 295 additions and 325 deletions

160
boot Executable file
View File

@ -0,0 +1,160 @@
#!/usr/bin/env node
require("dotenv").config()
require("sucrase/register")
const path = require("path")
const Module = require("module")
const { Buffer } = require("buffer")
const { webcrypto: crypto } = require("crypto")
const { InfisicalClient } = require("@infisical/sdk")
const moduleAlias = require("module-alias")
// Override file execution arg
process.argv.splice(1, 1)
process.argv[1] = path.resolve(process.argv[1])
// Expose boot function to global
global.Boot = Boot
global.isProduction = process.env.NODE_ENV === "production"
global["__root"] = path.resolve(process.cwd())
global["__src"] = path.resolve(globalThis["__root"], path.dirname(process.argv[1]))
global["aliases"] = {
"root": global["__root"],
"src": global["__src"],
// expose shared resources
"@db_models": path.resolve(__dirname, "db_models"),
"@shared-utils": path.resolve(__dirname, "utils"),
"@shared-classes": path.resolve(__dirname, "classes"),
"@shared-lib": path.resolve(__dirname, "lib"),
"@shared-middlewares": path.resolve(__dirname, "middlewares"),
// expose internal resources
"@lib": path.resolve(global["__src"], "lib"),
"@middlewares": path.resolve(global["__src"], "middlewares"),
"@controllers": path.resolve(global["__src"], "controllers"),
"@config": path.resolve(global["__src"], "config"),
"@shared": path.resolve(global["__src"], "shared"),
"@classes": path.resolve(global["__src"], "classes"),
"@models": path.resolve(global["__src"], "models"),
"@services": path.resolve(global["__src"], "services"),
"@utils": path.resolve(global["__src"], "utils"),
}
function registerBaseAliases(fromPath, customAliases = {}) {
if (typeof fromPath === "undefined") {
if (module.parent.filename.includes("dist")) {
fromPath = path.resolve(process.cwd(), "dist")
} else {
fromPath = path.resolve(process.cwd(), "src")
}
}
moduleAlias.addAliases({
...customAliases,
"@controllers": path.resolve(fromPath, "controllers"),
"@middlewares": path.resolve(fromPath, "middlewares"),
"@models": path.resolve(fromPath, "models"),
"@classes": path.resolve(fromPath, "classes"),
"@lib": path.resolve(fromPath, "lib"),
"@utils": path.resolve(fromPath, "utils"),
})
}
function registerPatches() {
global.b64Decode = (data) => {
return Buffer.from(data, "base64").toString("utf-8")
}
global.b64Encode = (data) => {
return Buffer.from(data, "utf-8").toString("base64")
}
global.nanoid = (t = 21) => crypto.getRandomValues(new Uint8Array(t)).reduce(((t, e) => t += (e &= 63) < 36 ? e.toString(36) : e < 62 ? (e - 26).toString(36).toUpperCase() : e > 62 ? "-" : "_"), "");
Array.prototype.updateFromObjectKeys = function (obj) {
this.forEach((value, index) => {
if (obj[value] !== undefined) {
this[index] = obj[value]
}
})
return this
}
global.ToBoolean = (value) => {
if (typeof value === "boolean") {
return value
}
if (typeof value === "string") {
return value.toLowerCase() === "true"
}
return false
}
}
function registerAliases() {
registerBaseAliases(global["__src"], global["aliases"])
}
async function injectEnvFromInfisical() {
const envMode = global.FORCE_ENV ?? global.isProduction ? "prod" : "dev"
console.log(`[BOOT] 🔑 Injecting env variables from INFISICAL in [${envMode}] mode...`)
const client = new InfisicalClient({
accessToken: process.env.INFISICAL_TOKEN,
})
const secrets = await client.listSecrets({
environment: envMode,
path: process.env.INFISICAL_PATH ?? "/",
projectId: process.env.INFISICAL_PROJECT_ID ?? null,
includeImports: false,
})
//inject to process.env
secrets.forEach((secret) => {
if (!(process.env[secret.secretKey])) {
process.env[secret.secretKey] = secret.secretValue
}
})
}
async function Boot(main) {
if (!main) {
throw new Error("main class is not defined")
}
if (process.env.INFISICAL_TOKEN) {
console.log(`[BOOT] INFISICAL_TOKEN found, injecting env variables from INFISICAL...`)
await injectEnvFromInfisical()
}
const instance = new main()
await instance.initialize()
if (process.env.lb_service && process.send) {
process.send({
status: "ready"
})
}
return instance
}
console.log(`[BOOT] Booting in [${global.isProduction ? "production" : "development"}] mode...`)
// Apply patches
registerPatches()
// Apply aliases
registerAliases()
// execute main
Module.runMain()

123
bootstrap.js vendored
View File

@ -1,123 +0,0 @@
require("dotenv").config()
const path = require("path")
const { webcrypto: crypto } = require("crypto")
const infisical = require("infisical-node")
const { registerBaseAliases } = require("./dist/server")
const EventEmitter = require("./dist/lib/event_emitter").default
global.isProduction = process.env.NODE_ENV === "production"
globalThis["__root"] = path.resolve(process.cwd())
globalThis["__src"] = path.resolve(globalThis["__root"], global.isProduction ? "dist" : "src")
const customAliases = {
"root": globalThis["__root"],
"src": globalThis["__src"],
"@shared-classes": path.resolve(globalThis["__src"], "_shared/classes"),
"@services": path.resolve(globalThis["__src"], "services"),
}
if (!global.isProduction) {
customAliases["comty.js"] = path.resolve(globalThis["__src"], "../../comty.js/src")
customAliases["@shared-classes"] = path.resolve(globalThis["__src"], "shared-classes")
}
if (process.env.USE_LINKED_SHARED) {
customAliases["@shared-classes"] = path.resolve(globalThis["__src"], "shared-classes")
}
registerBaseAliases(globalThis["__src"], customAliases)
// patches
const { Buffer } = require("buffer")
global.b64Decode = (data) => {
return Buffer.from(data, "base64").toString("utf-8")
}
global.b64Encode = (data) => {
return Buffer.from(data, "utf-8").toString("base64")
}
global.nanoid = (t = 21) => crypto.getRandomValues(new Uint8Array(t)).reduce(((t, e) => t += (e &= 63) < 36 ? e.toString(36) : e < 62 ? (e - 26).toString(36).toUpperCase() : e > 62 ? "-" : "_"), "");
global.eventBus = new EventEmitter()
Array.prototype.updateFromObjectKeys = function (obj) {
this.forEach((value, index) => {
if (obj[value] !== undefined) {
this[index] = obj[value]
}
})
return this
}
global.toBoolean = (value) => {
if (typeof value === "boolean") {
return value
}
if (typeof value === "string") {
return value.toLowerCase() === "true"
}
return false
}
async function injectEnvFromInfisical() {
const envMode = "dev"
console.log(`🔑 Injecting env variables from INFISICAL in [${envMode}] mode...`)
const client = new infisical({
token: process.env.INFISICAL_TOKEN,
})
const secrets = await client.getAllSecrets({
path: process.env.INFISICAL_PATH ?? "/",
environment: envMode,
attachToProcessEnv: false,
})
// inject to process.env
secrets.forEach((secret) => {
if (!(process.env[secret.secretName])) {
process.env[secret.secretName] = secret.secretValue
}
})
}
async function handleExit(code, e) {
if (code !== 0) {
console.log(`🚫 Unexpected exit >`, code, e)
}
await global.eventBus.awaitEmit("exit", code)
return process.exit(code)
}
async function main(api) {
if (!api) {
throw new Error("API is not defined")
}
if (process.env.INFISICAL_TOKEN) {
await injectEnvFromInfisical()
}
const instance = new api()
process.on("exit", handleExit)
process.on("SIGINT", handleExit)
process.on("uncaughtException", handleExit)
process.on("unhandledRejection", handleExit)
await instance.initialize()
return instance
}
module.exports = main

View File

@ -1,7 +1,7 @@
{
"name": "linebridge",
"version": "0.18.0",
"description": "A simple, fast, and powerful REST API interface library",
"version": "0.18.2",
"description": "API Framework for RageStudio backends",
"author": "RageStudio",
"main": "./dist/client/index.js",
"scripts": {
@ -15,7 +15,8 @@
"files": [
"src/**/**",
"dist/**/**",
"./package.json"
"./package.json",
"boot"
],
"license": "MIT",
"dependencies": {
@ -25,22 +26,20 @@
"@socket.io/redis-adapter": "^8.2.1",
"@socket.io/redis-emitter": "^5.1.0",
"@socket.io/sticky": "^1.0.4",
"axios": "^1.5.1",
"axios": "^1.6.7",
"axios-retry": "3.4.0",
"cors": "2.8.5",
"express": "4.18.2",
"hyper-express": "6.5.5",
"infisical-node": "^1.5.0",
"express": "^4.18.3",
"hyper-express": "^6.14.12",
"ioredis": "^5.3.2",
"md5": "2.3.0",
"md5": "^2.3.0",
"module-alias": "2.2.2",
"morgan": "1.10.0",
"socket.io": "^4.7.2",
"socket.io": "^4.7.4",
"socket.io-client": "4.5.4",
"uuid": "3.4.0"
"uuid": "^9.0.1"
},
"devDependencies": {
"@corenode/utils": "0.28.26",
"@ragestudio/hermes": "^0.1.0",
"mocha": "^9.1.3"
}

View File

@ -1,65 +0,0 @@
export default class EventEmitter {
#events = {}
on = (eventName, listener) => {
if (!this.#events[eventName]) {
this.#events[eventName] = []
}
this.#events[eventName].push(listener)
return this
}
emit = (eventName, ...args) => {
if (!this.#events[eventName]) {
return false
}
this.#events[eventName].forEach((listener) => {
listener(...args)
})
}
off = (eventName, listener) => {
if (!this.#events[eventName]) {
return false
}
const index = this.#events[eventName].indexOf(listener)
if (index > -1) {
this.#events[eventName].splice(index, 1)
} else {
return false
}
return this
}
removeAllListeners = (eventName) => {
if (!this.#events[eventName]) {
return false
}
this.#events[eventName] = []
return this
}
awaitEmit = async (eventName, ...args) => {
if (!this.#events[eventName]) {
return false
}
await Promise.all(this.#events[eventName].map(async (listener) => {
await listener(...args)
}))
return this
}
hasEvent = (eventName) => {
return !!this.#events[eventName]
}
}

View File

@ -17,6 +17,8 @@ export default class IPCClient {
return false
}
//console.log(`[IPC:CLIENT] Received event [${event}] from [${payload.from}]`)
if (event.startsWith("ipc:exec")) {
return this.handleExecution(payload)
}
@ -90,6 +92,8 @@ export default class IPCClient {
call = async (to_service_id, command, ...args) => {
const remote_call_id = Date.now()
//console.debug(`[IPC:CLIENT] Invoking command [${command}] on service [${to_service_id}]`)
const response = await new Promise((resolve, reject) => {
try {

View File

@ -46,6 +46,8 @@ export default class IPCRouter {
return false
}
//console.log(`[IPC:ROUTER] Routing event [${event}] to service [${target}] from [${from}]`)
targetService.instance.send({
event: event,
payload: payload

View File

@ -1,4 +1,4 @@
const { EventEmitter } = require("events")
const { EventEmitter } = require("@foxify/events")
const Endpoint = require("../endpoint")
module.exports = class Controller {

View File

@ -1,24 +1,19 @@
import socketio from "socket.io"
import cluster from "node:cluster"
import redis from "ioredis"
import EventEmitter from "@foxify/events"
import { EventEmitter } from "@foxify/events"
import { createAdapter as createRedisAdapter } from "@socket.io/redis-adapter"
import { createAdapter as createClusterAdapter } from "@socket.io/cluster-adapter"
import { setupWorker } from "@socket.io/sticky"
import { Emitter } from "@socket.io/redis-emitter"
import http from "node:http"
import cluster from "node:cluster"
import RedisMap from "../../lib/redis_map"
export default class RTEngineServer {
constructor(params = {}) {
this.params = params
// servers
this.http = this.params.http ?? undefined
this.io = this.params.io ?? undefined
this.redis = this.params.redis ?? undefined
this.redisEmitter = null
@ -27,6 +22,10 @@ export default class RTEngineServer {
this.connections = null
this.users = null
if (!this.io) {
throw new Error("No io provided")
}
}
onConnect = async (socket) => {
@ -62,9 +61,9 @@ export default class RTEngineServer {
console.log(`⚙️ Awaiting authentication for client [${socket.id}]`)
if (this.params.requireAuth) {
await this.authenticateClient(socket, null, this.handleAuth ?? this.params.handleAuth)
} else if (socket.handshake.auth.token) {
await this.authenticateClient(socket, socket.handshake.auth.token, this.handleAuth ?? this.params.handleAuth)
await this.authenticateClient(socket, null, (this.params.handleAuth ?? this.handleAuth))
} else if (socket.handshake.auth.token ?? socket.handshake.query.auth) {
await this.authenticateClient(socket, (socket.handshake.auth.token ?? socket.handshake.query.auth), (this.params.handleAuth ?? this.handleAuth))
}
if (process.env.NODE_ENV === "development") {
@ -108,6 +107,9 @@ export default class RTEngineServer {
if (socket.handshake.auth.token) {
token = socket.handshake.auth.token
}
if (socket.handshake.query.auth) {
token = socket.handshake.query.auth
}
}
function err(code, message) {
@ -239,20 +241,6 @@ export default class RTEngineServer {
})
}
if (typeof this.http === "undefined") {
this.http = http.createServer()
}
if (typeof this.io === "undefined") {
this.io = new socketio.Server(this.http, {
cors: {
origin: "*",
methods: ["GET", "POST"],
credentials: true,
},
})
}
// create mappers
this.connections = new RedisMap(this.redis, {
refKey: "connections",
@ -303,7 +291,7 @@ export default class RTEngineServer {
await this.onInit()
}
console.log(`✅ RTEngine server is running on port [${process.env.LISTEN_PORT}] ${this.clusterMode ? `on clustered mode [${cluster.worker.id}]` : ""}`)
console.log(`✅ RTEngine server is running on port [${this.params.listen_port}] ${this.clusterMode ? `on clustered mode [${cluster.worker.id}]` : ""}`)
return true
}
@ -311,7 +299,9 @@ export default class RTEngineServer {
cleanUp = async () => {
console.log(`Cleaning up RTEngine server...`)
if (this.clusterMode) {
this.connections.flush(cluster.worker.id)
}
if (this.io) {
this.io.close()

View File

@ -42,7 +42,7 @@ export default {
logs: require("./middlewares/logger").default,
},
useMiddlewares: [
"cors",
//"cors",
"logs",
],
controllers: [],

View File

@ -16,6 +16,8 @@ export default class Engine {
router = express.Router()
init = async (params) => {
console.warn("⚠️ Warning: Express engine is deprecated, use HyperExpress instead!")
this.app = express()
this.http = createServer(this.app)
this.io = new socketio.Server(this.http)

View File

@ -1,15 +1,27 @@
import he from "hyper-express"
import rtengine from "../../classes/rtengine"
import SocketIO from "socket.io"
export default class Engine {
constructor(params) {
this.params = params
}
app = new he.Server()
app = new he.Server({
max_body_length: 50 * 1024 * 1024, //50MB in bytes
})
router = new he.Router()
io = null
ws = null
init = async (params) => {
this.io = new SocketIO.Server({
path: `/${params.refName}`,
})
// register 404
await this.router.any("*", (req, res) => {
return res.status(404).json({
@ -20,11 +32,75 @@ export default class Engine {
// register body parser
await this.app.use(async (req, res, next) => {
if (req.headers["content-type"]) {
if (!req.headers["content-type"].startsWith("multipart/form-data")) {
req.body = await req.urlencoded()
req.body = await req.json(req.body)
}
}
})
this.io.attachApp(this.app.uws_instance)
this.ws = global.rtengine = new rtengine({
...params,
handleAuth: params.handleWsAuth,
io: this.io,
})
}
listen = async (params) => {
if (process.env.lb_service) {
let pathOverrides = Object.keys(this.router.map).map((key) => {
return key.split("/")[1]
})
// remove duplicates
pathOverrides = [...new Set(pathOverrides)]
// remove "" and _map
pathOverrides = pathOverrides.filter((key) => {
if (key === "" || key === "_map") {
return false
}
return true
})
process.send({
type: "router:ws:register",
id: process.env.lb_service.id,
index: process.env.lb_service.index,
data: {
namespace: params.refName,
listen: {
ip: this.params.listen_ip,
port: this.params.listen_port,
},
}
})
// try to send router map to host
process.send({
type: "router:register",
id: process.env.lb_service.id,
index: process.env.lb_service.index,
data: {
router_map: this.router.map,
path_overrides: pathOverrides,
listen: {
ip: this.params.listen_ip,
port: this.params.listen_port,
},
}
})
}
listen = async () => {
await this.app.listen(this.params.listen_port)
}
close = async () => {
this.io.close()
await this.app.close()
}
}

View File

@ -1,4 +0,0 @@
module.exports = {
serverManifest: require("./serverManifest"),
internalConsole: require("./internalConsole"),
}

View File

@ -1,36 +0,0 @@
module.exports = class InternalConsole {
constructor(params = {}) {
this.params = params
}
exec = (type, ...args) => {
if (global.consoleSilent) {
return false
}
// fix unsupported types
switch (type) {
case "table": {
return console.table(...args)
}
}
if (this.params.server_name) {
args.unshift(`[${this.params.server_name}]`)
}
return console[type](...args)
}
log = (...args) => this.exec("log", ...args)
error = (...args) => this.exec("error", ...args)
warn = (...args) => this.exec("warn", ...args)
info = (...args) => this.exec("info", ...args)
debug = (...args) => this.exec("debug", ...args)
table = (...args) => this.exec("table", ...args)
}

View File

@ -1,42 +0,0 @@
const tokenizer = require("corenode/libs/tokenizer")
const path = require("path")
const fs = require("fs")
const SERVER_MANIFEST = global.SERVER_MANIFEST ?? "server.manifest"
const SERVER_MANIFEST_PATH = global.SERVER_MANIFEST_PATH ?? path.resolve(process.cwd(), SERVER_MANIFEST)
const serverManifest = {
stat: () => {
return fs.lstatSync(SERVER_MANIFEST)
},
get: (key) => {
let data = {}
if (fs.existsSync(SERVER_MANIFEST)) {
data = JSON.parse(fs.readFileSync(SERVER_MANIFEST_PATH, "utf8"))
}
if (typeof key === "string") {
return data[key]
}
return data
},
write: (mutation) => {
let data = serverManifest.get()
data = { ...data, ...mutation }
serverManifest.data = data
return fs.writeFileSync(SERVER_MANIFEST_PATH, JSON.stringify(data, null, 2), { encoding: "utf-8" })
},
create: () => {
let data = {
created: Date.now(),
server_token: tokenizer.generateOSKID()
}
serverManifest.write(data)
},
file: SERVER_MANIFEST,
filepath: SERVER_MANIFEST_PATH,
}
module.exports = serverManifest

View File

@ -2,17 +2,18 @@ export default (req, res, next) => {
const startHrTime = process.hrtime()
res.on("finish", () => {
let url = req.url
const elapsedHrTime = process.hrtime(startHrTime)
const elapsedTimeInMs = elapsedHrTime[0] * 1000 + elapsedHrTime[1] / 1e6
res._responseTimeMs = elapsedTimeInMs
// cut req.url if is too long
if (req.url.length > 100) {
req.url = req.url.substring(0, 100) + "..."
if (url.length > 100) {
url = url.substring(0, 100) + "..."
}
console.log(`${req.method} ${res._status_code ?? res.statusCode ?? 200} ${req.url} ${elapsedTimeInMs}ms`)
console.log(`${req.method} ${res._status_code ?? res.statusCode ?? 200} ${url} ${elapsedTimeInMs}ms`)
})
next()

View File

@ -54,11 +54,13 @@ class Server {
this.params.useMiddlewares = this.params.useMiddlewares ?? []
this.params.name = this.constructor.refName ?? this.params.refName
this.params.useEngine = this.constructor.useEngine ?? this.params.useEngine ?? "express"
this.params.listen_ip = this.params.listen_ip ?? "0.0.0.0"
this.params.listen_port = this.constructor.listen_port ?? this.params.listen_port ?? 3000
this.params.listen_ip = this.constructor.listenIp ?? this.constructor.listen_ip ?? this.params.listen_ip ?? "0.0.0.0"
this.params.listen_port = this.constructor.listenPort ?? this.constructor.listen_port ?? this.params.listen_port ?? 3000
this.params.http_protocol = this.params.http_protocol ?? "http"
this.params.http_address = `${this.params.http_protocol}://${defaults.localhost_address}:${this.params.listen_port}`
this.params.routesPath = this.constructor.routesPath ?? this.params.routesPath
this.params.wsRoutesPath = this.constructor.wsRoutesPath ?? this.params.wsRoutesPath
return this
}
@ -87,22 +89,26 @@ class Server {
}
}
const engineParams = {
...this.params,
handleWsAuth: this.handleWsAuth,
handleAuth: this.handleHttpAuth,
requireAuth: this.constructor.requireHttpAuth,
refName: this.constructor.refName ?? this.params.refName,
}
// initialize engine
this.engine = await loadEngine(this.params.useEngine)
this.engine = new this.engine({
...this.params,
handleAuth: this.handleHttpAuth,
requireAuth: this.constructor.requireHttpAuth,
})
this.engine = new this.engine(engineParams)
if (typeof this.engine.init === "function") {
await this.engine.init(this.params)
await this.engine.init(engineParams)
}
// create a router map
if (typeof this.engine.router.map !== "object") {
this.engine.router.map = []
this.engine.router.map = {}
}
// try to execute onInitialize hook
@ -146,7 +152,7 @@ class Server {
}
// listen
await this.engine.listen()
await this.engine.listen(engineParams)
// calculate elapsed time on ms, to fixed 2
const elapsedHrTime = process.hrtime(startHrTime)