From 078d418684d8ec50662d68a076ea7cda70805253 Mon Sep 17 00:00:00 2001 From: SrGooglo Date: Thu, 25 May 2023 01:18:22 +0000 Subject: [PATCH] improve sync by updating remote state --- packages/app/src/cores/player/index.js | 41 ++++++++++-- packages/app/src/cores/sync/index.js | 44 +++++++++++-- packages/music_server/src/roomsServer.js | 83 ++++++++++++++++++++++-- 3 files changed, 154 insertions(+), 14 deletions(-) diff --git a/packages/app/src/cores/player/index.js b/packages/app/src/cores/player/index.js index b912ebee..d484e739 100755 --- a/packages/app/src/cores/player/index.js +++ b/packages/app/src/cores/player/index.js @@ -157,6 +157,7 @@ export default class Player extends Core { velocity: this.velocity.bind(this), close: this.close.bind(this), toogleSyncMode: this.toogleSyncMode.bind(this), + currentState: this.currentState.bind(this), } async initializeAudioProcessors() { @@ -212,7 +213,10 @@ export default class Player extends Core { app.eventBus.emit("player.loading.update", change.object.loading) if (this.state.syncMode) { - useMusicSync("music:player:loading", change.object.loading) + useMusicSync("music:player:loading", { + loading: change.object.loading, + state: this.currentState() + }) } break @@ -236,7 +240,8 @@ export default class Player extends Core { if (this.state.syncMode) { useMusicSync("music:player:start", { - manifest: change.object.currentAudioManifest + manifest: change.object.currentAudioManifest, + state: this.currentState() }) } @@ -290,6 +295,7 @@ export default class Player extends Core { time: this.currentAudioInstance.audioElement.currentTime, duration: this.currentAudioInstance.audioElement.duration, startingNew: this.state.startingNew, + state: this.currentState(), }) } @@ -437,6 +443,9 @@ export default class Player extends Core { // stop playback if (this.currentAudioInstance.audioElement) { + this.currentAudioInstance.audioElement.srcObj = null + this.currentAudioInstance.audioElement.src = null + // if is in sync mode, just seek to last position to stop playback and avoid sync issues this.currentAudioInstance.audioElement.pause() } @@ -560,7 +569,8 @@ export default class Player extends Core { if (this.state.syncMode) { useMusicSync("music:player:seek", { - position: instanceObj.audioElement.currentTime + position: instanceObj.audioElement.currentTime, + state: this.currentState(), }) } }) @@ -655,6 +665,13 @@ export default class Player extends Core { instance.audioElement.muted = this.state.audioMuted + console.log("Playing audio", instance.audioElement.src) + + // reconstruct audio src if is not set + if (instance.audioElement.src !== instance.manifest.source) { + instance.audioElement.src = instance.manifest.source + } + instance.audioElement.load() instance.audioElement.play() @@ -715,7 +732,7 @@ export default class Player extends Core { this.play(this.audioQueue[0]) } - async start(manifest, { sync = false } = {}) { + async start(manifest, { sync = false, time } = {}) { if (this.state.syncModeLocked && !sync) { console.warn("Sync mode is locked, cannot do this action") return false @@ -738,7 +755,9 @@ export default class Player extends Core { this.state.loading = true - this.play(this.audioQueue[0]) + this.play(this.audioQueue[0], { + time: time ?? 0 + }) } next({ sync = false } = {}) { @@ -991,4 +1010,16 @@ export default class Player extends Core { return this.state.syncMode } + + currentState() { + return { + playbackStatus: this.state.playbackStatus, + manifest: this.currentAudioInstance?.manifest ?? null, + loading: this.state.loading, + time: this.seek(), + duration: this.currentAudioInstance?.audioElement?.duration ?? null, + audioMuted: this.state.audioMuted, + audioVolume: this.state.audioVolume, + } + } } \ No newline at end of file diff --git a/packages/app/src/cores/sync/index.js b/packages/app/src/cores/sync/index.js index 857f861c..20ff59a6 100644 --- a/packages/app/src/cores/sync/index.js +++ b/packages/app/src/cores/sync/index.js @@ -67,6 +67,8 @@ class MusicSyncSubCore { // check if user is owner app.cores.player.toogleSyncMode(true, data.ownerUserId !== app.userData._id) + this.startSendStateInterval() + this.eventBus.emit("room:joined", data) }, "room:left": (data) => { @@ -120,7 +122,7 @@ class MusicSyncSubCore { }, // Room Control "music:player:start": (data) => { - if (data.selfUser.user_id === app.userData._id) { + if (data.command_issuer === app.userData._id) { return false } @@ -128,10 +130,11 @@ class MusicSyncSubCore { app.cores.player.start(data.manifest, { sync: true, + time: data.time, }) }, "music:player:seek": (data) => { - if (data.selfUser.user_id === app.userData._id) { + if (data.command_issuer === app.userData._id) { return false } @@ -142,13 +145,14 @@ class MusicSyncSubCore { }) }, "music:player:status": (data) => { - if (data.selfUser.user_id === app.userData._id) { + if (data.command_issuer === app.userData._id) { return false } // avoid dispatch if event pause and current time is the audio duration if (data.startingNew || data.status === "paused" && data.time === data.duration) { - return app.cores.player.stop() + //return app.cores.player.playback.stop() + return false } switch (data.status) { @@ -190,6 +194,33 @@ class MusicSyncSubCore { }) } + startSendStateInterval() { + if (this.sendStateInterval) { + clearInterval(this.sendStateInterval) + } + + this.firstStateSent = true + + this.sendStateInterval = setInterval(() => { + if (!this.currentRoomData) { + return false + } + + let state = app.cores.player.currentState() + + console.log("state", state) + + this.musicWs.emit("music:state:update", { + ...state, + firstSync: this.firstStateSent + }) + + if (this.firstStateSent) { + this.firstStateSent = false + } + }, 2000) + } + dispatchEvent(eventName, data) { if (!eventName) { return false @@ -254,6 +285,11 @@ class MusicSyncSubCore { this.currentRoomData = null + if (this.sendStateInterval) { + this.firstStateSent = false + clearInterval(this.sendStateInterval) + } + Object.keys(this.roomEvents).forEach((eventName) => { this.musicWs.off(eventName, this.roomEvents[eventName]) }) diff --git a/packages/music_server/src/roomsServer.js b/packages/music_server/src/roomsServer.js index 3e646371..34ff5f99 100644 --- a/packages/music_server/src/roomsServer.js +++ b/packages/music_server/src/roomsServer.js @@ -24,14 +24,15 @@ function generateFnHandler(fn, socket) { } } -function composePayloadData(socket, data) { +function composePayloadData(socket, data = {}) { return { - selfUser: { + user: { user_id: socket.userData._id, username: socket.userData.username, fullName: socket.userData.fullName, avatar: socket.userData.avatar, }, + command_issuer: data.command_issuer ?? socket.userData._id, ...data } } @@ -47,6 +48,9 @@ class Room { this.roomOptions = roomOptions } + // declare the maximum audio offset from owner + static maxOffsetFromOwner = 1 + ownerUserId = null connections = [] @@ -65,6 +69,10 @@ class Room { return false } + if (data.state) { + this.currentState = data.state + } + this.io.to(this.roomId).emit("music:player:start", composePayloadData(socket, data)) }, "music:player:seek": (socket, data) => { @@ -74,21 +82,86 @@ class Room { return false } + if (data.state) { + this.currentState = data.state + } + this.io.to(this.roomId).emit("music:player:seek", composePayloadData(socket, data)) }, - "music:player:loading": () => { + "music:player:loading": (socket, data) => { // TODO: Softmode and Hardmode + // Ignore if is the owner + if (socket.userData._id === this.ownerUserId) { + return false + } - // sync with current state, seek if needed (if is not owner) + // if not loading, check if need to sync + if (!data.loading) { + // try to sync with current state + if (data.state.time > this.currentState.time + Room.maxOffsetFromOwner) { + socket.emit("music:player:seek", composePayloadData(socket, { + position: this.currentState.time, + command_issuer: this.ownerUserId, + })) + } + } }, "music:player:status": (socket, data) => { if (socket.userData._id !== this.ownerUserId) { return false } + if (data.state) { + this.currentState = data.state + } + this.io.to(this.roomId).emit("music:player:status", composePayloadData(socket, data)) }, - // ROOM CONTROL + // UPDATE TICK + "music:state:update": (socket, data) => { + if (socket.userData._id === this.ownerUserId) { + // update current state + this.currentState = data + + return true + } + + if (!this.currentState) { + return false + } + + if (data.loading) { + return false + } + + // check if match with current manifest + if (!data.manifest || data.manifest._id !== this.currentState.manifest._id) { + socket.emit("music:player:start", composePayloadData(socket, { + manifest: this.currentState.manifest, + time: this.currentState.time, + command_issuer: this.ownerUserId, + })) + } + + if (data.firstSync) { + // if not owner, try to sync with current state + if (data.time > this.currentState.time + Room.maxOffsetFromOwner) { + socket.emit("music:player:seek", composePayloadData(socket, { + position: this.currentState.time, + command_issuer: this.ownerUserId, + })) + } + + // check if match with current playing status + if (data.playbackStatus !== this.currentState.playbackStatus && data.firstSync) { + socket.emit("music:player:status", composePayloadData(socket, { + status: this.currentState.playbackStatus, + command_issuer: this.ownerUserId, + })) + } + } + }, + // ROOM MODERATION CONTROL "room:moderation:kick": (socket, data) => { if (socket.userData._id !== this.ownerUserId) { return socket.emit("error", {