Implement music sync room and refine related features

- Add WebSocket-based sync room for real-time music playback sync.
- Expand music exploration search to include albums and artists.
- Adjust track and release data fetching and deletion on server.
- Enhance DASH segmentation job with codec overrides and MPD updates.
- Update music service configuration for websockets and middlewares.
- Make minor UI adjustments to the search component.
This commit is contained in:
SrGooglo 2025-05-21 19:04:59 +00:00
parent 0eaecf6fd3
commit a478432d61
31 changed files with 375 additions and 176 deletions

View File

@ -184,12 +184,12 @@ const Searcher = (props) => {
if (typeof props.model === "function") {
result = await props.model(value, {
...props.modelParams,
limit_per_section: app.isMobile ? 3 : 5,
limit: app.isMobile ? 3 : 5,
})
} else {
result = await SearchModel.search(value, {
...props.modelParams,
limit_per_section: app.isMobile ? 3 : 5,
limit: app.isMobile ? 3 : 5,
})
}

View File

@ -90,7 +90,6 @@ html {
align-items: center;
border: 0;
padding: 0;
margin: 0;
@ -101,6 +100,7 @@ html {
padding: 0 10px;
background-color: var(--background-color-primary);
border: 3px solid var(--border-color) !important;
.ant-input-prefix {
font-size: 2rem;
@ -127,6 +127,9 @@ html {
flex-wrap: wrap;
width: 100%;
height: fit-content;
max-height: 70vh;
gap: 10px;
padding: 20px;
@ -137,6 +140,7 @@ html {
color: var(--text-color);
background-color: var(--background-color-primary);
border: 3px solid var(--border-color);
.ant-result,
.ant-result-title,

View File

@ -8,11 +8,19 @@ import MusicService from "@models/music"
import "./index.less"
const ListView = (props) => {
const { type, id } = props.params
const { id } = props.params
const [loading, result, error, makeRequest] = app.cores.api.useRequest(
const query = new URLSearchParams(window.location.search)
const type = query.get("type")
const service = query.get("service")
const [loading, result, error] = app.cores.api.useRequest(
MusicService.getReleaseData,
id,
{
type: type,
service: service,
},
)
if (error) {
@ -29,6 +37,8 @@ const ListView = (props) => {
return <antd.Skeleton active />
}
console.log(result)
return (
<PlaylistView
playlist={result}

View File

@ -10,7 +10,11 @@ const MusicNavbar = React.forwardRef((props, ref) => {
useUrlQuery
renderResults={false}
model={async (keywords, params) =>
SearchModel.search(keywords, params, ["tracks"])
SearchModel.search(keywords, params, [
"tracks",
"albums",
"artists",
])
}
onSearchResult={props.setSearchResults}
onEmpty={() => props.setSearchResults(false)}

View File

@ -8,11 +8,11 @@ import MusicTrack from "@components/Music/Track"
import Playlist from "@components/Music/Playlist"
const ResultGroupsDecorators = {
playlists: {
icon: "MdPlaylistPlay",
label: "Playlists",
albums: {
icon: "MdAlbum",
label: "Albums",
renderItem: (props) => {
return <Playlist key={props.key} playlist={props.item} />
return <Playlist row playlist={props.item} />
},
},
tracks: {
@ -23,7 +23,6 @@ const ResultGroupsDecorators = {
<MusicTrack
key={props.key}
track={props.item}
//onClickPlayBtn={() => app.cores.player.start(props.item)}
onClick={() => app.location.push(`/play/${props.item._id}`)}
/>
)
@ -40,6 +39,10 @@ const SearchResults = ({ data }) => {
// filter out groups with no items array property
groupsKeys = groupsKeys.filter((key) => {
if (!data[key]) {
return false
}
if (!Array.isArray(data[key].items)) {
return false
}

View File

@ -3,6 +3,10 @@ import path from "node:path"
import { FFMPEGLib, Utils } from "../FFMPEGLib"
const codecOverrides = {
wav: "flac",
}
export default class SegmentedAudioMPDJob extends FFMPEGLib {
constructor(params = {}) {
super()
@ -26,11 +30,11 @@ export default class SegmentedAudioMPDJob extends FFMPEGLib {
`-c:a ${this.params.audioCodec}`,
`-map 0:a`,
`-f dash`,
`-dash_segment_type mp4`,
`-segment_time ${this.params.segmentTime}`,
`-use_template 1`,
`-use_timeline 1`,
`-init_seg_name "init.m4s"`,
//`-dash_segment_type mp4`,
//`-init_seg_name "init.m4s"`,
]
if (this.params.includeMetadata === false) {
@ -89,25 +93,69 @@ export default class SegmentedAudioMPDJob extends FFMPEGLib {
}
}
_updateMpdBandwidthAndSamplingRate = async ({
mpdPath,
bandwidth,
samplingRate,
} = {}) => {
try {
let mpdContent = await fs.promises.readFile(mpdPath, "utf-8")
// Regex to find all <Representation ...> tags
const representationRegex = /(<Representation\b[^>]*)(>)/g
mpdContent = mpdContent.replace(
representationRegex,
(match, startTag, endTag) => {
// Remove existing bandwidth and audioSamplingRate attributes if present
let newTag = startTag
.replace(/\sbandwidth="[^"]*"/, "")
.replace(/\saudioSamplingRate="[^"]*"/, "")
// Add new attributes
newTag += ` bandwidth="${bandwidth}" audioSamplingRate="${samplingRate}"`
return newTag + endTag
},
)
await fs.promises.writeFile(mpdPath, mpdContent, "utf-8")
} catch (error) {
console.error(
`[SegmentedAudioMPDJob] Error updating MPD bandwidth/audioSamplingRate for ${mpdPath}:`,
error,
)
}
}
run = async () => {
const segmentationCmd = this.buildSegmentationArgs()
const outputPath =
this.params.outputDir ?? `${path.dirname(this.params.input)}/dash`
const outputFile = path.join(outputPath, this.params.outputMasterName)
this.emit("start", {
input: this.params.input,
output: outputPath,
params: this.params,
})
if (!fs.existsSync(outputPath)) {
fs.mkdirSync(outputPath, { recursive: true })
}
try {
this.emit("start", {
input: this.params.input,
output: outputPath,
params: this.params,
})
const inputProbe = await Utils.probe(this.params.input)
if (
this.params.audioCodec === "copy" &&
codecOverrides[inputProbe.format.format_name]
) {
this.params.audioCodec =
codecOverrides[inputProbe.format.format_name]
}
const segmentationCmd = this.buildSegmentationArgs()
if (!fs.existsSync(outputPath)) {
fs.mkdirSync(outputPath, { recursive: true })
}
const ffmpegResult = await this.ffmpeg({
args: segmentationCmd,
onProcess: (process) => {
@ -135,6 +183,29 @@ export default class SegmentedAudioMPDJob extends FFMPEGLib {
let outputProbe = await Utils.probe(outputFile)
let bandwidth = null
let samplingRate = null
if (
outputProbe &&
outputProbe.streams &&
outputProbe.streams.length > 0
) {
bandwidth =
outputProbe.format.bit_rate ??
outputProbe.streams[0].bit_rate
samplingRate = outputProbe.streams[0].sample_rate
}
if (bandwidth && samplingRate) {
await this._updateMpdBandwidthAndSamplingRate({
mpdPath: outputFile,
bandwidth: bandwidth,
samplingRate: samplingRate,
})
}
this.emit("end", {
probe: {
input: inputProbe,

View File

@ -0,0 +1,35 @@
export class SyncRoom {
constructor(ownerSocket) {
this.ownerSocket = ownerSocket
}
id = global.nanoid()
buffer = new Set()
members = new Set()
push = async (data) => {
if (this.buffer.size > 5) {
this.buffer.delete(this.buffer.keys().next().value)
}
this.buffer.add(data)
for (const socket of this.members) {
socket.emit(`syncroom:push`, data)
}
}
join = (socket) => {
this.members.add(socket)
// send the latest buffer
socket.emit("syncroom.buffer", this.buffer[0])
}
leave = (socket) => {
this.members.delete(socket)
}
}
export default class SyncRoomManager {}

View File

@ -2,7 +2,7 @@ import path from "node:path"
import SegmentedAudioMPDJob from "@shared-classes/SegmentedAudioMPDJob"
export default async ({ filePath, workPath, onProgress }) => {
return new Promise(async (resolve, reject) => {
return new Promise((resolve) => {
const outputDir = path.resolve(workPath, "a-dash")
const job = new SegmentedAudioMPDJob({
@ -10,7 +10,7 @@ export default async ({ filePath, workPath, onProgress }) => {
outputDir: outputDir,
// set to default as raw flac
audioCodec: "flac",
audioCodec: "copy",
audioBitrate: "default",
audioSampleRate: "default",
})

View File

@ -130,7 +130,7 @@ export default class Release {
const items = release.items ?? release.list
const items_ids = items.map((item) => item._id.toString())
const items_ids = items.map((item) => item._id ?? item)
// delete all releated tracks
await Track.deleteMany({

View File

@ -6,12 +6,12 @@ async function fullfillData(list, { user_id = null }) {
list = [list]
}
const trackIds = list.map((track) => {
return track._id
})
// if user_id is provided, fetch likes
if (user_id) {
const trackIds = list.map((track) => {
return track._id
})
const tracksLikes = await Library.isFavorite(
user_id,
trackIds,
@ -32,21 +32,15 @@ async function fullfillData(list, { user_id = null }) {
})
list = await Promise.all(list)
} else {
list = list.map((track) => {
delete track.source
delete track.publisher
return track
})
}
// process some metadata
list = list.map(async (track) => {
if (track.metadata) {
if (track.metadata.bitrate && track.metadata.bitrate > 9000) {
track.metadata.lossless = true
}
}
return track
})
list = await Promise.all(list)
return list
}

View File

@ -7,12 +7,20 @@ import RedisClient from "@shared-classes/RedisClient"
import SharedMiddlewares from "@shared-middlewares"
import LimitsClass from "@shared-classes/Limits"
import InjectedAuth from "@shared-lib/injectedAuth"
export default class API extends Server {
static refName = "music"
static useEngine = "hyper-express-ng"
static enableWebsockets = true
static routesPath = `${__dirname}/routes`
static listen_port = process.env.HTTP_LISTEN_PORT ?? 3003
static listenPort = process.env.HTTP_LISTEN_PORT ?? 3003
static websockets = {
enabled: true,
path: "/music",
}
static bypassCors = true
static useMiddlewares = ["logs"]
middlewares = {
...SharedMiddlewares,
@ -24,9 +32,28 @@ export default class API extends Server {
redis: RedisClient(),
}
handleWsUpgrade = async (context, token, res) => {
if (!token) {
return res.upgrade(context)
}
context = await InjectedAuth(context, token, res).catch(() => {
res.close(401, "Failed to verify auth token")
return false
})
if (!context || !context.user) {
res.close(401, "Unauthorized or missing auth token")
return false
}
return res.upgrade(context)
}
async onInitialize() {
global.sse = this.contexts.SSEManager
global.redis = this.contexts.redis.client
global.syncRoomLyrics = new Map()
await this.contexts.db.initialize()
await this.contexts.redis.initialize()

View File

@ -1,3 +1,6 @@
{
"name": "music"
"name": "music",
"dependencies": {
"linebridge": "^1.0.0-alpha.4"
}
}

View File

@ -1,7 +1,7 @@
import Library from "@classes/library"
export default {
middlewares: ["withAuthentication"],
useMiddlewares: ["withAuthentication"],
fn: async (req) => {
const { kind, item_id } = req.query

View File

@ -1,7 +1,7 @@
import Library from "@classes/library"
export default {
middlewares: ["withAuthentication"],
useMiddlewares: ["withAuthentication"],
fn: async (req) => {
const { kind, item_id, to } = req.body

View File

@ -1,7 +1,7 @@
import Library from "@classes/library"
export default {
middlewares: ["withAuthentication"],
useMiddlewares: ["withAuthentication"],
fn: async (req) => {
const userId = req.auth.session.user_id
const { limit = 50, offset = 0, kind } = req.query

View File

@ -1,7 +1,7 @@
import { MusicRelease, Track } from "@db_models"
export default {
middlewares: ["withAuthentication"],
useMiddlewares: ["withAuthentication"],
fn: async (req) => {
const { keywords, limit = 10, offset = 0 } = req.query

View File

@ -3,7 +3,7 @@ import { RecentActivity } from "@db_models"
import TrackClass from "@classes/track"
export default {
middlewares: ["withAuthentication"],
useMiddlewares: ["withAuthentication"],
fn: async (req, res) => {
const user_id = req.auth.session.user_id

View File

@ -1,7 +1,7 @@
import ReleaseClass from "@classes/release"
export default {
middlewares: ["withOptionalAuthentication"],
useMiddlewares: ["withOptionalAuthentication"],
fn: async (req) => {
const { release_id } = req.params
const { limit = 50, offset = 0 } = req.query

View File

@ -1,7 +1,7 @@
import ReleaseClass from "@classes/release"
export default {
middlewares: ["withAuthentication"],
useMiddlewares: ["withAuthentication"],
fn: async (req) => {
return await ReleaseClass.delete(req.params.release_id, {
user_id: req.auth.session.user_id,

View File

@ -1,18 +1,18 @@
import ReleaseClass from "@classes/release"
export default {
middlewares: ["withAuthentication"],
fn: async (req) => {
if (req.body._id) {
return await ReleaseClass.update(req.body._id, {
...req.body,
user_id: req.auth.session.user_id,
})
} else {
return await ReleaseClass.create({
...req.body,
user_id: req.auth.session.user_id,
})
}
}
}
useMiddlewares: ["withAuthentication"],
fn: async (req) => {
if (req.body._id) {
return await ReleaseClass.update(req.body._id, {
...req.body,
user_id: req.auth.session.user_id,
})
} else {
return await ReleaseClass.create({
...req.body,
user_id: req.auth.session.user_id,
})
}
},
}

View File

@ -1,15 +1,15 @@
import TrackClass from "@classes/track"
export default {
middlewares: ["withOptionalAuthentication"],
fn: async (req) => {
const { track_id } = req.params
const user_id = req.auth?.session?.user_id
useMiddlewares: ["withOptionalAuthentication"],
fn: async (req) => {
const { track_id } = req.params
const user_id = req.auth?.session?.user_id
const track = await TrackClass.get(track_id, {
user_id
})
const track = await TrackClass.get(track_id, {
user_id,
})
return track
}
}
return track
},
}

View File

@ -1,21 +1,21 @@
import TrackClass from "@classes/track"
export default {
middlewares: ["withAuthentication"],
fn: async (req) => {
const { track_id } = req.params
useMiddlewares: ["withAuthentication"],
fn: async (req) => {
const { track_id } = req.params
const track = await TrackClass.get(track_id)
const track = await TrackClass.get(track_id)
if (track.publisher.user_id !== req.auth.session.user_id) {
throw new Error("Forbidden, you don't own this track")
}
if (track.publisher.user_id !== req.auth.session.user_id) {
throw new Error("Forbidden, you don't own this track")
}
await TrackClass.delete(track_id)
await TrackClass.delete(track_id)
return {
success: true,
track: track,
}
}
}
return {
success: true,
track: track,
}
},
}

View File

@ -1,66 +1,66 @@
import { TrackLyric, Track } from "@db_models"
export default {
middlewares: ["withAuthentication"],
fn: async (req) => {
const { track_id } = req.params
const { video_source, lrc, sync_audio_at } = req.body
useMiddlewares: ["withAuthentication"],
fn: async (req) => {
const { track_id } = req.params
const { video_source, lrc, sync_audio_at } = req.body
// check if track exists
let track = await Track.findById(track_id).catch(() => null)
// check if track exists
let track = await Track.findById(track_id).catch(() => null)
if (!track) {
throw new OperationError(404, "Track not found")
}
if (!track) {
throw new OperationError(404, "Track not found")
}
if (track.publisher.user_id !== req.auth.session.user_id) {
throw new OperationError(403, "Unauthorized")
}
if (track.publisher.user_id !== req.auth.session.user_id) {
throw new OperationError(403, "Unauthorized")
}
console.log(`Setting lyrics for track ${track_id} >`, {
track_id: track_id,
video_source: video_source,
lrc: lrc,
})
console.log(`Setting lyrics for track ${track_id} >`, {
track_id: track_id,
video_source: video_source,
lrc: lrc,
})
// check if trackLyric exists
let trackLyric = await TrackLyric.findOne({
track_id: track_id
})
// check if trackLyric exists
let trackLyric = await TrackLyric.findOne({
track_id: track_id,
})
// if trackLyric exists, update it, else create it
if (!trackLyric) {
trackLyric = new TrackLyric({
track_id: track_id,
video_source: video_source,
lrc: lrc,
sync_audio_at: sync_audio_at,
})
// if trackLyric exists, update it, else create it
if (!trackLyric) {
trackLyric = new TrackLyric({
track_id: track_id,
video_source: video_source,
lrc: lrc,
sync_audio_at: sync_audio_at,
})
await trackLyric.save()
} else {
const update = Object()
await trackLyric.save()
} else {
const update = Object()
if (typeof video_source !== "undefined") {
update.video_source = video_source
}
if (typeof video_source !== "undefined") {
update.video_source = video_source
}
if (typeof lrc !== "undefined") {
update.lrc = lrc
}
if (typeof lrc !== "undefined") {
update.lrc = lrc
}
if (typeof sync_audio_at !== "undefined") {
update.sync_audio_at = sync_audio_at
}
if (typeof sync_audio_at !== "undefined") {
update.sync_audio_at = sync_audio_at
}
trackLyric = await TrackLyric.findOneAndUpdate(
{
track_id: track_id,
},
update,
)
}
trackLyric = await TrackLyric.findOneAndUpdate(
{
track_id: track_id,
},
update,
)
}
return trackLyric
}
}
return trackLyric
},
}

View File

@ -1,36 +1,36 @@
import { TrackOverride } from "@db_models"
export default {
middlewares: ["withAuthentication", "onlyAdmin"],
fn: async (req) => {
const { track_id } = req.params
const { service, override } = req.body
useMiddlewares: ["withAuthentication", "onlyAdmin"],
fn: async (req) => {
const { track_id } = req.params
const { service, override } = req.body
let trackOverride = await TrackOverride.findOne({
track_id: track_id,
service: service,
}).catch(() => null)
let trackOverride = await TrackOverride.findOne({
track_id: track_id,
service: service,
}).catch(() => null)
if (!trackOverride) {
trackOverride = new TrackOverride({
track_id: track_id,
service: service,
override: override,
})
if (!trackOverride) {
trackOverride = new TrackOverride({
track_id: track_id,
service: service,
override: override,
})
await trackOverride.save()
} else {
trackOverride = await TrackOverride.findOneAndUpdate(
{
track_id: track_id,
service: service,
},
{
override: override,
},
)
}
await trackOverride.save()
} else {
trackOverride = await TrackOverride.findOneAndUpdate(
{
track_id: track_id,
service: service,
},
{
override: override,
},
)
}
return trackOverride.override
}
}
return trackOverride.override
},
}

View File

@ -19,6 +19,7 @@ export default async (req) => {
const items = await Track.find(query)
.limit(limit)
.select("-source -publisher -public")
.skip(trim)
.sort({ _id: -1 })

View File

@ -2,7 +2,7 @@ import requiredFields from "@shared-utils/requiredFields"
import TrackClass from "@classes/track"
export default {
middlewares: ["withAuthentication"],
useMiddlewares: ["withAuthentication"],
fn: async (req) => {
if (Array.isArray(req.body.items)) {
let results = []

View File

@ -0,0 +1,13 @@
import leave from "./leave"
export default async (client, user_id) => {
console.log(`[SYNC-ROOM] Join ${client.userId} -> ${user_id}`)
if (client.syncroom) {
await leave(client, client.syncroom)
}
// subscribe to stream topic
await client.subscribe(`syncroom/${user_id}`)
client.syncroom = user_id
}

View File

@ -0,0 +1,8 @@
export default async (client, user_id) => {
console.log(`[SYNC-ROOM] Leave ${client.userId} -> ${user_id}`)
// unsubscribe from sync topic
await client.unsubscribe(`syncroom/${user_id}`)
client.syncroom = null
}

View File

@ -0,0 +1,7 @@
export default async (client, payload) => {
console.log(`[SYNC-ROOM] Pushing to sync ${client.userId}`, payload)
const roomId = `syncroom/${client.userId}`
global.websockets.senders.toTopic(roomId, "sync:receive", payload)
}

View File

@ -0,0 +1,14 @@
export default async (client, payload) => {
console.log(`[SYNC-ROOM] Pushing lyrics to sync ${client.userId}`)
const roomId = `syncroom/${client.userId}`
if (!payload) {
// delete lyrics
global.syncRoomLyrics.delete(client.userId)
} else {
global.syncRoomLyrics.set(client.userId, payload)
}
global.websockets.senders.toTopic(roomId, "sync:lyrics:receive", payload)
}

View File

@ -0,0 +1,5 @@
export default async (client) => {
console.log(`[SYNC-ROOM] Requesting lyrics of room ${client.syncroom}`)
return global.syncRoomLyrics.get(client.syncroom)
}