diff options
Diffstat (limited to 'src/server')
| -rw-r--r-- | src/server/HttpServer.hx | 26 | ||||
| -rw-r--r-- | src/server/Main.hx | 243 | ||||
| -rw-r--r-- | src/server/Utils.hx | 2 | ||||
| -rw-r--r-- | src/server/VideoTimer.hx | 4 | ||||
| -rw-r--r-- | src/server/cache/Cache.hx | 39 | ||||
| -rw-r--r-- | src/server/cache/CacheCallbacks.hx | 15 | ||||
| -rw-r--r-- | src/server/cache/RawCache.hx | 158 | ||||
| -rw-r--r-- | src/server/cache/YoutubeCache.hx | 156 | ||||
| -rw-r--r-- | src/server/cache/YtdlpCache.hx | 227 |
9 files changed, 633 insertions, 237 deletions
diff --git a/src/server/HttpServer.hx b/src/server/HttpServer.hx index bcbf9aa..7bd42f4 100644 --- a/src/server/HttpServer.hx +++ b/src/server/HttpServer.hx @@ -22,6 +22,7 @@ private class HttpServerConfig { public final dir:String; public final customDir:String = null; public final allowLocalRequests = false; + public final enableCORS = false; public final cache:Cache = null; } @@ -68,6 +69,7 @@ class HttpServer { final hasCustomRes = false; final allowedLocalFiles:Map<String, Bool> = []; final allowLocalRequests = false; + final enableCORS = false; final cache:Cache = null; final CHUNK_SIZE = 1024 * 1024 * 5; // 5 MB // temp media data while file is uploading to allow instant streaming @@ -79,6 +81,7 @@ class HttpServer { dir = config.dir; customDir = config.customDir; allowLocalRequests = config.allowLocalRequests; + enableCORS = config.enableCORS; cache = config.cache; if (customDir != null) hasCustomRes = FileSystem.exists(customDir); @@ -93,6 +96,18 @@ class HttpServer { var filePath = getPath(dir, url); final ext = Path.extension(filePath).toLowerCase(); + if (enableCORS) { + res.setHeader("access-control-allow-origin", "*"); + res.setHeader("access-control-allow-headers", "content-type, range, content-name, x-client-uuid"); + res.setHeader("access-control-expose-headers", "accept-ranges, content-encoding, content-length, content-range"); + if (req.method == "OPTIONS") { + res.setHeader("access-control-allow-methods", "GET, HEAD, POST, OPTIONS"); + res.statusCode = 204; + res.end(); + return; + } + } + res.setHeader("accept-ranges", "bytes"); res.setHeader("content-type", getMimeType(ext)); @@ -255,7 +270,6 @@ class HttpServer { info: cache.notEnoughSpaceErrorText, errorId: "freeSpace", }); - cache.remove(name); req.destroy(); final client = main.clients.getByName(name) ?? return; main.serverMessage(client, cache.notEnoughSpaceErrorText); @@ -453,10 +467,16 @@ class HttpServer { } function parseRangeHeader(rangeHeader:String, videoSize:Int):{start:Int, end:Int} { - final ranges = ~/[-=]/g.split(rangeHeader); + final ranges = ~/[=-]/g.split(rangeHeader); var start = Std.parseInt(ranges[1]); - if (Utils.isOutOfRange(start, 0, videoSize - 1)) start = 0; var end = Std.parseInt(ranges[2]); + // handle file end requests like `bytes=-500`: ["bytes","","500"] + if (start == null) { + final suffix = end ?? 0; + start = (videoSize - suffix).limitMin(0); + end = videoSize - 1; + } + if (Utils.isOutOfRange(start, 0, videoSize - 1)) start = 0; if (end == null) end = start + CHUNK_SIZE; if (Utils.isOutOfRange(end, start, videoSize - 1)) end = (videoSize - 1).limitMin(0); return { diff --git a/src/server/Main.hx b/src/server/Main.hx index 12a3bfa..b55f9b0 100644 --- a/src/server/Main.hx +++ b/src/server/Main.hx @@ -25,6 +25,7 @@ import js.npm.ws.WebSocket; import json2object.ErrorUtils; import json2object.JsonParser; import server.cache.Cache; +import server.cache.CacheCallbacks; import sys.FileSystem; import sys.io.File; import utils.macro.Macro; @@ -81,7 +82,6 @@ class Main { to stopped server time. **/ var emptyRoomCallbackTimer:Null<Timer>; - var isServerPause = false; var activeSkipVoteUrl:Null<String>; final skipVotes:Map<Int, Bool> = []; @@ -135,7 +135,11 @@ class Main { consoleInput = new ConsoleInput(this); consoleInput.initConsoleInput(); cache = new Cache(this, cacheDir); - if (cache.isYtReady) playersCacheSupport.push(YoutubeType); + if (cache.isYtReady) { + playersCacheSupport.push(YoutubeType); + playersCacheSupport.push(VkType); + playersCacheSupport.push(VimeoType); + } initIntergationHandlers(); loadState(); cache.setStorageLimit(cast config.cacheStorageLimitGiB * 1024 * 1024 * 1024); @@ -203,6 +207,7 @@ class Main { dir: dir, customDir: '$userDir/res', allowLocalRequests: config.localAdmins, + enableCORS: config.enableCORS, cache: cache, }); Lang.init('$dir/langs'); @@ -302,6 +307,7 @@ class Main { if (isNoState) return config; final customPath = '$userDir/config.json'; if (!FileSystem.exists(customPath)) return config; + final defaultYtDlp = config.ytDlp; final customConfig:ServerConfig = Json.parse(File.getContent(customPath)); for (field in Reflect.fields(customConfig)) { if (Reflect.field(config, field) == null) { @@ -309,6 +315,8 @@ class Main { } Reflect.setField(config, field, Reflect.field(customConfig, field)); } + config.ytDlp.channel ??= defaultYtDlp.channel; + config.ytDlp.jsRuntime ??= defaultYtDlp.jsRuntime; final emoteCopies:Map<String, Bool> = []; for (emote in config.emotes) { if (emoteCopies[emote.name]) trace('Warning: emote name "${emote.name}" has copy'); @@ -567,7 +575,7 @@ class Main { if (!internal) return; emptyRoomCallbackTimer?.stop(); if (clients.length == 1 && videoList.length > 0) { - if (!isServerPause) { + if (!videoTimer.isServerPause) { if (videoTimer.isPaused()) videoTimer.play(); } } @@ -590,6 +598,7 @@ class Main { } }); sendClientListExcept(client); + broadcastSkipVoteStats(); case Disconnected: if (!internal) return; @@ -600,9 +609,11 @@ class Main { if (client.isLeader) { if (videoList.length > 0) { videoTimer.pause(); - isServerPause = true; + videoTimer.isServerPause = true; } } + skipVotes.remove(client.id); + updateSkipVote(); if (clients.length == 0) { emptyRoomCallbackTimer?.stop(); emptyRoomCallbackTimer = Timer.delay(() -> { @@ -697,6 +708,7 @@ class Main { } }); sendClientListExcept(client); + broadcastSkipVoteStats(); case PasswordRequest: case LoginError: @@ -715,6 +727,7 @@ class Main { } }); sendClientListExcept(client); + broadcastSkipVoteStats(); case Message: if (!checkPermission(client, WriteChatPerm)) return; @@ -734,6 +747,20 @@ class Main { case ServerMessage: case Progress: + final p = data.progress; + final url = p.url ?? return; + final index = videoList.findIndex(i -> i.url == url); + if (index == -1) return; + final item = videoList.getItem(index); + if (item.isIncomplete != true) return; + if (item.doCache) return; + if (item.author != client.name) return; + switch p.type { + case Completed: completeIncompleteItem(url); + case Canceled: removeIncompleteItem(url); + case _: broadcast(data); + } + case AddVideo: if (isPlaylistLockedFor(client)) return; if (!checkPermission(client, AddVideoPerm)) return; @@ -766,7 +793,7 @@ class Main { videoList.addItem(item, data.addVideo.atEnd); broadcast(data); // Initial timer start if VideoLoaded is not happen - if (videoList.length == 1) restartWaitTimer(); + if (videoList.length == 1 && item.isPlayable()) restartWaitTimer(); } if (item.doCache && !client.isAdmin) { item.doCache = false; @@ -774,34 +801,45 @@ class Main { } if (!item.doCache) { addVideo(); - } else { - switch item.playerType { - case RawType: - cache.cacheRawVideo(client, item.url, (name) -> { - item = item.withUrl(cache.getFileUrl(name)); - addVideo(); - }); - case YoutubeType: - cache.cacheYoutubeVideo(client, item.url, (name) -> { - item = item.withUrl(cache.getFileUrl(name)); - if (item.duration > 1) item.duration -= 1; - addVideo(); - }); - case type: - final name = '$type'.replace("Type", ""); - serverMessage(client, 'No cache support for $name player.'); - addVideo(); - } + return; + } + final isYoutube = item.playerType == YoutubeType; + if (!playersCacheSupport.contains(item.playerType)) { + final name = '${item.playerType}'.replace("Type", ""); + serverMessage(client, 'No cache support for $name player.'); + addVideo(); + return; + } + final callbacks:CacheCallbacks = { + onResolved: finalUrl -> { + item = item.withUrl(finalUrl); + item.isIncomplete = true; + if (isYoutube && item.duration > 1) item.duration -= 1; + addVideo(); + }, + onProgress: ratio -> broadcastCacheProgress(item.url, ratio), + onComplete: () -> completeIncompleteItem(item.url), + onError: () -> removeIncompleteItem(item.url), + registerCancel: cancel -> cacheCancelers[item.url] = cancel, + onMetadata: (title, duration) -> + updateIncompleteItemMeta(item.url, title, duration), + }; + switch (item.playerType) { + case YoutubeType: + cache.cacheYoutubeVideo(client, item.url, callbacks); + case VkType, VimeoType: + cache.cacheYtdlpVideo(client, item.url, callbacks); + case _: + cache.cacheRawVideo(client, item.url, callbacks); } case VideoLoaded: // Called if client loads next video and can play it - if (isServerPause) return; + if (videoTimer.isServerPause) return; prepareVideoPlayback(); case RemoveVideo: if (videoList.length == 0) return; - if (activeSkipVoteUrl == data.removeVideo.url) resetSkipVote(); if (isPlaylistLockedFor(client)) return; if (!checkPermission(client, RemoveVideoPerm)) return; final url = data.removeVideo.url; @@ -809,17 +847,27 @@ class Main { if (index == -1) return; final isCurrent = videoList.currentItem.url == url; + if (isCurrent) resetSkipVote(); if (isCurrent && videoTimer.getTime() > FLASHBACK_DIST) { saveFlashbackTime(videoList.currentItem); } videoList.removeItem(index); + cancelCache(url); broadcast(data); - if (isCurrent && videoList.length > 0) restartWaitTimer(); + if (isCurrent && videoList.length > 0) { + if (!videoList.currentItem.isPlayable()) pauseForIncompleteItem(); + else restartWaitTimer(); + } + if (videoList.length == 0) { + videoTimer.stop(); + } case SkipVideo: if (!checkPermission(client, RemoveVideoPerm)) return; handleSkipVote(client, data); + case SkipVideoStats: + case Pause: if (videoList.length == 0) return; if (!client.isLeader) return; @@ -840,7 +888,6 @@ class Main { saveFlashbackTime(videoList.currentItem); } videoTimer.setTime(data.play.time); - isServerPause = false; videoTimer.play(); broadcast({ type: data.type, @@ -849,6 +896,7 @@ class Main { case GetTime: if (videoList.length == 0) return; + if (videoList.currentItem.isIncomplete) return; final maxTime = videoList.currentItem.duration - 0.01; if (videoTimer.getTime() > maxTime) { videoTimer.pause(); @@ -871,7 +919,7 @@ class Main { } }; if (videoTimer.isPaused()) obj.getTime.paused = true; - if (isServerPause) obj.getTime.pausedByServer = true; + if (videoTimer.isServerPause) obj.getTime.pausedByServer = true; if (videoTimer.getRate() != 1) { if (!clients.hasLeader()) videoTimer.setRate(1); else obj.getTime.rate = videoTimer.getRate(); @@ -944,7 +992,7 @@ class Main { serverMessage(client, 'Client "$clientName" not found.'); return; } - isServerPause = false; + videoTimer.isServerPause = false; clients.setLeader(clientName); broadcast({ type: SetLeader, @@ -966,9 +1014,9 @@ class Main { case PlayItem: if (!checkPermission(client, ChangeOrderPerm)) return; - resetSkipVote(); final pos = data.playItem.pos; if (!videoList.hasItem(pos)) return; + resetSkipVote(); if (videoTimer.getTime() > FLASHBACK_DIST) { saveFlashbackTime(videoList.currentItem); } @@ -991,6 +1039,17 @@ class Main { if (!checkPermission(client, ToggleItemTypePerm)) return; final pos = data.toggleItemType.pos; if (!videoList.hasItem(pos)) return; + if (data.toggleItemType.all == true) { + final targetIsTemp = !videoList.getItem(pos).isTemp; + videoList.setAllItemTypes(targetIsTemp); + broadcast({ + type: UpdatePlaylist, + updatePlaylist: { + videoList: videoList.getItems() + } + }); + return; + } videoList.toggleItemType(pos); broadcast(data); @@ -1001,8 +1060,8 @@ class Main { case ClearPlaylist: if (isPlaylistLockedFor(client)) return; - resetSkipVote(); if (!checkPermission(client, RemoveVideoPerm)) return; + resetSkipVote(); if (videoList.length != 0) { if (videoTimer.getTime() > FLASHBACK_DIST) { saveFlashbackTime(videoList.currentItem); @@ -1200,36 +1259,137 @@ class Main { } skipVotes[client.id] = true; + updateSkipVote(); + } + + function updateSkipVote():Void { + if (activeSkipVoteUrl == null) return; + if (videoList.length == 0 || videoList.currentItem.url != activeSkipVoteUrl) { + resetSkipVote(); + return; + } final votes = getSkipVoteCount(); final required = getSkipVoteRequiredCount(); + if (votes >= required) { + skipVideo({ + type: SkipVideo, + skipVideo: {url: activeSkipVoteUrl} + }); + return; + } + broadcastSkipVoteStats(); + } + + function broadcastSkipVoteStats():Void { + if (activeSkipVoteUrl == null) return; broadcast({ - type: ServerMessage, - serverMessage: { - textId: '${client.name} voted to skip (${votes}/${required}).' + type: SkipVideoStats, + skipVideoStats: { + voted: getSkipVoteCount(), + total: getSkipVoteEligibleClients().length, + required: getSkipVoteRequiredCount() } }); - - if (votes >= required) { - resetSkipVote(); - skipVideo(data); - } } function skipVideo(data:WsEvent):Void { if (videoList.length == 0) return; - resetSkipVote(); final item = videoList.currentItem; if (item.url != data.skipVideo.url) return; + resetSkipVote(); final dur = videoList.currentItem.duration; if (videoTimer.getTime() > FLASHBACK_DIST && videoTimer.getTime() < dur - FLASHBACK_DIST) { saveFlashbackTime(videoList.currentItem); } videoList.skipItem(); - if (videoList.length > 0) restartWaitTimer(); + if (videoList.length > 0) { + if (!videoList.currentItem.isPlayable()) pauseForIncompleteItem(); + else restartWaitTimer(); + } else { + videoTimer.stop(); + } broadcast(data); } + function pauseForIncompleteItem():Void { + videoTimer.stop(); + waitVideoStart?.stop(); + } + + final cacheCancelers:Map<String, () -> Void> = []; + + function cancelCache(url:String):Void { + final cancel = cacheCancelers[url] ?? return; + cacheCancelers.remove(url); + cancel(); + } + + function broadcastCacheProgress(url:String, ratio:Float):Void { + broadcast({ + type: Progress, + progress: { + type: Downloading, + ratio: ratio, + url: url + } + }); + } + + function updateIncompleteItemMeta(url:String, title:Null<String>, duration:Float):Void { + final item = videoList.find(item -> item.url == url) ?? return; + if (item.isIncomplete != true) return; + var changed = false; + if (item.duration <= 0 && duration > 0) { + item.duration = duration; + changed = true; + } + if (title != null && title.length > 0 && item.title.length == 0) { + item.title = title; + changed = true; + } + if (!changed) return; + broadcast({ + type: UpdatePlaylist, + updatePlaylist: { + videoList: videoList.getItems() + } + }); + } + + function completeIncompleteItem(url:String):Void { + cacheCancelers.remove(url); + final index = videoList.findIndex(i -> i.url == url); + if (index == -1) return; + final item = videoList.getItem(index); + if (item.isIncomplete != true) return; + item.isIncomplete = false; + broadcast({ + type: Progress, + progress: { + type: Completed, + ratio: 1, + url: url + } + }); + // uploaded videos don't need timer restart + if (videoList.pos == index && item.doCache) restartWaitTimer(); + } + + function removeIncompleteItem(url:String):Void { + cacheCancelers.remove(url); + final index = videoList.findIndex(i -> i.url == url); + if (index == -1) return; + if (videoList.getItem(index).isIncomplete != true) return; + videoList.removeItem(index); + broadcast({ + type: RemoveVideo, + removeVideo: { + url: url + } + }); + } + function checkPermission(client:Client, perm:Permission):Bool { if (client.isBanned) checkBan(client); final state = client.hasPermission(perm, config.permissions); @@ -1291,7 +1451,6 @@ class Main { waitVideoStart?.stop(); loadedClientsCount = 0; broadcast({type: VideoLoaded}); - isServerPause = false; videoTimer.start(); } diff --git a/src/server/Utils.hx b/src/server/Utils.hx index eb28115..fc71b0d 100644 --- a/src/server/Utils.hx +++ b/src/server/Utils.hx @@ -70,7 +70,7 @@ class Utils { r.setEncoding("utf8"); final data = new StringBuf(); r.on("data", chunk -> data.add(chunk)); - r.on("end", () -> callback(data.toString())); + r.on("end", () -> callback(data.toString().trim())); }).on("error", onError).on("timeout", onError); } diff --git a/src/server/VideoTimer.hx b/src/server/VideoTimer.hx index 311e7f4..ac6a655 100644 --- a/src/server/VideoTimer.hx +++ b/src/server/VideoTimer.hx @@ -4,6 +4,7 @@ import haxe.Timer.stamp; class VideoTimer { public var isStarted(default, null) = false; + public var isServerPause = false; var startTime = 0.0; var pauseStartTime = 0.0; @@ -14,6 +15,7 @@ class VideoTimer { public function start():Void { isStarted = true; + isServerPause = false; startTime = stamp(); pauseStartTime = 0; rateStartTime = stamp(); @@ -21,6 +23,7 @@ class VideoTimer { public function stop():Void { isStarted = false; + isServerPause = false; startTime = 0; pauseStartTime = 0; } @@ -38,6 +41,7 @@ class VideoTimer { public function play():Void { if (!isStarted) start(); + isServerPause = false; startTime += pauseTime(); pauseStartTime = 0; rateStartTime = stamp(); diff --git a/src/server/cache/Cache.hx b/src/server/cache/Cache.hx index cfe2f66..f0fe741 100644 --- a/src/server/cache/Cache.hx +++ b/src/server/cache/Cache.hx @@ -18,6 +18,7 @@ class Cache { public final freeSpaceBlock = 10 * 1024 * 1024; // 10MB final cachedFiles:Array<String> = []; + var ytdlpCache:YtdlpCache; var youtubeCache:YoutubeCache; var rawCache:RawCache; @@ -25,12 +26,13 @@ class Cache { this.main = main; this.cacheDir = cacheDir; Utils.ensureDir(cacheDir); - youtubeCache = new YoutubeCache(main, this); + ytdlpCache = new YtdlpCache(main, this); + youtubeCache = new YoutubeCache(main, this, ytdlpCache); rawCache = new RawCache(main, this); - isYtReady = youtubeCache.checkYtDeps(); + isYtReady = ytdlpCache.checkYtDeps(); if (isYtReady) { - youtubeCache.cleanYtInputFiles(); - youtubeCache.checkUpdate(); + ytdlpCache.cleanInputFiles(); + ytdlpCache.checkUpdate(); } } @@ -75,12 +77,31 @@ class Cache { } } - public function cacheYoutubeVideo(client:Client, url:String, callback:(name:String) -> Void) { - youtubeCache.cacheYoutubeVideo(client, url, callback); + public function cacheYoutubeVideo(client:Client, url:String, callbacks:CacheCallbacks) { + youtubeCache.cacheYoutubeVideo(client, url, callbacks); } - public function cacheRawVideo(client:Client, url:String, callback:(name:String) -> Void) { - rawCache.cacheRawVideo(client, url, callback); + /** Site-specific extractor download via yt-dlp, no fallback. **/ + public function cacheYtdlpVideo(client:Client, url:String, callbacks:CacheCallbacks) { + if (!isYtReady) { + rawCache.cacheRawVideo(client, url, callbacks); + return; + } + ytdlpCache.cacheVideo(client, url, callbacks); + } + + /** + use yt-dlp if a site-specific extractor supports the url, otherwise fallback to raw download. + **/ + public function cacheRawVideo(client:Client, url:String, callbacks:CacheCallbacks) { + if (!isYtReady) { + rawCache.cacheRawVideo(client, url, callbacks); + return; + } + ytdlpCache.checkUrlSupported(url, supported -> { + if (supported) ytdlpCache.cacheVideo(client, url, callbacks); + else rawCache.cacheRawVideo(client, url, callbacks); + }); } public function setStorageLimit(bytes:Int) { @@ -105,7 +126,7 @@ class Cache { callback(storageLimit); return; } - statfs("/", (err, stats) -> { + statfs(cacheDir, (err, stats) -> { if (err != null) { trace(err); callback(storageLimit); diff --git a/src/server/cache/CacheCallbacks.hx b/src/server/cache/CacheCallbacks.hx new file mode 100644 index 0000000..28ced3c --- /dev/null +++ b/src/server/cache/CacheCallbacks.hx @@ -0,0 +1,15 @@ +package server.cache; + +typedef CacheCallbacks = { + /** Called once the final cache url is known, before download starts. **/ + final onResolved:(url:String) -> Void; + final onProgress:(ratio:Float) -> Void; + final onComplete:() -> Void; + final onError:() -> Void; + final ?registerCancel:(cancel:() -> Void) -> Void; + /** + Called when the real title/duration become known (for items added + with a zero duration, e.g. when the client could not resolve them). + **/ + final ?onMetadata:(title:Null<String>, duration:Float) -> Void; +} diff --git a/src/server/cache/RawCache.hx b/src/server/cache/RawCache.hx index 1fb251d..3505119 100644 --- a/src/server/cache/RawCache.hx +++ b/src/server/cache/RawCache.hx @@ -27,7 +27,7 @@ class RawCache { this.cache = cache; } - public function cacheRawVideo(client:Client, url:String, callback:(name:String) -> Void) { + public function cacheRawVideo(client:Client, url:String, cb:CacheCallbacks) { final isM3U8 = url.contains(".m3u8"); final ext = isM3U8 ? "m3u8" : "mp4"; @@ -39,71 +39,72 @@ class RawCache { outName = cache.getFreeFileName(outName); if (cache.exists(outName)) { - callback(outName); + cb.onResolved(cache.getFileUrl(outName)); + cb.onComplete(); return; } trace('Caching $url to $outName...'); - main.send(client, { - type: Progress, - progress: { - type: Caching, - ratio: 0, - data: outName - } - }); + cb.onResolved(cache.getFileUrl(outName)); if (isM3U8) { - handleM3u8(client, url, outName, callback); + handleM3u8(client, url, outName, cb); } else { - handleMp4(client, url, outName, callback); + handleMp4(client, url, outName, cb); } } - function handleMp4(client:Client, url:String, outName:String, callback:(name:String) -> Void) { + function handleMp4(client:Client, url:String, outName:String, cb:CacheCallbacks) { final clientName = client.name; + var lastSentRatio = 0.0; + var canceled = false; downloadFile(client, url, outName, (downloaded, total) -> { - main.sendByName(clientName, { - type: Progress, - progress: { - type: Downloading, - ratio: (downloaded / total).clamp(0, 1).toFixed(4) - } - }); + if (canceled) return; + final ratio = (downloaded / total).clamp(0, 1); + if (ratio - lastSentRatio < 0.01 && ratio < 1) return; + lastSentRatio = ratio; + cb.onProgress(ratio.toFixed(4)); }, () -> { - cache.add(outName); - callback(outName); + if (canceled) return; + probeDuration(outName, duration -> { + if (canceled) return; + if (duration > 0 && cb.onMetadata != null) cb.onMetadata(null, duration); + cache.add(outName); + cb.onComplete(); + }); }, (err) -> { + if (canceled) return; log(clientName, 'Mp4 download failed: $err'); - cancelProgress(clientName); + cb.onError(); + }, (cancel) -> { + if (cb.registerCancel == null) return; + cb.registerCancel(() -> { + canceled = true; + cancel(); + }); }); } - function handleM3u8(client:Client, url:String, outName:String, callback:(name:String) -> Void):Void { + function handleM3u8(client:Client, url:String, outName:String, cb:CacheCallbacks):Void { final clientName = client.name; final useProxy = true; + var canceled = false; + if (cb.registerCancel != null) cb.registerCancel(() -> canceled = true); downloadM3u8Playlist(client, url, useProxy, (playlist, totalSize, segments) -> { + if (canceled) return; // only playlist file donwloaded if (useProxy) totalSize = playlist.length; if (!cache.removeOlderCache(totalSize + cache.freeSpaceBlock)) { log(clientName, cache.notEnoughSpaceErrorText); - cancelProgress(clientName); + cb.onError(); return; } if (useProxy) { - main.sendByName(clientName, { - type: Progress, - progress: { - type: Caching, - ratio: 1, - data: outName - } - }); File.saveContent('${cache.cacheDir}/$outName', playlist); cache.add(outName); - callback(outName); + cb.onComplete(); return; } @@ -128,25 +129,19 @@ class RawCache { downloaded++; final progress = downloaded / segments.length; - main.sendByName(clientName, { - type: Progress, - progress: { - type: Downloading, - ratio: progress.clamp(0, 1) - } - }); + cb.onProgress(progress.clamp(0, 1)); if (downloaded == segments.length) { trace('All ${downloaded}/${segments.length} segments downloaded'); File.saveContent('${cache.cacheDir}/$outName', playlist); cache.add(outName); - callback(outName); + cb.onComplete(); // buildTsFiles( // segments.map(item -> item.name), // outName, // client, - // callback + // cb // ); } else { downloadNextBatch(); @@ -157,7 +152,7 @@ class RawCache { activeDownloads--; downloaded++; log(clientName, 'TS segment ${segment.i} download failed: $err'); - cancelProgress(clientName); + cb.onError(); cleanupFiles(segments.map(item -> item.name)); } ); @@ -168,7 +163,7 @@ class RawCache { downloadNextBatch(); }, (err) -> { log(clientName, 'M3U8 processing failed: $err'); - cancelProgress(clientName); + cb.onError(); }); } @@ -247,6 +242,7 @@ class RawCache { // Head request can return full stream size, so lets do loose assumption final req = request(segments[0].url, {method: Get}, (res:IncomingMessage) -> { final contentLength = Std.parseInt(res.headers["content-length"]) ?? 0; + res.destroy(); final totalSize = contentLength * (segments.length + 1); if (totalSize == 0) { onError("Failed to get segment sizes: no content-length"); @@ -272,7 +268,8 @@ class RawCache { client:Client, url:String, fileName:String, onProgress:(downloaded:Int, total:Int) -> Void, onComplete:() -> Void, - onError:(err:String) -> Void + onError:(err:String) -> Void, + ?onCancelable:(cancel:() -> Void) -> Void ):Void { final outPath = '${cache.cacheDir}/$fileName'; final file = Fs.createWriteStream(outPath); @@ -325,16 +322,22 @@ class RawCache { onError('Request failed: $err'); }); + if (onCancelable != null) onCancelable(() -> { + req.destroy(); + file.destroy(); + if (FileSystem.exists(outPath)) FileSystem.deleteFile(outPath); + }); + req.end(); } - function buildTsFiles(tempFiles:Array<String>, outName:String, client:Client, callback:String->Void) { + function buildTsFiles(tempFiles:Array<String>, outName:String, client:Client, cb:CacheCallbacks) { final clientName = client.name; final missingFiles = tempFiles.filter(f -> !FileSystem.exists('${cache.cacheDir}/$f')); if (missingFiles.length > 0) { log(clientName, 'Concatenation failed: ${missingFiles.length} segments are missing'); - cancelProgress(clientName); + cb.onError(); cleanupFiles(tempFiles); return; } @@ -376,7 +379,7 @@ class RawCache { final timeoutId = js.Node.setTimeout(() -> { process.kill(); log(clientName, 'FFmpeg process timed out after ${timeout / 1000} seconds'); - cancelProgress(clientName); + cb.onError(); cleanupFiles(tempFiles.concat([concatFile])); }, timeout); @@ -393,28 +396,16 @@ class RawCache { final admins = main.clients.filter(client -> client.isAdmin); for (admin in admins) main.serverMessage(admin, ffmpegErr); - main.send(client, { - type: Progress, - progress: { - type: Canceled, - ratio: 1 - } - }); + cb.onError(); } else { // Verify the output file exists and has content if (FileSystem.exists('${cache.cacheDir}/$outName') && FileSystem.stat('${cache.cacheDir}/$outName').size > 0) { cache.add(outName); - callback(outName); + cb.onComplete(); } else { log(clientName, 'FFmpeg process completed but output file is missing or empty'); - main.send(client, { - type: Progress, - progress: { - type: Canceled, - ratio: 1 - } - }); + cb.onError(); } } @@ -426,34 +417,37 @@ class RawCache { process.on("error", (err) -> { js.Node.clearTimeout(timeoutId); log(clientName, 'Failed to start FFmpeg: $err'); - main.send(client, { - type: Progress, - progress: { - type: Canceled, - ratio: 1 - } - }); + cb.onError(); cleanupFiles(tempFiles.concat([concatFile])); }); } + function probeDuration(fileName:String, callback:(duration:Float) -> Void):Void { + final path = '${cache.cacheDir}/$fileName'; + final args = [ + "-v", "error", + "-show_entries", "format=duration", + "-of", "default=noprint_wrappers=1:nokey=1", + path + ]; + ChildProcess.execFile("ffprobe", args, (err, stdout, _) -> { + if (err != null) { + callback(0); + return; + } + final duration = Std.parseFloat(Std.string(stdout).trim()); + callback(Math.isNaN(duration) ? 0 : duration); + }); + } + function cleanupFiles(files:Array<String>):Void { for (file in files) { - if (FileSystem.exists(file)) FileSystem.deleteFile(file); + final path = '${cache.cacheDir}/$file'; + if (FileSystem.exists(path)) FileSystem.deleteFile(path); } } function log(clientName:String, msg:String):Void { cache.logByName(clientName, msg); } - - function cancelProgress(clientName:String):Void { - main.sendByName(clientName, { - type: Progress, - progress: { - type: Canceled, - ratio: 0 - } - }); - } } diff --git a/src/server/cache/YoutubeCache.hx b/src/server/cache/YoutubeCache.hx index 09ab67b..50bd37c 100644 --- a/src/server/cache/YoutubeCache.hx +++ b/src/server/cache/YoutubeCache.hx @@ -1,54 +1,24 @@ package server.cache; import haxe.io.Path; -import js.lib.Promise; -import js.node.ChildProcess; +import js.node.child_process.ChildProcess as ChildProcessObject; import sys.FileSystem; import utils.YoutubeUtils; import ytdlp_nodejs.VideoFormat; import ytdlp_nodejs.VideoInfo; -import ytdlp_nodejs.YtDlp; class YoutubeCache { final main:Main; final cache:Cache; - var ytDlp:Null<YtDlp>; + final ytdlp:YtdlpCache; - public function new(main:Main, cache:Cache):Void { + public function new(main:Main, cache:Cache, ytdlp:YtdlpCache):Void { this.main = main; this.cache = cache; + this.ytdlp = ytdlp; } - public function checkYtDeps():Bool { - try { - ChildProcess.execSync("ffmpeg -version", {stdio: "ignore", timeout: 5000}); - ytDlp = js.Syntax.code("new (require('ytdlp-nodejs')).YtDlp()"); - return true; - } catch (e) { - return false; - } - } - - public function checkUpdate():Void { - ytDlp.execAsync("", { - updateTo: main.config.ytDlp.channel, - onData: d -> { - trace(d); - } - }).catchError(e -> { - trace(e); - }); - } - - public function cleanYtInputFiles(prefix = "__tmp"):Void { - final names = FileSystem.readDirectory(cache.cacheDir); - for (name in names) { - if (!name.startsWith(prefix)) continue; - cache.remove(name); - } - } - - public function cacheYoutubeVideo(client:Client, url:String, callback:(name:String) -> Void) { + public function cacheYoutubeVideo(client:Client, url:String, cb:CacheCallbacks) { if (!cache.isYtReady) { trace("Do `npm i https://github.com/RblSb/ytdlp-nodejs` to use cache feature (you also need to install `ffmpeg` to build mp4 from downloaded audio/video tracks)."); return; @@ -59,20 +29,23 @@ class YoutubeCache { log(clientName, 'Error: youtube video id not found in url: $url'); return; } + // to prevent playlist in url handlings + url = 'https://youtu.be/$videoId'; final outName = videoId + ".mp4"; if (cache.exists(outName)) { - callback(outName); + cb.onResolved(cache.getFileUrl(outName)); + cb.onComplete(); return; } final inVideoName = '__tmp-video-$videoId'; inline function removeInputFiles():Void { - cleanYtInputFiles(inVideoName); + ytdlp.cleanInputFiles(inVideoName); } inline function checkEnoughSpace(contentLength:Int):Bool { final hasSpace = cache.removeOlderCache(contentLength + cache.freeSpaceBlock); if (!hasSpace) { removeInputFiles(); - cancelProgress(clientName); + cb.onError(); log(clientName, cache.notEnoughSpaceErrorText); } return hasSpace; @@ -83,18 +56,21 @@ class YoutubeCache { return; } trace('Caching $url to $outName...'); - main.sendByName(clientName, { - type: Progress, - progress: { - type: Caching, - ratio: 0, - data: outName - } - }); + cb.onResolved(cache.getFileUrl(outName)); var useCookies = false; + var lastSentRatio = 0.0; + var canceled = false; + var process:Null<ChildProcessObject> = null; + if (cb.registerCancel != null) cb.registerCancel(() -> { + canceled = true; + process?.kill(); + removeInputFiles(); + }); function onGetInfo(info:VideoInfo):Void { + if (canceled) return; + if (cb.onMetadata != null) cb.onMetadata(info.title, info.duration); trace('Get info with ${info.formats.length} formats'); var aformats = info.formats.filter(f -> f.acodec != "none" && f.vcodec == "none" && f.format_note?.contains("original")); @@ -154,16 +130,16 @@ class YoutubeCache { var videoRatioCache = 0.0; var audioRatioCache = 0.0; - final dlVideo:Promise<String> = ytDlp.downloadAsync(url, { + process = ytdlp.download(url, { format: formatIds, output: '${cache.cacheDir}/$inVideoName', remuxVideo: "mp4", - additionalOptions: ["--no-js-runtimes", "--js-runtimes", main.config.ytDlp.jsRuntime], - // verbose: true, - cookies: useCookies ? getCookiesPathOrNull() : null, + additionalOptions: ytdlp.ytExtraOptions(), + cookies: useCookies ? ytdlp.getCookiesPathOrNull() : null, forceIpv4: true, socketTimeout: 2, extractorRetries: 0, + }, { onProgress: p -> { final isFinished = p.status == "finished"; if (isFinished) { @@ -182,58 +158,48 @@ class YoutubeCache { ratio = videoRatioCache * videoSizeRatio + audioRatioCache * audioSizeRatio; - main.sendByName(clientName, { - type: Progress, - progress: { - type: Downloading, - ratio: ratio.toFixed(4) - } - }); - } - }).catchError(err -> { - final err = "Error during video download: " + err; - cache.logWithAdmins(client, err); - removeInputFiles(); - cancelProgress(clientName); - }); - - dlVideo.then((v) -> { - final name = cache.findFile(n -> n.startsWith(inVideoName)) ?? { - final err = 'Error: cannot find downloaded file with prefix $inVideoName'; + if (canceled) return; + if (ratio - lastSentRatio < 0.01 && ratio < 1) return; + lastSentRatio = ratio; + cb.onProgress(ratio.toFixed(4)); + }, + onComplete: () -> { + if (canceled) { + removeInputFiles(); + return; + } + final name = cache.findFile(n -> n.startsWith(inVideoName)) ?? { + final err = 'Error: cannot find downloaded file with prefix $inVideoName'; + cache.logWithAdmins(client, err); + cb.onError(); + return; + }; + FileSystem.rename('${cache.cacheDir}/$name', '${cache.cacheDir}/$outName'); + removeInputFiles(); + cache.add(outName); + cb.onComplete(); + }, + onError: err -> { + if (canceled) return; + final err = "Error during video download: " + err; cache.logWithAdmins(client, err); - cancelProgress(clientName); - return; - }; - FileSystem.rename('${cache.cacheDir}/$name', '${cache.cacheDir}/$outName'); - removeInputFiles(); - cache.add(outName); - callback(outName); + removeInputFiles(); + cb.onError(); + } }); } - getInfoAsync(url, useCookies).then(onGetInfo).catchError(err -> { + ytdlp.getInfoAsync(url, useCookies).then(onGetInfo).catchError(err -> { trace(err); useCookies = true; - getInfoAsync(url, useCookies).then(onGetInfo).catchError(err -> { + ytdlp.getInfoAsync(url, useCookies).then(onGetInfo).catchError(err -> { removeInputFiles(); - cancelProgress(clientName); + cb.onError(); log(clientName, "" + err); }); }); } - function getInfoAsync(url:String, useCookies = false):Promise<VideoInfo> { - return cast ytDlp.getInfoAsync(url, cast { - cookies: useCookies ? getCookiesPathOrNull() : null, - additionalOptions: ["--no-js-runtimes", "--js-runtimes", main.config.ytDlp.jsRuntime], - }); - } - - function getCookiesPathOrNull():Null<String> { - final cookiesPath = '${main.userDir}/cookies.txt'; - return FileSystem.exists(cookiesPath) ? cookiesPath : null; - } - function getBestYoutubeVideoFormat(formats:Array<VideoFormat>, ?ignoreQualities:Array<Int>):Null<VideoFormat> { final qPriority = [1080, 720, 480, 360, 240, 144]; if (ignoreQualities != null) { @@ -274,14 +240,4 @@ class YoutubeCache { function log(clientName:String, msg:String):Void { cache.logByName(clientName, msg); } - - function cancelProgress(clientName:String):Void { - main.sendByName(clientName, { - type: Progress, - progress: { - type: Canceled, - ratio: 0 - } - }); - } } diff --git a/src/server/cache/YtdlpCache.hx b/src/server/cache/YtdlpCache.hx new file mode 100644 index 0000000..31df7b2 --- /dev/null +++ b/src/server/cache/YtdlpCache.hx @@ -0,0 +1,227 @@ +package server.cache; + +import haxe.io.Path; +import js.lib.Promise; +import js.node.ChildProcess; +import js.node.child_process.ChildProcess as ChildProcessObject; +import sys.FileSystem; +import ytdlp_nodejs.ArgsOptions; +import ytdlp_nodejs.VideoInfo; +import ytdlp_nodejs.VideoProgress; +import ytdlp_nodejs.YtDlp; + +typedef YtdlpHandlers = { + final onProgress:(p:VideoProgress) -> Void; + final onComplete:() -> Void; + final onError:(err:String) -> Void; +} + +class YtdlpCache { + final main:Main; + final cache:Cache; + var ytDlp:Null<YtDlp>; + + public function new(main:Main, cache:Cache):Void { + this.main = main; + this.cache = cache; + } + + public function checkYtDeps():Bool { + try { + ChildProcess.execSync("ffmpeg -version", {stdio: "ignore", timeout: 5000}); + ytDlp = js.Syntax.code("new (require('ytdlp-nodejs')).YtDlp()"); + return true; + } catch (e) { + return false; + } + } + + public function checkUpdate():Void { + ytDlp.execAsync("", { + updateTo: main.config.ytDlp.channel, + onData: d -> { + trace(d); + } + }).catchError(e -> { + trace(e); + }); + } + + public function cleanInputFiles(prefix = "__tmp"):Void { + final names = FileSystem.readDirectory(cache.cacheDir); + for (name in names) { + if (!name.startsWith(prefix)) continue; + cache.remove(name); + } + } + + public function getInfoAsync(url:String, useCookies = false):Promise<VideoInfo> { + return cast ytDlp.getInfoAsync(url, cast { + cookies: useCookies ? getCookiesPathOrNull() : null, + additionalOptions: ytExtraOptions(), + }); + } + + public function getCookiesPathOrNull():Null<String> { + final cookiesPath = '${main.userDir}/cookies.txt'; + return FileSystem.exists(cookiesPath) ? cookiesPath : null; + } + + public function ytExtraOptions():Array<String> { + return ["--no-js-runtimes", "--js-runtimes", main.config.ytDlp.jsRuntime]; + } + + /** + Runs `yt-dlp <url> --ies default,-generic` to check if a site-specific + extractor handles the url. Returns the process so it can be killed on cancel. + **/ + public function checkUrlSupported(url:String, callback:(supported:Bool) -> Void):ChildProcessObject { + final process:ChildProcessObject = ytDlp.exec(url, { + useExtractors: ["default", "-generic"], + simulate: true, + quiet: true, + socketTimeout: 5, + extractorRetries: 0, + }); + var done = false; + inline function finish(supported:Bool):Void { + if (done) return; + done = true; + callback(supported); + } + process.on("close", (code:Int, _signal:String) -> finish(code == 0)); + process.on("error", (_err:Dynamic) -> finish(false)); + return process; + } + + /** + Spawns yt-dlp as a manual process and wires progress/completion to `handlers`. + Returns the process so the caller can kill it on cancel. + **/ + public function download(url:String, options:ArgsOptions, handlers:YtdlpHandlers):ChildProcessObject { + final process:ChildProcessObject = ytDlp.exec(url, options); + var stderr = ""; + var done = false; + process.stderr.on("data", (d:Dynamic) -> stderr += d); + process.on("progress", (p:VideoProgress) -> handlers.onProgress(p)); + process.on("close", (code:Int, _signal:String) -> { + if (done) return; + done = true; + if (code == 0) handlers.onComplete(); + else handlers.onError('yt-dlp exited with code $code: $stderr'); + }); + process.on("error", (err:Dynamic) -> { + if (done) return; + done = true; + handlers.onError('Failed to start yt-dlp process: $err'); + }); + return process; + } + + /** + Generic best-quality download to mp4 for vk and other site-specific + extractors. Used after `checkUrlSupported` succeeds (or directly for vk). + **/ + public function cacheVideo(client:Client, url:String, cb:CacheCallbacks):Void { + final clientName = client.name; + final outName = deriveOutName(url); + trace('Caching $url to $outName...'); + cb.onResolved(cache.getFileUrl(outName)); + + final inName = '__tmp-${Std.random(0xFFFFFF)}-${Path.withoutExtension(outName)}'; + var canceled = false; + var process:Null<ChildProcessObject> = null; + inline function cleanup():Void { + cleanInputFiles(inName); + } + if (cb.registerCancel != null) cb.registerCancel(() -> { + canceled = true; + process?.kill(); + cleanup(); + }); + + var useCookies = false; + var lastSentRatio = 0.0; + + function onGetInfo(info:VideoInfo):Void { + if (canceled) return; + if (cb.onMetadata != null) cb.onMetadata(info.title, info.duration); + if (!cache.removeOlderCache(estimateSize(info) * 2 + cache.freeSpaceBlock)) { + cleanup(); + cb.onError(); + log(clientName, cache.notEnoughSpaceErrorText); + return; + } + process = download(url, { + output: '${cache.cacheDir}/$inName', + remuxVideo: "mp4", + additionalOptions: ytExtraOptions(), + cookies: useCookies ? getCookiesPathOrNull() : null, + forceIpv4: true, + socketTimeout: 5, + extractorRetries: 0, + }, { + onProgress: p -> { + if (canceled) return; + final ratio = p.status == "finished" ? 1.0 : (p.downloaded / p.total) + .clamp(0, 1); + if (ratio - lastSentRatio < 0.01 && ratio < 1) return; + lastSentRatio = ratio; + cb.onProgress(ratio.toFixed(4)); + }, + onComplete: () -> { + if (canceled) { + cleanup(); + return; + } + final name = cache.findFile(n -> n.startsWith(inName)) ?? { + final err = 'Error: cannot find downloaded file with prefix $inName'; + cache.logWithAdmins(client, err); + cb.onError(); + return; + }; + FileSystem.rename('${cache.cacheDir}/$name', '${cache.cacheDir}/$outName'); + cleanup(); + cache.add(outName); + cb.onComplete(); + }, + onError: err -> { + if (canceled) return; + cache.logWithAdmins(client, "Error during video download: " + err); + cleanup(); + cb.onError(); + } + }); + } + + getInfoAsync(url, useCookies).then(onGetInfo).catchError(err -> { + if (canceled) return; + useCookies = true; + getInfoAsync(url, useCookies).then(onGetInfo).catchError(err -> { + cleanup(); + cb.onError(); + log(clientName, "" + err); + }); + }); + } + + function deriveOutName(url:String):String { + final decoded = try url.urlDecode() catch (e) url; + var last = decoded.substr(decoded.lastIndexOf("/") + 1); + final q = last.indexOf("?"); + if (q != -1) last = last.substr(0, q); + last = Path.withoutExtension(last); + if (last == "") last = "video"; + return cache.getFreeFileName('$last.mp4'); + } + + function estimateSize(info:VideoInfo):Int { + final d:Dynamic = info; + final size:Null<Float> = d.filesize_approx ?? d.filesize; + return Std.int(size ?? 0); + } + + function log(clientName:String, msg:String):Void { + cache.logByName(clientName, msg); + } +} |
