diff options
| author | RblSb <msrblsb@gmail.com> | 2025-01-16 03:07:31 +0300 |
|---|---|---|
| committer | RblSb <msrblsb@gmail.com> | 2025-01-17 01:00:09 +0300 |
| commit | d9ca7beaa9494cf34590853901cf8be44e243775 (patch) | |
| tree | f09ce979460bdf28363a922298283dfee0c506fb /src | |
| parent | f84fdc40ba817b6a2d907484b1e1500197ceeafe (diff) | |
Cache on server feature
Server will download video from supported players and add as raw video to playlist (only youtube is supported for now).
Cache for YT player is available after installing optional dependencies, see readme. For cache size see `cacheStorageLimitGiB ` in config.
There is also minor ux improvement, latest checkbox states will be keeped in local storage now.
Diffstat (limited to 'src')
| -rw-r--r-- | src/Types.hx | 24 | ||||
| -rw-r--r-- | src/client/Buttons.hx | 34 | ||||
| -rw-r--r-- | src/client/ClientSettings.hx | 3 | ||||
| -rw-r--r-- | src/client/IPlayer.hx | 2 | ||||
| -rw-r--r-- | src/client/JsApi.hx | 5 | ||||
| -rw-r--r-- | src/client/Main.hx | 74 | ||||
| -rw-r--r-- | src/client/Player.hx | 43 | ||||
| -rw-r--r-- | src/client/players/Iframe.hx | 7 | ||||
| -rw-r--r-- | src/client/players/Raw.hx | 5 | ||||
| -rw-r--r-- | src/client/players/Youtube.hx | 55 | ||||
| -rw-r--r-- | src/server/Cache.hx | 165 | ||||
| -rw-r--r-- | src/server/Main.hx | 55 | ||||
| -rw-r--r-- | src/server/ServerState.hx | 3 | ||||
| -rw-r--r-- | src/server/YoutubeFallback.hx | 142 | ||||
| -rw-r--r-- | src/utils/YoutubeUtils.hx | 6 |
15 files changed, 352 insertions, 271 deletions
diff --git a/src/Types.hx b/src/Types.hx index 8567f7b..0f56392 100644 --- a/src/Types.hx +++ b/src/Types.hx @@ -1,7 +1,12 @@ package; import Client.ClientData; -import utils.YoutubeUtils.YouTubeVideoInfo; + +enum abstract PlayerType(String) { + var RawType; + var YoutubeType; + var IframeType; +} typedef VideoDataRequest = { final url:String; @@ -14,7 +19,7 @@ typedef VideoData = { var ?url:String; var ?subs:String; var ?voiceOverTrack:String; - var ?isIframe:Bool; + var ?playerType:PlayerType; } typedef Config = { @@ -32,6 +37,7 @@ typedef Config = { templateUrl:String, youtubeApiKey:String, youtubePlaylistLimit:Int, + cacheStorageLimitGiB:Float, permissions:Permissions, emotes:Array<Emote>, filters:Array<Filter>, @@ -109,7 +115,8 @@ typedef VideoItem = { var ?subs:String; var ?voiceOverTrack:String; var isTemp:Bool; - var isIframe:Bool; + var doCache:Bool; + var playerType:PlayerType; } private class VideoItemTools { @@ -122,7 +129,8 @@ private class VideoItemTools { subs: item.subs, voiceOverTrack: item.voiceOverTrack, isTemp: item.isTemp, - isIframe: item.isIframe + doCache: item.doCache, + playerType: item.playerType }; } } @@ -145,7 +153,8 @@ typedef WsEvent = { videoList:Array<VideoItem>, isPlaylistOpen:Bool, itemPos:Int, - globalIp:String + globalIp:String, + playersCacheSupport:Array<PlayerType>, }, ?login:{ clientName:String, @@ -226,10 +235,6 @@ typedef WsEvent = { ?dump:{ data:String }, - ?getYoutubeVideoInfo:{ - url:String, - ?response:YouTubeVideoInfo - } } enum abstract WsEventType(String) { @@ -267,5 +272,4 @@ enum abstract WsEventType(String) { var UpdatePlaylist; var TogglePlaylistLock; var Dump; - var GetYoutubeVideoInfo; } diff --git a/src/client/Buttons.hx b/src/client/Buttons.hx index c3581e2..f5f26e8 100644 --- a/src/client/Buttons.hx +++ b/src/client/Buttons.hx @@ -22,6 +22,12 @@ class Buttons { setSplitSize(settings.chatSize); initChatInput(main); + for (item in settings.checkboxes) { + if (item.checked == null) continue; + final checkbox:InputElement = ge('#${item.id}') ?? continue; + checkbox.checked = item.checked; + } + final passIcon = ge("#guestpass_icon"); passIcon.onclick = e -> { final icon = passIcon.firstElementChild; @@ -213,12 +219,15 @@ class Buttons { final mediaUrl:InputElement = cast ge("#mediaurl"); mediaUrl.oninput = () -> { - final value = mediaUrl.value; - final isRawSingleVideo = value != "" && main.isRawPlayerLink(value) - && main.isSingleVideoLink(value); - ge("#mediatitleblock").style.display = isRawSingleVideo ? "" : "none"; - ge("#subsurlblock").style.display = isRawSingleVideo ? "" : "none"; - ge("#voiceoverblock").style.display = value.length > 0 ? "" : "none"; + final url = mediaUrl.value; + final playerType = main.getLinkPlayerType(url); + final isSingle = main.isSingleVideoUrl(url); + final isSingleRawVideo = url != "" && playerType == RawType && isSingle; + ge("#mediatitleblock").style.display = isSingleRawVideo ? "" : "none"; + ge("#subsurlblock").style.display = isSingleRawVideo ? "" : "none"; + ge("#voiceoverblock").style.display = (url.length > 0 && isSingle) ? "" : "none"; + final showCache = isSingle && main.playersCacheSupport.contains(playerType); + ge("#cache-on-server").parentElement.style.display = showCache ? "" : "none"; final panel = ge("#addfromurl"); final oldH = panel.style.height; // save for animation panel.style.height = ""; // to calculate height from content @@ -482,6 +491,19 @@ class Buttons { if (Utils.isTouch()) chatline.blur(); return true; }); + final checkboxes:Array<InputElement> = [ + ge("#add-temp"), + ge("#cache-on-server"), + ]; + for (checkbox in checkboxes) { + checkbox.addEventListener("change", () -> { + final checked = checkbox.checked; + final item = settings.checkboxes.find(item -> item.id == checkbox.id); + settings.checkboxes.remove(item); + settings.checkboxes.push({id: checkbox.id, checked: checked}); + Settings.write(settings); + }); + } } static inline function getVisualViewport():Null<VisualViewport> { diff --git a/src/client/ClientSettings.hx b/src/client/ClientSettings.hx index a004213..cb5f99f 100644 --- a/src/client/ClientSettings.hx +++ b/src/client/ClientSettings.hx @@ -14,5 +14,6 @@ typedef ClientSettings = { latestLinks:Array<String>, latestSubs:Array<String>, hotkeysEnabled:Bool, - showHintList:Bool + showHintList:Bool, + checkboxes:Array<{id:String, checked:Null<Bool>}>, } diff --git a/src/client/IPlayer.hx b/src/client/IPlayer.hx index 032ac97..e8835d1 100644 --- a/src/client/IPlayer.hx +++ b/src/client/IPlayer.hx @@ -1,10 +1,12 @@ package client; +import Types.PlayerType; import Types.VideoData; import Types.VideoDataRequest; import Types.VideoItem; interface IPlayer { + function getPlayerType():PlayerType; function isSupportedLink(url:String):Bool; function getVideoData(data:VideoDataRequest, callback:(data:VideoData) -> Void):Void; function loadVideo(item:VideoItem):Void; diff --git a/src/client/JsApi.hx b/src/client/JsApi.hx index 1532231..abf9a6a 100644 --- a/src/client/JsApi.hx +++ b/src/client/JsApi.hx @@ -72,8 +72,9 @@ class JsApi { } @:expose - static function addVideoItem(url:String, atEnd:Bool, isTemp:Bool, ?callback:() -> Void):Void { - main.addVideo(url, atEnd, isTemp, callback); + static function addVideoItem(url:String, atEnd:Bool, isTemp:Bool, ?callback:() -> + Void, doCache = false):Void { + main.addVideo(url, atEnd, isTemp, doCache, callback); } @:expose diff --git a/src/client/Main.hx b/src/client/Main.hx index 248e205..72dbf8e 100644 --- a/src/client/Main.hx +++ b/src/client/Main.hx @@ -3,6 +3,7 @@ package client; import Client.ClientData; import Types.Config; import Types.Permission; +import Types.PlayerType; import Types.VideoData; import Types.VideoDataRequest; import Types.WsEvent; @@ -25,15 +26,16 @@ import js.html.WebSocket; using ClientTools; class Main { - static inline var SETTINGS_VERSION = 4; + static inline var SETTINGS_VERSION = 5; public final settings:ClientSettings; public var isSyncActive = true; public var forceSyncNextTick = false; - public var isVideoEnabled = true; + public var isVideoEnabled(default, null) = true; public final host:String; public var globalIp(default, null) = ""; - public var isPlaylistOpen = true; + public var isPlaylistOpen(default, null) = true; + public var playersCacheSupport(default, null):Array<PlayerType> = []; final clients:Array<Client> = []; var pageTitle = document.title; @@ -74,7 +76,8 @@ class Main { latestLinks: [], latestSubs: [], hotkeysEnabled: true, - showHintList: true + showHintList: true, + checkboxes: [], } Settings.init(defaults, settingsPatcher); settings = Settings.read(); @@ -104,7 +107,7 @@ class Main { if (!player.isVideoLoaded()) return; gotFirstPageInteraction = true; player.unmute(); - player.play(); + if (!hasLeader()) player.play(); document.removeEventListener("click", onFirstInteraction); } @@ -119,6 +122,9 @@ class Main { case 3: final data:ClientSettings = data; data.showHintList = true; + case 4: + final data:ClientSettings = data; + data.checkboxes = []; case SETTINGS_VERSION, _: throw 'skipped version $version'; } @@ -236,19 +242,19 @@ class Main { return personal.hasPermission(permission, config.permissions); } - final mask = ~/\${([0-9]+)-([0-9]+)}/g; + public final urlMask = ~/\${([0-9]+)-([0-9]+)}/g; function handleUrlMasks(links:Array<String>):Void { for (link in links) { - if (!mask.match(link)) continue; - final start = Std.parseInt(mask.matched(1)); - var end = Std.parseInt(mask.matched(2)); + if (!urlMask.match(link)) continue; + final start = Std.parseInt(urlMask.matched(1)); + var end = Std.parseInt(urlMask.matched(2)); if (Math.abs(start - end) > 100) continue; final step = end > start ? -1 : 1; final i = links.indexOf(link); links.remove(link); while (end != start + step) { - links.insert(i, mask.replace(link, '$end')); + links.insert(i, urlMask.replace(link, '$end')); end += step; } } @@ -257,8 +263,11 @@ class Main { function addVideoUrl(atEnd:Bool):Void { final mediaUrl:InputElement = cast ge("#mediaurl"); final subsUrl:InputElement = cast ge("#subsurl"); - final checkbox:InputElement = cast ge("#addfromurl").querySelector(".add-temp"); - final isTemp = checkbox.checked; + final checkboxTemp:InputElement = cast ge("#addfromurl .add-temp"); + final isTemp = checkboxTemp.checked; + final checkboxCache:InputElement = cast ge("#cache-on-server"); + final doCache = checkboxCache.checked + && checkboxCache.parentElement.style.display != "none"; final url = mediaUrl.value; final subs = subsUrl.value; if (url.length == 0) return; @@ -273,17 +282,15 @@ class Main { handleUrlMasks(links); // if videos added as next, we need to load them in reverse order if (!atEnd) sortItemsForQueueNext(links); - addVideoArray(links, atEnd, isTemp); + addVideoArray(links, atEnd, isTemp, doCache); } - public function isRawPlayerLink(url:String):Bool { - return player.isRawPlayerLink(url); + public function getLinkPlayerType(url:String):PlayerType { + return player.getLinkPlayerType(url); } - public function isSingleVideoLink(url:String):Bool { - if (~/, ?(https?)/g.match(url)) return false; - if (mask.match(url)) return false; - return true; + public function isSingleVideoUrl(url:String):Bool { + return player.isSingleVideoUrl(url); } public function sortItemsForQueueNext<T>(items:Array<T>):Void { @@ -295,13 +302,14 @@ class Main { if (first != null) items.unshift(first); } - function addVideoArray(links:Array<String>, atEnd:Bool, isTemp:Bool):Void { + function addVideoArray(links:Array<String>, atEnd:Bool, isTemp:Bool, doCache:Bool):Void { if (links.length == 0) return; final link = links.shift(); - addVideo(link, atEnd, isTemp, () -> addVideoArray(links, atEnd, isTemp)); + addVideo(link, atEnd, isTemp, doCache, () -> + addVideoArray(links, atEnd, isTemp, doCache)); } - public function addVideo(url:String, atEnd:Bool, isTemp:Bool, ?callback:() -> Void):Void { + public function addVideo(url:String, atEnd:Bool, isTemp:Bool, doCache:Bool, ?callback:() -> Void):Void { final protocol = Browser.location.protocol; if (url.startsWith("/")) { final host = Browser.location.hostname; @@ -330,9 +338,10 @@ class Main { author: personal.name, duration: data.duration, isTemp: isTemp, + doCache: doCache, subs: data.subs, voiceOverTrack: data.voiceOverTrack, - isIframe: data.isIframe == true + playerType: data.playerType }, atEnd: atEnd } @@ -349,7 +358,7 @@ class Main { final mediaTitle:InputElement = cast ge("#customembed-title"); final title = mediaTitle.value; mediaTitle.value = ""; - final checkbox:InputElement = cast ge("#customembed").querySelector(".add-temp"); + final checkbox:InputElement = cast ge("#customembed .add-temp"); final isTemp = checkbox.checked; final obj:VideoDataRequest = { url: iframe, @@ -372,7 +381,8 @@ class Main { author: personal.name, duration: data.duration, isTemp: isTemp, - isIframe: true + doCache: false, + playerType: IframeType }, atEnd: atEnd } @@ -504,12 +514,14 @@ class Main { case Pause: player.setPauseIndicator(false); + updateUserList(); if (isLeader()) return; player.pause(); player.setTime(data.pause.time); case Play: player.setPauseIndicator(true); + updateUserList(); if (isLeader()) return; final synchThreshold = settings.synchThreshold; final newTime = data.play.time; @@ -596,9 +608,6 @@ class Main { case Dump: Utils.saveFile("dump.json", ApplicationJson, data.dump.data); - - case GetYoutubeVideoInfo: - // handled by event listeners like `JsApi.once` } } @@ -609,6 +618,7 @@ class Main { Settings.write(settings); globalIp = connected.globalIp; + playersCacheSupport = connected.playersCacheSupport; setConfig(connected.config); if (connected.isUnknownClient) { updateClients(connected.clients); @@ -906,7 +916,8 @@ class Main { final list = new StringBuf(); for (client in clients) { list.add('<div class="userlist_item">'); - if (client.isLeader) list.add('<ion-icon name="play"></ion-icon>'); + final iconName = player.isPaused() ? "pause" : "play"; + if (client.isLeader) list.add('<ion-icon name="$iconName"></ion-icon>'); var klass = client.isBanned ? "userlist_banned" : ""; if (client.isAdmin) klass += " userlist_owner"; list.add('<span class="$klass">${client.name}</span></div>'); @@ -1232,7 +1243,8 @@ class Main { return ~/([.*+?^${}()|[\]\\])/g.replace(regex, "\\$1"); } - public static inline function ge(id:String):Element { - return document.querySelector(id); + @:generic + public static inline function ge<T:Element>(id:String):T { + return cast document.querySelector(id); } } diff --git a/src/client/Player.hx b/src/client/Player.hx index f40c34c..70efcf9 100644 --- a/src/client/Player.hx +++ b/src/client/Player.hx @@ -1,5 +1,6 @@ package client; +import Types.PlayerType; import Types.VideoData; import Types.VideoDataRequest; import Types.VideoItem; @@ -41,7 +42,7 @@ class Player { streamable = new Streamable(main, this); players = [ youtube, - streamable + streamable, ]; iframePlayer = new Iframe(main, this); rawPlayer = new Raw(main, this); @@ -118,26 +119,40 @@ class Player { var player = players.find(player -> player.isSupportedLink(req.url)); player ??= rawPlayer; player.getVideoData(req, data -> { + data.playerType ??= player.getPlayerType(); final voiceOverTrack = voiceOverInput.value.trim(); - data.voiceOverTrack = voiceOverTrack; voiceOverInput.value = ""; + data.voiceOverTrack ??= voiceOverTrack; callback(data); }); } - public function isRawPlayerLink(url:String):Bool { - if (streamable.isSupportedLink(url)) return true; - return !players.exists(player -> player.isSupportedLink(url)); + public function getLinkPlayerType(url:String):PlayerType { + final player = players.find(player -> player.isSupportedLink(url)); + if (player == null) return rawPlayer.getPlayerType(); + return player.getPlayerType(); + } + + public function isSingleVideoUrl(url:String):Bool { + if (youtube.isSupportedLink(url)) { + if (youtube.isPlaylistUrl(url)) return false; + } + if (~/, ?(https?)/g.match(url)) return false; + if (main.urlMask.match(url)) return false; + return true; } public function getIframeData(data:VideoDataRequest, callback:(data:VideoData) -> Void):Void { - iframePlayer.getVideoData(data, callback); + iframePlayer.getVideoData(data, data -> { + data.playerType = IframeType; + callback(data); + }); } public function setVideo(i:Int):Void { if (!main.isSyncActive) return; final item = videoList.getItem(i); - setSupportedPlayer(item.url, item.isIframe); + setSupportedPlayer(item.url, item.playerType); removeActiveLabel(videoList.pos); videoList.setPos(i); @@ -190,17 +205,17 @@ class Player { needsVolumeReset = true; } - function setSupportedPlayer(url:String, isIframe:Bool):Void { + function setSupportedPlayer(url:String, playerType:PlayerType):Void { final currentPlayer = players.find(p -> p.isSupportedLink(url)); if (currentPlayer != null) setPlayer(currentPlayer); - else if (isIframe) setPlayer(iframePlayer); + else if (playerType == IframeType) setPlayer(iframePlayer); else setPlayer(rawPlayer); } public function changeVideoSrc(url:String):Void { if (!main.isVideoEnabled) return; final item:VideoItem = videoList.currentItem ?? return; - setSupportedPlayer(url, item.isIframe); + setSupportedPlayer(url, item.playerType); player.loadVideo(item.withUrl(url)); } @@ -247,10 +262,6 @@ class Player { final item = videoList.currentItem ?? return; // do not send pause if video is ended if (getTime() >= item.duration - 0.01) return; - // youtube raw fallback has around one second difference from rounded youtube duration - if (player == rawPlayer && youtube.isSupportedLink(item.url)) { - if (getTime() >= item.duration - 1) return; - } final hasAutoPause = main.hasLeaderOnPauseRequest() && videoList.length > 0 && getTime() > 1 @@ -316,7 +327,7 @@ class Player { public function addVideoItem(item:VideoItem, atEnd:Bool):Void { final url = item.url.htmlEscape(true); - final duration = item.isIframe ? "" : duration(item.duration); + final duration = item.playerType == IframeType ? "" : duration(item.duration); final itemEl = Utils.nodeFromString( '<li class="queue_entry info" title="${Lang.get("addedBy")}: ${item.author}"> <header> @@ -449,7 +460,7 @@ class Player { function totalDuration():String { var time = 0.0; for (item in videoList.getItems()) { - if (item.isIframe) continue; + if (item.playerType == IframeType) continue; time += item.duration; } return duration(time); diff --git a/src/client/players/Iframe.hx b/src/client/players/Iframe.hx index 56cf319..8a2e06e 100644 --- a/src/client/players/Iframe.hx +++ b/src/client/players/Iframe.hx @@ -1,5 +1,6 @@ package client.players; +import Types.PlayerType; import Types.VideoData; import Types.VideoDataRequest; import Types.VideoItem; @@ -18,13 +19,17 @@ class Iframe implements IPlayer { this.player = player; } + public function getPlayerType():PlayerType { + return IframeType; + } + public function isSupportedLink(url:String):Bool { return true; } public function getVideoData(data:VideoDataRequest, callback:(data:VideoData) -> Void):Void { final iframe = document.createDivElement(); - iframe.innerHTML = data.url; + iframe.innerHTML = data.url.trim(); if (isValidIframe(iframe)) { callback({duration: 99 * 60 * 60}); } else { diff --git a/src/client/players/Raw.hx b/src/client/players/Raw.hx index 5c5b7a4..7e5e50c 100644 --- a/src/client/players/Raw.hx +++ b/src/client/players/Raw.hx @@ -1,5 +1,6 @@ package client.players; +import Types.PlayerType; import Types.VideoData; import Types.VideoDataRequest; import Types.VideoItem; @@ -30,6 +31,10 @@ class Raw implements IPlayer { this.player = player; } + public function getPlayerType():PlayerType { + return RawType; + } + public function isSupportedLink(url:String):Bool { return true; } diff --git a/src/client/players/Youtube.hx b/src/client/players/Youtube.hx index 3ad9030..5a8921a 100644 --- a/src/client/players/Youtube.hx +++ b/src/client/players/Youtube.hx @@ -1,5 +1,6 @@ package client.players; +import Types.PlayerType; import Types.VideoData; import Types.VideoDataRequest; import Types.VideoItem; @@ -23,7 +24,6 @@ class Youtube implements IPlayer { var apiKey:String; var video:Element; var youtube:YoutubePlayer; - var tempYoutube:YoutubePlayer; var isLoaded = false; public function new(main:Main, player:Player) { @@ -31,6 +31,10 @@ class Youtube implements IPlayer { this.player = player; } + public function getPlayerType():PlayerType { + return YoutubeType; + } + public function isSupportedLink(url:String):Bool { return extractVideoId(url) != "" || extractPlaylistId(url) != ""; } @@ -43,6 +47,10 @@ class Youtube implements IPlayer { return YoutubeUtils.extractPlaylistId(url); } + public function isPlaylistUrl(url:String):Bool { + return extractVideoId(url) == "" && extractPlaylistId(url) != ""; + } + final matchHours = ~/([0-9]+)H/; final matchMinutes = ~/([0-9]+)M/; final matchSeconds = ~/([0-9]+)S/; @@ -93,7 +101,7 @@ class Youtube implements IPlayer { url: '<iframe src="https://www.youtube.com/embed/$id?autoplay=1$mute" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>', - isIframe: true + playerType: IframeType }); continue; } @@ -168,6 +176,7 @@ class Youtube implements IPlayer { final video = document.createDivElement(); video.id = "temp-videoplayer"; Utils.prepend(playerEl, video); + var tempYoutube:YoutubePlayer = null; tempYoutube = new YoutubePlayer(video.id, { videoId: extractVideoId(url), playerVars: { @@ -181,12 +190,14 @@ class Youtube implements IPlayer { callback({ duration: tempYoutube.getDuration() }); + tempYoutube.destroy(); }, onError: e -> { // TODO message error codes trace('Error ${e.data}'); if (playerEl.contains(video)) playerEl.removeChild(video); callback({duration: 0}); + tempYoutube.destroy(); } } }); @@ -243,49 +254,13 @@ class Youtube implements IPlayer { onError: e -> { // TODO message error codes trace('Error ${e.data}'); - final item = player.getCurrentItem() ?? return; - rawSourceFallback(item.url); + // final item = player.getCurrentItem() ?? return; + // rawSourceFallback(item.url); } } }); } - function rawSourceFallback(url:String):Void { - JsApi.once(GetYoutubeVideoInfo, event -> { - final data = event.getYoutubeVideoInfo; - final info = data.response; - final format = getBestStreamFormat(info) ?? { - trace("format not found in response info:"); - trace(info); - return; - }; - player.changeVideoSrc(format.url); - }); - main.send({ - type: GetYoutubeVideoInfo, - getYoutubeVideoInfo: { - url: url - } - }); - } - - function getBestStreamFormat(info:YouTubeVideoInfo):Null<YoutubeVideoFormat> { - info.formats ??= []; - info.adaptiveFormats ??= []; - final formats = info.adaptiveFormats.concat(info.formats); - trace(formats); - final qPriority = [1080, 720, 480, 360, 240]; - for (q in qPriority) { - final quality = '${q}p'; - for (format in formats) { - if (format.audioQuality == null) continue; // no sound - if (format.width == null) continue; // no video - if (format.qualityLabel == quality) return format; - } - } - return null; - } - public function removeVideo():Void { if (video == null) return; isLoaded = false; diff --git a/src/server/Cache.hx b/src/server/Cache.hx new file mode 100644 index 0000000..8348476 --- /dev/null +++ b/src/server/Cache.hx @@ -0,0 +1,165 @@ +package server; + +import js.lib.Promise; +import js.node.ChildProcess; +import js.node.Fs.Fs; +import js.node.stream.Readable; +import sys.FileSystem; +import utils.YoutubeUtils; + +class Cache { + final main:Main; + final cacheDir:String; + + public final cachedFiles:Array<String> = []; + + public final isYtReady = false; + + /** In bytes **/ + public var storageLimit = 3 * 1024 * 1024 * 1024; + + public function new(main:Main, cacheDir:String) { + this.main = main; + this.cacheDir = cacheDir; + Utils.ensureDir(cacheDir); + isYtReady = checkYtDeps(); + } + + function checkYtDeps():Bool { + final ytdl = try { + untyped require("@distube/ytdl-core"); + } catch (e) { + return false; + } + try { + ChildProcess.execSync("ffmpeg -version", {stdio: "ignore", timeout: 3000}); + return true; + } catch (e) { + return false; + } + } + + function log(client:Client, msg:String):Void { + main.serverMessage(client, msg); + trace(msg); + } + + public function cacheYoutubeVideo(client:Client, url:String, callback:(name:String) -> Void) { + if (!isYtReady) { + trace("Do `npm i @distube/ytdl-core@latest` to use cache feature (you also need to install `ffmpeg` to build mp4 from downloaded audio/video tracks)."); + return; + } + final videoId = YoutubeUtils.extractVideoId(url); + if (videoId == "") { + log(client, 'Error: youtube video id not found in url: $url'); + return; + } + final outName = videoId + ".mp4"; + if (cachedFiles.contains(outName)) { + callback(outName); + return; + } + final ytdl:Dynamic = untyped require("@distube/ytdl-core"); + log(client, 'Caching $url to $outName...'); + final opts = {playerClients: ["IOS", "WEB_CREATOR"]}; + final promise:Promise<YouTubeVideoInfo> = ytdl.getInfo(url, opts); + promise.then(info -> { + // trace(info.formats.filter(item -> item.audioCodec != null)); + trace('Get info with ${info.formats.length} formats'); + final audioFormat:YoutubeVideoFormat = try { + ytdl.chooseFormat(info.formats.filter(item -> { + return item.audioCodec?.startsWith("mp4a"); + }), {quality: "highestaudio"}); + } catch (e) { + log(client, "Error: audio format not found"); + trace(e); + trace(info.formats); + return; + } + final videoFormat = getBestYoutubeVideoFormat(info.formats) ?? { + log(client, "Error: video format not found"); + trace(info.formats); + return; + } + trace("Picked audio and video formats"); + + final dlVideo:Readable<Dynamic> = ytdl(url, { + format: videoFormat, + playerClients: opts.playerClients + }); + dlVideo.pipe(Fs.createWriteStream('$cacheDir/input-video')); + dlVideo.on("error", err -> log(client, "Error during video download: " + err)); + + final dlAudio:Readable<Dynamic> = ytdl(url, { + format: audioFormat, + playerClients: opts.playerClients + }); + dlAudio.pipe(Fs.createWriteStream('$cacheDir/input-audio')); + dlAudio.on("error", err -> log(client, "Error during audio download: " + err)); + + var count = 0; + function onComplete(type:String):Void { + count++; + log(client, '$type track downloaded ($count/2)'); + if (count < 2) return; + final args = '-y -i input-video -i input-audio -c copy -map 0:v -map 1:a $outName'.split(" "); + final process = ChildProcess.spawn("ffmpeg", args, { + cwd: cacheDir, + stdio: "ignore" + }); + process.on("close", (code:Int) -> { + if (code != 0) { + log(client, 'Error: ffmpeg closed with code $code'); + return; + } + final inVideo = '$cacheDir/input-video'; + final inAudio = '$cacheDir/input-audio'; + if (FileSystem.exists(inVideo)) FileSystem.deleteFile(inVideo); + if (FileSystem.exists(inAudio)) FileSystem.deleteFile(inAudio); + + cachedFiles.push(outName); + removeOlderCache(); + + callback(outName); + }); + } + dlVideo.on("finish", () -> onComplete("Video")); + dlAudio.on("finish", () -> onComplete("Audio")); + // dlVideo.on('progress', (c, d, t) -> { + // final progress = Std.int((d / t * 100) * 10) / 10; + // trace(progress); + // }); + }).catchError(err -> { + log(client, "" + err); + }); + } + + function removeOlderCache():Void { + while (getUsedSpace() > storageLimit) { + final name = cachedFiles.shift(); + final path = '$cacheDir/$name'; + if (FileSystem.exists(path)) FileSystem.deleteFile(path); + } + } + + function getUsedSpace():Int { + var total = 0; + for (name in cachedFiles) { + final path = '$cacheDir/$name'; + total += FileSystem.stat(path).size; + } + return total; + } + + function getBestYoutubeVideoFormat(formats:Array<YoutubeVideoFormat>):Null<YoutubeVideoFormat> { + final qPriority = [1080, 720, 480, 360, 240]; + for (q in qPriority) { + final quality = '${q}p'; + for (format in formats) { + if (format.videoCodec == null) continue; + if (format.qualityLabel == quality) return format; + } + } + return null; + } +} diff --git a/src/server/Main.hx b/src/server/Main.hx index 9c2f6e2..b257e91 100644 --- a/src/server/Main.hx +++ b/src/server/Main.hx @@ -5,6 +5,7 @@ import Types.Config; import Types.FlashbackItem; import Types.Message; import Types.Permission; +import Types.PlayerType; import Types.UserList; import Types.VideoItem; import Types.WsEvent; @@ -48,12 +49,14 @@ class Main { var wss:WSServer; final localIp:String; var globalIp:String; + final playersCacheSupport:Array<PlayerType> = []; var port:Int; final userList:UserList; final clients:Array<Client> = []; final freeIds:Array<Int> = []; final wsEventParser = new JsonParser<WsEvent>(); final consoleInput:ConsoleInput; + final cache:Cache; final videoList = new VideoList(); final videoTimer = new VideoTimer(); final messages:Array<Message> = []; @@ -99,9 +102,12 @@ class Main { logger = new Logger(logsDir, 10, verbose); consoleInput = new ConsoleInput(this); consoleInput.initConsoleInput(); + cache = new Cache(this, '$rootDir/user/res/cache'); + if (cache.isYtReady) playersCacheSupport.push(YoutubeType); initIntergationHandlers(); loadState(); config = loadUserConfig(); + cache.storageLimit = cast config.cacheStorageLimitGiB * 1024 * 1024 * 1024; userList = loadUsers(); config.isVerbose = verbose; config.salt = generateConfigSalt(); @@ -276,7 +282,8 @@ class Main { time: videoTimer.getTime(), paused: videoTimer.isPaused() }, - flashbacks: flashbacks + flashbacks: flashbacks, + cachedFiles: cache.cachedFiles } } @@ -285,6 +292,9 @@ class Main { if (!FileSystem.exists(statePath)) return; trace("Loading state..."); final state:ServerState = Json.parse(File.getContent(statePath)); + state.flashbacks ??= []; + state.cachedFiles ??= []; + videoList.setItems(state.videoList); videoList.isOpen = state.isPlaylistOpen; videoList.setPos(state.itemPos); @@ -293,7 +303,10 @@ class Main { for (message in state.messages) messages.push(message); flashbacks.resize(0); - for (flashback in state.flashbacks ?? []) flashbacks.push(flashback); + for (flashback in state.flashbacks) flashbacks.push(flashback); + + cache.cachedFiles.resize(0); + for (name in state.cachedFiles) cache.cachedFiles.push(name); videoTimer.start(); videoTimer.setTime(state.timer.time); @@ -479,7 +492,8 @@ class Main { videoList: videoList.getItems(), isPlaylistOpen: videoList.isOpen, itemPos: videoList.pos, - globalIp: globalIp + globalIp: globalIp, + playersCacheSupport: playersCacheSupport, } }); sendClientListExcept(client); @@ -623,17 +637,6 @@ class Main { broadcast(data); case ServerMessage: - case GetYoutubeVideoInfo: - final url = data.getYoutubeVideoInfo.url; - YoutubeFallback.getInfo(url, info -> { - send(client, { - type: data.type, - getYoutubeVideoInfo: { - url: url, - response: info - } - }); - }); case AddVideo: if (isPlaylistLockedFor(client)) return; if (!checkPermission(client, AddVideoPerm)) return; @@ -660,11 +663,23 @@ class Main { serverMessage(client, "videoAlreadyExistsError"); return; } - data.addVideo.item = item; - videoList.addItem(item, data.addVideo.atEnd); - broadcast(data); - // Initial timer start if VideoLoaded is not happen - if (videoList.length == 1) restartWaitTimer(); + + inline function addVideo():Void { + data.addVideo.item = item; + videoList.addItem(item, data.addVideo.atEnd); + broadcast(data); + // Initial timer start if VideoLoaded is not happen + if (videoList.length == 1) restartWaitTimer(); + } + if (!item.doCache) { + addVideo(); + } else { + cache.cacheYoutubeVideo(client, item.url, (name) -> { + item = item.withUrl('/cache/$name'); + if (item.duration > 1) item.duration -= 1; + addVideo(); + }); + } case VideoLoaded: // Called if client loads next video and can play it @@ -939,7 +954,7 @@ class Main { }); } - function serverMessage(client:Client, textId:String):Void { + public function serverMessage(client:Client, textId:String):Void { send(client, { type: ServerMessage, serverMessage: { diff --git a/src/server/ServerState.hx b/src/server/ServerState.hx index 0369353..41f86ba 100644 --- a/src/server/ServerState.hx +++ b/src/server/ServerState.hx @@ -12,5 +12,6 @@ typedef ServerState = { timer:{ time:Float, paused:Bool }, - ?flashbacks:Array<FlashbackItem> + ?flashbacks:Array<FlashbackItem>, + ?cachedFiles:Array<String>, } diff --git a/src/server/YoutubeFallback.hx b/src/server/YoutubeFallback.hx deleted file mode 100644 index 5360283..0000000 --- a/src/server/YoutubeFallback.hx +++ /dev/null @@ -1,142 +0,0 @@ -package server; - -import haxe.Json; -import js.lib.Function; -import js.lib.Object; -import js.lib.Promise; -import js.node.Https.Https; -import js.node.Https.HttpsRequestOptions; -import js.node.url.URLSearchParams; -import utils.YoutubeUtils; - -class YoutubeFallback { - static function httpsGet( - url:String, - ?options:HttpsRequestOptions, - ?callback:(status:Int, data:String) -> Void - ):Void { - final request = Https.get(url, options, res -> { - var data = ""; - res.on("data", chunk -> data += chunk.toString()); - res.on("end", () -> callback(res.statusCode, data)); - }); - request.on("error", err -> { - trace(url); - trace("request error: ", err); - }); - } - - public static function resolvePlayerResponse(watchHtml:String):String { - if (watchHtml == null) return ""; - final resReg = ~/ytInitialPlayerResponse = (.*)}}};/; - var matches = resReg.match(watchHtml); - return matches ? resReg.matched(1) + "}}}" : ""; - } - - public static function resoleM3U8Link(watchHtml:String):Null<String> { - if (watchHtml == null) return null; - final hlsReg = ~/hlsManifestUrl":"(.*\/file\/index\.m3u8)/; - return hlsReg.match(watchHtml) ? hlsReg.matched(1) : null; - } - - public static function buildDecoder(watchHtml:String, callback:(decoder:(cipher:String) -> String) -> Void):Void { - if (watchHtml == null) return callback(null); - - final jsFileUrlReg = ~/\/s\/player\/[A-Za-z0-9]+\/[A-Za-z0-9_.]+\/[A-Za-z0-9_]+\/base\.js/; - if (!jsFileUrlReg.match(watchHtml)) return callback(null); - - final url = "https://www.youtube.com" + jsFileUrlReg.matched(0); - httpsGet(url, {}, (status, jsFileContent) -> { - final funcReg = ~/function.*\.split\(""\).*\.join\(""\)}/; - if (!funcReg.match(jsFileContent)) return callback(null); - - final decodeFunction = funcReg.matched(0); - final varNameReg = ~/\.split\(""\);([a-zA-Z0-9]+)\./i; - if (!varNameReg.match(decodeFunction)) return callback(null); - - final varStartIndex = jsFileContent.indexOf("var " + varNameReg.matched(1) + "={"); - if (varStartIndex < 0) return callback(null); - - final varEndIndex = jsFileContent.indexOf("}};", varStartIndex); - if (varEndIndex < 0) return callback(null); - - final varDeclares = jsFileContent.substring(varStartIndex, varEndIndex + 3); - if (varDeclares.length == 0) return callback(null); - - callback(signatureCipher -> { - final params = new URLSearchParams(signatureCipher); - final obj = Object.fromEntries(params); - final signature = obj.s; - final signatureParam = obj.sp ?? "signature"; - final url = obj.url; - final decodedSignature = new Function(' - "use strict"; - $varDeclares - return ($decodeFunction)("$signature"); - ').call(null); - return '$url&$signatureParam=${untyped encodeURIComponent(decodedSignature)}'; - }); - }); - } - - public static function getInfo(url:String, callback:(info:Null<YouTubeVideoInfo>) -> Void):Void { - final videoId = YoutubeUtils.extractVideoId(url); - if (videoId.length == 0) { - trace("youtube videoId is not found"); - return callback(null); - } - - final url = 'https://www.youtube.com/watch?v=$videoId'; - httpsGet(url, {}, (status, data) -> { - if (status != 200 || data.length == 0) { - trace("Cannot get youtube video response"); - return callback(null); - } - - final ytInitialPlayerResponse = resolvePlayerResponse(data); - final parsedResponse = Json.parse(ytInitialPlayerResponse); - final streamingData:YouTubeVideoInfo = parsedResponse.streamingData ?? cast {}; - streamingData.formats ??= []; - streamingData.adaptiveFormats ??= []; - var formats:Array<YoutubeVideoFormat> = streamingData.formats.concat(streamingData.adaptiveFormats); - - final promises:Array<Promise<Any>> = []; - - final isEncryptedVideo = formats.exists(it -> it.signatureCipher != null); - if (isEncryptedVideo) { - final promise = new Promise((resolve, reject) -> { - buildDecoder(data, decoder -> { - if (decoder != null) { - formats = formats.map(item -> { - if (item.url != null || item.signatureCipher == null) return item; - - item.url = decoder(item.signatureCipher); - item.signatureCipher = null; - return item; - }); - } - resolve(null); - }); - }); - promises.push(promise); - } - - Promise.all(promises).then(_ -> { - final result:YouTubeVideoInfo = { - videoDetails: parsedResponse.videoDetails ?? cast {}, - formats: formats.filter(format -> format.url != null) - }; - if (result.videoDetails.isLiveContent) { - final m3u8Link = resoleM3U8Link(data); - try { - result.liveData = { - manifestUrl: m3u8Link, - // data: m3u8Parser.getResult() - }; - } - } - callback(result); - }); - }); - } -} diff --git a/src/utils/YoutubeUtils.hx b/src/utils/YoutubeUtils.hx index 7429565..0bb8016 100644 --- a/src/utils/YoutubeUtils.hx +++ b/src/utils/YoutubeUtils.hx @@ -41,7 +41,11 @@ typedef YoutubeVideoFormat = { ?indexRange:{start:Int, end:Int}, ?audioQuality:String, // AUDIO_QUALITY_LOW ?audioSampleRate:Int, - ?audioChannels:Int + ?audioChannels:Int, + + ?container:String, + ?videoCodec:String, + ?audioCodec:String, } typedef YouTubeVideoInfo = { |
