diff options
| author | RblSb <msrblsb@gmail.com> | 2025-05-19 03:06:41 +0300 |
|---|---|---|
| committer | RblSb <msrblsb@gmail.com> | 2025-05-24 13:59:03 +0300 |
| commit | 623d85f88bb42834b335801ad5d703f6945d38d2 (patch) | |
| tree | 857983c5c90f429e4764e4880f95d8f12a0f4595 /src/server | |
| parent | 4b48de4f824ce48f1466014a9e9aa24023212181 (diff) | |
Migrate to yt-dlp
- yt-dlp should be more stable and allows more cool stuff in future
- easier to export cookies.txt with yt-dlp utility (cookies.json support removed)
- better quality fallback if not enough space
- keep progress reports after reconnections
Diffstat (limited to 'src/server')
| -rw-r--r-- | src/server/HttpServer.hx | 11 | ||||
| -rw-r--r-- | src/server/Main.hx | 8 | ||||
| -rw-r--r-- | src/server/cache/Cache.hx | 28 | ||||
| -rw-r--r-- | src/server/cache/RawCache.hx | 68 | ||||
| -rw-r--r-- | src/server/cache/YoutubeCache.hx | 266 |
5 files changed, 208 insertions, 173 deletions
diff --git a/src/server/HttpServer.hx b/src/server/HttpServer.hx index b15018b..4f283a4 100644 --- a/src/server/HttpServer.hx +++ b/src/server/HttpServer.hx @@ -41,10 +41,12 @@ class HttpServer { "jpeg" => "image/jpeg", "gif" => "image/gif", "webp" => "image/webp", + "avif" => "image/avif", "svg" => "image/svg+xml", "ico" => "image/x-icon", "wav" => "audio/wav", "mp3" => "audio/mpeg", + "ogg" => "audio/ogg", "mp4" => "video/mp4", "webm" => "video/webm", "woff" => "application/font-woff", @@ -371,7 +373,7 @@ class HttpServer { if (Utils.isOutOfRange(start, 0, videoSize - 1)) start = 0; var end = Std.parseInt(ranges[2]); if (end == null) end = start + CHUNK_SIZE; - if (Utils.isOutOfRange(end, start, videoSize - 1)) end = videoSize - 1; + if (Utils.isOutOfRange(end, start, videoSize - 1)) end = (videoSize - 1).limitMin(0); return { start: start, end: end @@ -379,7 +381,10 @@ class HttpServer { } function isMediaExtension(ext:String):Bool { - return ext == "mp4" || ext == "webm" || ext == "mp3" || ext == "wav"; + return switch ext { + case "mp4", "webm", "mp3", "ogg", "wav": true; + case _: false; + } } final matchLang = ~/^[A-z]+/; @@ -452,7 +457,7 @@ class HttpServer { } function getMimeType(ext:String):String { - return mimeTypes[ext] ?? return "application/octet-stream"; + return mimeTypes[ext] ?? "application/octet-stream"; } final ctrlCharacters = ~/[\u0000-\u001F\u007F-\u009F\u2000-\u200D\uFEFF]/g; diff --git a/src/server/Main.hx b/src/server/Main.hx index 8164922..3cde049 100644 --- a/src/server/Main.hx +++ b/src/server/Main.hx @@ -58,7 +58,9 @@ class Main { public final clients:Array<Client> = []; final freeIds:Array<Int> = []; + #if !display final wsEventParser = new JsonParser<WsEvent>(); + #end final consoleInput:ConsoleInput; final cache:Cache; final cacheDir:String; @@ -960,6 +962,7 @@ class Main { serverMessage(client, "Free space: " + (cache.getFreeSpace() / 1024).toFixed() + "KiB"); + serverMessage(client, "Memory usage: " + js.Node.process.memoryUsage()); send(client, { type: Dump, dump: { @@ -1006,6 +1009,11 @@ class Main { client.ws.send(jsonStringify(data), null); } + public function sendByName(clientName:String, data:WsEvent):Void { + final client = clients.getByName(clientName) ?? return; + client.ws.send(jsonStringify(data), null); + } + public function broadcast(data:WsEvent):Void { final json = jsonStringify(data); for (client in clients) diff --git a/src/server/cache/Cache.hx b/src/server/cache/Cache.hx index f71b465..56749d8 100644 --- a/src/server/cache/Cache.hx +++ b/src/server/cache/Cache.hx @@ -39,19 +39,37 @@ class Cache { cachedFiles.resize(0); for (name in names) cachedFiles.push(name); + removeUntrackedFiles(); + } + + function removeUntrackedFiles():Void { final names = FileSystem.readDirectory(cacheDir); for (name in names) { if (name.startsWith(".")) continue; if (FileSystem.isDirectory('$cacheDir/$name')) continue; if (cachedFiles.contains(name)) continue; - trace('Remove non-tracked cache $name'); + trace('Remove untracked cache $name'); remove(name); } } public function log(client:Client, msg:String):Void { + trace(msg); main.serverMessage(client, msg); + } + + public function logByName(clientName:String, msg:String):Void { trace(msg); + final client = main.clients.getByName(clientName) ?? return; + main.serverMessage(client, msg); + } + + public function logWithAdmins(client:Client, msg:String):Void { + log(client, msg); + final admins = main.clients.filter(client -> client.isAdmin); + for (admin in admins) { + if (client != admin) main.serverMessage(admin, msg); + } } public function cacheYoutubeVideo(client:Client, url:String, callback:(name:String) -> Void) { @@ -152,6 +170,14 @@ class Cache { return FileSystem.exists(getFilePath(name)); } + public function findFile(callback:(name:String) -> Bool):Null<String> { + final names = FileSystem.readDirectory(cacheDir); + for (name in names) { + if (callback(name)) return name; + } + return null; + } + public function getFreeSpace():Int { return storageLimit - getUsedSpace(); } diff --git a/src/server/cache/RawCache.hx b/src/server/cache/RawCache.hx index ed8679c..1fb251d 100644 --- a/src/server/cache/RawCache.hx +++ b/src/server/cache/RawCache.hx @@ -61,37 +61,39 @@ class RawCache { } function handleMp4(client:Client, url:String, outName:String, callback:(name:String) -> Void) { + final clientName = client.name; downloadFile(client, url, outName, (downloaded, total) -> { - main.send(client, { + main.sendByName(clientName, { type: Progress, progress: { type: Downloading, - ratio: (downloaded / total).clamp(0, 1) + ratio: (downloaded / total).clamp(0, 1).toFixed(4) } }); }, () -> { cache.add(outName); callback(outName); }, (err) -> { - log(client, 'Mp4 download failed: $err'); - cancelProgress(client); + log(clientName, 'Mp4 download failed: $err'); + cancelProgress(clientName); }); } function handleM3u8(client:Client, url:String, outName:String, callback:(name:String) -> Void):Void { + final clientName = client.name; final useProxy = true; downloadM3u8Playlist(client, url, useProxy, (playlist, totalSize, segments) -> { // only playlist file donwloaded if (useProxy) totalSize = playlist.length; if (!cache.removeOlderCache(totalSize + cache.freeSpaceBlock)) { - log(client, cache.notEnoughSpaceErrorText); - cancelProgress(client); + log(clientName, cache.notEnoughSpaceErrorText); + cancelProgress(clientName); return; } if (useProxy) { - main.send(client, { + main.sendByName(clientName, { type: Progress, progress: { type: Caching, @@ -126,7 +128,7 @@ class RawCache { downloaded++; final progress = downloaded / segments.length; - main.send(client, { + main.sendByName(clientName, { type: Progress, progress: { type: Downloading, @@ -154,8 +156,8 @@ class RawCache { (err) -> { activeDownloads--; downloaded++; - log(client, 'TS segment ${segment.i} download failed: $err'); - cancelProgress(client); + log(clientName, 'TS segment ${segment.i} download failed: $err'); + cancelProgress(clientName); cleanupFiles(segments.map(item -> item.name)); } ); @@ -165,8 +167,8 @@ class RawCache { // Start the initial batch of downloads downloadNextBatch(); }, (err) -> { - log(client, 'M3U8 processing failed: $err'); - cancelProgress(client); + log(clientName, 'M3U8 processing failed: $err'); + cancelProgress(clientName); }); } @@ -327,17 +329,12 @@ class RawCache { } function buildTsFiles(tempFiles:Array<String>, outName:String, client:Client, callback:String->Void) { + final clientName = client.name; final missingFiles = tempFiles.filter(f -> !FileSystem.exists('${cache.cacheDir}/$f')); if (missingFiles.length > 0) { - log(client, 'Concatenation failed: ${missingFiles.length} segments are missing'); - main.send(client, { - type: Progress, - progress: { - type: Canceled, - ratio: 1 - } - }); + log(clientName, 'Concatenation failed: ${missingFiles.length} segments are missing'); + cancelProgress(clientName); cleanupFiles(tempFiles); return; } @@ -378,14 +375,8 @@ class RawCache { final timeout = 5 * 60 * 1000; // 5 minutes final timeoutId = js.Node.setTimeout(() -> { process.kill(); - log(client, 'FFmpeg process timed out after ${timeout / 1000} seconds'); - main.send(client, { - type: Progress, - progress: { - type: Canceled, - ratio: 1 - } - }); + log(clientName, 'FFmpeg process timed out after ${timeout / 1000} seconds'); + cancelProgress(clientName); cleanupFiles(tempFiles.concat([concatFile])); }, timeout); @@ -394,14 +385,13 @@ class RawCache { if (code != 0) { final errorMsg = Buffer.concat(errorOutput).toString(); - log(client, 'FFmpeg concatenation failed with code $code'); - trace('FFmpeg error output: $errorMsg'); + cache.logWithAdmins(client, 'FFmpeg concatenation failed with code $code'); + final ffmpegErr = 'FFmpeg error output: $errorMsg'; + trace(ffmpegErr); // Log detailed error to admins final admins = main.clients.filter(client -> client.isAdmin); - for (admin in admins) { - log(admin, 'FFmpeg error: $errorMsg'); - } + for (admin in admins) main.serverMessage(admin, ffmpegErr); main.send(client, { type: Progress, @@ -417,7 +407,7 @@ class RawCache { cache.add(outName); callback(outName); } else { - log(client, 'FFmpeg process completed but output file is missing or empty'); + log(clientName, 'FFmpeg process completed but output file is missing or empty'); main.send(client, { type: Progress, progress: { @@ -435,7 +425,7 @@ class RawCache { // Handle process errors (like if FFmpeg isn't found) process.on("error", (err) -> { js.Node.clearTimeout(timeoutId); - log(client, 'Failed to start FFmpeg: $err'); + log(clientName, 'Failed to start FFmpeg: $err'); main.send(client, { type: Progress, progress: { @@ -453,12 +443,12 @@ class RawCache { } } - function log(client:Client, msg:String):Void { - cache.log(client, msg); + function log(clientName:String, msg:String):Void { + cache.logByName(clientName, msg); } - function cancelProgress(client:Client):Void { - main.send(client, { + function cancelProgress(clientName:String):Void { + main.sendByName(clientName, { type: Progress, progress: { type: Canceled, diff --git a/src/server/cache/YoutubeCache.hx b/src/server/cache/YoutubeCache.hx index c0a5c4c..af142e2 100644 --- a/src/server/cache/YoutubeCache.hx +++ b/src/server/cache/YoutubeCache.hx @@ -2,17 +2,17 @@ package server.cache; import haxe.Json; import js.lib.Promise; -import js.node.Buffer; import js.node.ChildProcess; -import js.node.Fs.Fs; -import js.node.stream.Readable; import sys.FileSystem; -import sys.io.File; import utils.YoutubeUtils; +import ytdlp_nodejs.VideoFormat; +import ytdlp_nodejs.VideoInfo; +import ytdlp_nodejs.YtDlp; class YoutubeCache { final main:Main; final cache:Cache; + var ytDlp:Null<YtDlp>; public function new(main:Main, cache:Cache):Void { this.main = main; @@ -20,35 +20,32 @@ class YoutubeCache { } public function checkYtDeps():Bool { - final ytdl = try { - untyped require("@distube/ytdl-core"); - } catch (e) { - return false; - } try { - ChildProcess.execSync("ffmpeg -version", {stdio: "ignore", timeout: 3000}); + ChildProcess.execSync("ffmpeg -version", {stdio: "ignore", timeout: 5000}); + ytDlp = js.Syntax.code("new (require('ytdlp-nodejs')).YtDlp()"); return true; } catch (e) { return false; } } - public function cleanYtInputFiles():Void { + public function cleanYtInputFiles(prefix = "__tmp"):Void { final names = FileSystem.readDirectory(cache.cacheDir); for (name in names) { - if (!name.startsWith("__tmp")) continue; + if (!name.startsWith(prefix)) continue; cache.remove(name); } } public function cacheYoutubeVideo(client:Client, url:String, callback:(name:String) -> Void) { if (!cache.isYtReady) { - trace("Do `npm i @distube/ytdl-core@latest` to use cache feature (you also need to install `ffmpeg` to build mp4 from downloaded audio/video tracks)."); + trace("Do `npm i https://github.com/RblSb/ytdlp-nodejs` to use cache feature (you also need to install `ffmpeg` to build mp4 from downloaded audio/video tracks)."); return; } + final clientName = client.name; final videoId = YoutubeUtils.extractVideoId(url); if (videoId == "") { - log(client, 'Error: youtube video id not found in url: $url'); + log(clientName, 'Error: youtube video id not found in url: $url'); return; } final outName = videoId + ".mp4"; @@ -57,28 +54,25 @@ class YoutubeCache { return; } final inVideoName = '__tmp-video-$videoId'; - final inAudioName = '__tmp-audio-$videoId'; inline function removeInputFiles():Void { - cache.remove(inVideoName); - cache.remove(inAudioName); + cleanYtInputFiles(inVideoName); } inline function checkEnoughSpace(contentLength:Int):Bool { final hasSpace = cache.removeOlderCache(contentLength + cache.freeSpaceBlock); if (!hasSpace) { removeInputFiles(); - cancelProgress(client); - log(client, cache.notEnoughSpaceErrorText); + cancelProgress(clientName); + log(clientName, cache.notEnoughSpaceErrorText); } return hasSpace; } if (cache.isFileExists(inVideoName)) { - log(client, 'Caching $outName already in progress'); + log(clientName, 'Caching $outName already in progress'); return; } - final ytdl:Dynamic = untyped require("@distube/ytdl-core"); trace('Caching $url to $outName...'); - main.send(client, { + main.sendByName(clientName, { type: Progress, progress: { type: Caching, @@ -86,146 +80,158 @@ class YoutubeCache { data: outName } }); - var agent:Any = null; - final cookiesPath = '${main.userDir}/cookies.json'; - if (FileSystem.exists(cookiesPath)) { - agent = ytdl.createAgent(Json.parse(File.getContent(cookiesPath))); - } - final promise:Promise<YouTubeVideoInfo> = ytdl.getInfo(url, { - agent: agent, - }); - promise.then(info -> { + + var useCookies = false; + + function onGetInfo(info:VideoInfo):Void { trace('Get info with ${info.formats.length} formats'); - final audioFormat:YoutubeVideoFormat = try { - ytdl.chooseFormat(info.formats.filter(item -> { - return item.audioCodec?.startsWith("mp4a"); - }), {quality: "highestaudio"}); - } catch (e) { - log(client, "Error: audio format not found"); - trace(e); - trace(info.formats.filter(item -> item.hasAudio)); + var aformats = info.formats.filter(format -> format.vcodec == "none"); + if (aformats.length == 0) { + aformats = info.formats.filter(format -> format.acodec != "none"); + } + aformats.sort((a, b) -> (a?.filesize ?? 0) < (b?.filesize ?? 0) ? 1 : -1); + final audioFormat:VideoFormat = aformats[0] ?? { + log(clientName, "Error: format with audio not found"); + for (format in info.formats) trace(format); return; } - var videoFormat = getBestYoutubeVideoFormat(info.formats) ?? { - log(client, "Error: video format not found"); - trace(info.formats.filter(item -> item.hasVideo)); + final vformats = info.formats.filter(format -> format.vcodec != "none"); + vformats.sort((a, b) -> (a?.filesize ?? 0) < (b?.filesize ?? 0) ? 1 : -1); + var videoFormat = getBestYoutubeVideoFormat(vformats) ?? { + log(clientName, "Error: video format not found"); + for (format in info.formats) trace(format); return; } inline function getTotalFormatsSize():Int { - final videoSize = Std.parseInt(videoFormat.contentLength) ?? 0; - final audioSize = Std.parseInt(audioFormat.contentLength) ?? 0; + final videoSize:Int = cast(videoFormat.filesize ?? 0); + final audioSize:Int = cast(audioFormat.filesize ?? 0); return videoSize + audioSize; } // check if we have space for formats and video build - final hasSpace = cache.removeOlderCache(getTotalFormatsSize() * 2 - + cache.freeSpaceBlock); - if (!hasSpace) { + final ignoreQualities:Array<Int> = []; + for (i in 0...3) { + final hasSpace = cache.removeOlderCache(getTotalFormatsSize() * 2 + + cache.freeSpaceBlock); + if (hasSpace) break; // try fallback to worse video quality - videoFormat = getBestYoutubeVideoFormat(info.formats, videoFormat.qualityLabel); - if (!checkEnoughSpace(getTotalFormatsSize() * 2)) return; + ignoreQualities.push(Std.int(videoFormat.height ?? 0)); + videoFormat = getBestYoutubeVideoFormat(vformats, ignoreQualities) ?? break; } + if (!checkEnoughSpace(getTotalFormatsSize() * 2)) return; - final dlVideo:Readable<Dynamic> = ytdl(url, { - format: videoFormat, - agent: agent, - }); - dlVideo.pipe(Fs.createWriteStream('${cache.cacheDir}/$inVideoName')); - dlVideo.on("error", err -> { - log(client, "Error during video download: " + err); + final formatIds = if (videoFormat.format_id == audioFormat.format_id) { + videoFormat.format_id; + } else { + '${videoFormat.format_id}+${audioFormat.format_id}'; + } + var totalSize = getTotalFormatsSize().limitMin(10); + var videoSizeRatio = (videoFormat.filesize ?? 0).limitMin(8) / totalSize; + var audioSizeRatio = (audioFormat.filesize ?? 0).limitMin(2) / totalSize; + var isVideoFormatDownloading = true; + final dlVideo:Promise<String> = ytDlp.downloadAsync(url, { + format: formatIds, + output: '${cache.cacheDir}/$inVideoName', + remuxVideo: "mp4", + cookies: useCookies ? getCookiesPathOrNull() : null, + onProgress: p -> { + final isFinished = p.status == "finished"; + var ratio = if (isFinished) { + 1; + } else { + (p.downloaded / p.total).clamp(0, 1); + } + if (isVideoFormatDownloading) { + ratio = ratio * videoSizeRatio; + } else { + ratio = videoSizeRatio + ratio * audioSizeRatio; + } + if (isFinished) isVideoFormatDownloading = false; + main.sendByName(clientName, { + type: Progress, + progress: { + type: Downloading, + ratio: ratio.toFixed(4) + } + }); + } + }).catchError(err -> { + final err = "Error during video download: " + err; + cache.logWithAdmins(client, err); removeInputFiles(); - cancelProgress(client); + cancelProgress(clientName); }); - final dlAudio:Readable<Dynamic> = ytdl(url, { - format: audioFormat, - agent: agent, - }); - dlAudio.pipe(Fs.createWriteStream('${cache.cacheDir}/$inAudioName')); - dlAudio.on("error", err -> { - log(client, "Error during audio download: " + err); + dlVideo.then((v) -> { + final name = cache.findFile(n -> n.startsWith(inVideoName) && n.endsWith(".mp4")) ?? { + final err = 'Error: cannot find downloaded file with prefix $inVideoName'; + cache.logWithAdmins(client, err); + return; + }; + FileSystem.rename('${cache.cacheDir}/$name', '${cache.cacheDir}/$outName'); removeInputFiles(); - cancelProgress(client); + cache.add(outName); + callback(outName); }); + } - var count = 0; - function onComplete(type:String):Void { - count++; - trace('$type track downloaded ($count/2)'); - if (count < 2) return; - if (!cache.isFileExists(inVideoName) || !cache.isFileExists(inAudioName)) { - log(client, "Input files not found for making final video"); - removeInputFiles(); - cancelProgress(client); - return; - } - var size = FileSystem.stat('${cache.cacheDir}/$inVideoName').size; - size += FileSystem.stat('${cache.cacheDir}/$inAudioName').size; - // clean some space for full mp4 - if (!checkEnoughSpace(size)) return; - - final args = '-y -i ./$inVideoName -i ./$inAudioName -c copy -map 0:v -map 1:a ./$outName'.split(" "); - final process = ChildProcess.spawn("ffmpeg", args, { - cwd: cache.cacheDir, - // stdio: "ignore" - }); - final outputData:Array<Buffer> = []; - process.stderr.on("data", (data) -> outputData.push(data)); - process.on("close", (code:Int) -> { - removeInputFiles(); - if (code != 0) { - cancelProgress(client); - final errCodeMsg = 'Error: ffmpeg closed with code $code'; - final admins = main.clients.filter(client -> client.isAdmin); - for (client in admins) { - log(client, Buffer.concat(outputData).toString()); - log(client, errCodeMsg); - } - if (!admins.contains(client)) log(client, errCodeMsg); - return; - } - cache.add(outName); - - callback(outName); - }); - } - dlVideo.on("finish", () -> onComplete("Video")); - dlAudio.on("finish", () -> onComplete("Audio")); - dlVideo.on("progress", (chunkLength:Int, downloaded:Int, contentLength:Int) -> { - final ratio = (downloaded / contentLength).clamp(0, 1); - main.send(client, { - type: Progress, - progress: { - type: Downloading, - ratio: ratio - } - }); + getInfoAsync(url, useCookies).then(onGetInfo).catchError(err -> { + trace(err); + useCookies = true; + getInfoAsync(url, useCookies).then(onGetInfo).catchError(err -> { + removeInputFiles(); + cancelProgress(clientName); + log(clientName, "" + err); }); - }).catchError(err -> { - removeInputFiles(); - cancelProgress(client); - log(client, "" + err); }); } - function getBestYoutubeVideoFormat(formats:Array<YoutubeVideoFormat>, ?ignoreQuality:String):Null<YoutubeVideoFormat> { + function getInfoAsync(url:String, useCookies = false):Promise<VideoInfo> { + return ytDlp.execAsync(url, { + dumpSingleJson: true, + quiet: true, + cookies: useCookies ? getCookiesPathOrNull() : null, + }).then(data -> Json.parse(data)); + } + + function getCookiesPathOrNull():Null<String> { + final cookiesPath = '${main.userDir}/cookies.txt'; + return FileSystem.exists(cookiesPath) ? cookiesPath : null; + } + + function getBestYoutubeVideoFormat(formats:Array<VideoFormat>, ?ignoreQualities:Array<Int>):Null<VideoFormat> { final qPriority = [1080, 720, 480, 360, 240, 144]; + if (ignoreQualities != null) { + for (q in ignoreQualities) qPriority.remove(q); + } + final format60 = findFormat(formats, qPriority, true); + return format60 ?? findFormat(formats, qPriority, false); + } + + function findFormat(formats:Array<VideoFormat>, qPriority:Array<Int>, is60fps:Bool):Null<VideoFormat> { for (q in qPriority) { - final quality = '${q}p'; - if (quality == ignoreQuality) continue; + final quality = '${q}p' + (is60fps ? "60" : ""); for (format in formats) { - if (format.videoCodec == null) continue; - if (format.qualityLabel == quality) return format; + final height = format.height ?? continue; + if (height > q) continue; + final format_note = formatVideoQuality(format); + if (format_note == quality) return format; } } return null; } - function log(client:Client, msg:String):Void { - cache.log(client, msg); + function formatVideoQuality(format:VideoFormat):Null<String> { + final height = format.height ?? return null; + // when there is 720p and 720p60 formats + return format.format_note ?? '${height}p'; + } + + function log(clientName:String, msg:String):Void { + cache.logByName(clientName, msg); } - function cancelProgress(client:Client):Void { - main.send(client, { + function cancelProgress(clientName:String):Void { + main.sendByName(clientName, { type: Progress, progress: { type: Canceled, |
