aboutsummaryrefslogtreecommitdiffstats
path: root/src/server/cache/YoutubeCache.hx
diff options
context:
space:
mode:
Diffstat (limited to 'src/server/cache/YoutubeCache.hx')
-rw-r--r--src/server/cache/YoutubeCache.hx266
1 files changed, 136 insertions, 130 deletions
diff --git a/src/server/cache/YoutubeCache.hx b/src/server/cache/YoutubeCache.hx
index c0a5c4c..af142e2 100644
--- a/src/server/cache/YoutubeCache.hx
+++ b/src/server/cache/YoutubeCache.hx
@@ -2,17 +2,17 @@ package server.cache;
import haxe.Json;
import js.lib.Promise;
-import js.node.Buffer;
import js.node.ChildProcess;
-import js.node.Fs.Fs;
-import js.node.stream.Readable;
import sys.FileSystem;
-import sys.io.File;
import utils.YoutubeUtils;
+import ytdlp_nodejs.VideoFormat;
+import ytdlp_nodejs.VideoInfo;
+import ytdlp_nodejs.YtDlp;
class YoutubeCache {
final main:Main;
final cache:Cache;
+ var ytDlp:Null<YtDlp>;
public function new(main:Main, cache:Cache):Void {
this.main = main;
@@ -20,35 +20,32 @@ class YoutubeCache {
}
public function checkYtDeps():Bool {
- final ytdl = try {
- untyped require("@distube/ytdl-core");
- } catch (e) {
- return false;
- }
try {
- ChildProcess.execSync("ffmpeg -version", {stdio: "ignore", timeout: 3000});
+ ChildProcess.execSync("ffmpeg -version", {stdio: "ignore", timeout: 5000});
+ ytDlp = js.Syntax.code("new (require('ytdlp-nodejs')).YtDlp()");
return true;
} catch (e) {
return false;
}
}
- public function cleanYtInputFiles():Void {
+ public function cleanYtInputFiles(prefix = "__tmp"):Void {
final names = FileSystem.readDirectory(cache.cacheDir);
for (name in names) {
- if (!name.startsWith("__tmp")) continue;
+ if (!name.startsWith(prefix)) continue;
cache.remove(name);
}
}
public function cacheYoutubeVideo(client:Client, url:String, callback:(name:String) -> Void) {
if (!cache.isYtReady) {
- trace("Do `npm i @distube/ytdl-core@latest` to use cache feature (you also need to install `ffmpeg` to build mp4 from downloaded audio/video tracks).");
+ trace("Do `npm i https://github.com/RblSb/ytdlp-nodejs` to use cache feature (you also need to install `ffmpeg` to build mp4 from downloaded audio/video tracks).");
return;
}
+ final clientName = client.name;
final videoId = YoutubeUtils.extractVideoId(url);
if (videoId == "") {
- log(client, 'Error: youtube video id not found in url: $url');
+ log(clientName, 'Error: youtube video id not found in url: $url');
return;
}
final outName = videoId + ".mp4";
@@ -57,28 +54,25 @@ class YoutubeCache {
return;
}
final inVideoName = '__tmp-video-$videoId';
- final inAudioName = '__tmp-audio-$videoId';
inline function removeInputFiles():Void {
- cache.remove(inVideoName);
- cache.remove(inAudioName);
+ cleanYtInputFiles(inVideoName);
}
inline function checkEnoughSpace(contentLength:Int):Bool {
final hasSpace = cache.removeOlderCache(contentLength + cache.freeSpaceBlock);
if (!hasSpace) {
removeInputFiles();
- cancelProgress(client);
- log(client, cache.notEnoughSpaceErrorText);
+ cancelProgress(clientName);
+ log(clientName, cache.notEnoughSpaceErrorText);
}
return hasSpace;
}
if (cache.isFileExists(inVideoName)) {
- log(client, 'Caching $outName already in progress');
+ log(clientName, 'Caching $outName already in progress');
return;
}
- final ytdl:Dynamic = untyped require("@distube/ytdl-core");
trace('Caching $url to $outName...');
- main.send(client, {
+ main.sendByName(clientName, {
type: Progress,
progress: {
type: Caching,
@@ -86,146 +80,158 @@ class YoutubeCache {
data: outName
}
});
- var agent:Any = null;
- final cookiesPath = '${main.userDir}/cookies.json';
- if (FileSystem.exists(cookiesPath)) {
- agent = ytdl.createAgent(Json.parse(File.getContent(cookiesPath)));
- }
- final promise:Promise<YouTubeVideoInfo> = ytdl.getInfo(url, {
- agent: agent,
- });
- promise.then(info -> {
+
+ var useCookies = false;
+
+ function onGetInfo(info:VideoInfo):Void {
trace('Get info with ${info.formats.length} formats');
- final audioFormat:YoutubeVideoFormat = try {
- ytdl.chooseFormat(info.formats.filter(item -> {
- return item.audioCodec?.startsWith("mp4a");
- }), {quality: "highestaudio"});
- } catch (e) {
- log(client, "Error: audio format not found");
- trace(e);
- trace(info.formats.filter(item -> item.hasAudio));
+ var aformats = info.formats.filter(format -> format.vcodec == "none");
+ if (aformats.length == 0) {
+ aformats = info.formats.filter(format -> format.acodec != "none");
+ }
+ aformats.sort((a, b) -> (a?.filesize ?? 0) < (b?.filesize ?? 0) ? 1 : -1);
+ final audioFormat:VideoFormat = aformats[0] ?? {
+ log(clientName, "Error: format with audio not found");
+ for (format in info.formats) trace(format);
return;
}
- var videoFormat = getBestYoutubeVideoFormat(info.formats) ?? {
- log(client, "Error: video format not found");
- trace(info.formats.filter(item -> item.hasVideo));
+ final vformats = info.formats.filter(format -> format.vcodec != "none");
+ vformats.sort((a, b) -> (a?.filesize ?? 0) < (b?.filesize ?? 0) ? 1 : -1);
+ var videoFormat = getBestYoutubeVideoFormat(vformats) ?? {
+ log(clientName, "Error: video format not found");
+ for (format in info.formats) trace(format);
return;
}
inline function getTotalFormatsSize():Int {
- final videoSize = Std.parseInt(videoFormat.contentLength) ?? 0;
- final audioSize = Std.parseInt(audioFormat.contentLength) ?? 0;
+ final videoSize:Int = cast(videoFormat.filesize ?? 0);
+ final audioSize:Int = cast(audioFormat.filesize ?? 0);
return videoSize + audioSize;
}
// check if we have space for formats and video build
- final hasSpace = cache.removeOlderCache(getTotalFormatsSize() * 2
- + cache.freeSpaceBlock);
- if (!hasSpace) {
+ final ignoreQualities:Array<Int> = [];
+ for (i in 0...3) {
+ final hasSpace = cache.removeOlderCache(getTotalFormatsSize() * 2
+ + cache.freeSpaceBlock);
+ if (hasSpace) break;
// try fallback to worse video quality
- videoFormat = getBestYoutubeVideoFormat(info.formats, videoFormat.qualityLabel);
- if (!checkEnoughSpace(getTotalFormatsSize() * 2)) return;
+ ignoreQualities.push(Std.int(videoFormat.height ?? 0));
+ videoFormat = getBestYoutubeVideoFormat(vformats, ignoreQualities) ?? break;
}
+ if (!checkEnoughSpace(getTotalFormatsSize() * 2)) return;
- final dlVideo:Readable<Dynamic> = ytdl(url, {
- format: videoFormat,
- agent: agent,
- });
- dlVideo.pipe(Fs.createWriteStream('${cache.cacheDir}/$inVideoName'));
- dlVideo.on("error", err -> {
- log(client, "Error during video download: " + err);
+ final formatIds = if (videoFormat.format_id == audioFormat.format_id) {
+ videoFormat.format_id;
+ } else {
+ '${videoFormat.format_id}+${audioFormat.format_id}';
+ }
+ var totalSize = getTotalFormatsSize().limitMin(10);
+ var videoSizeRatio = (videoFormat.filesize ?? 0).limitMin(8) / totalSize;
+ var audioSizeRatio = (audioFormat.filesize ?? 0).limitMin(2) / totalSize;
+ var isVideoFormatDownloading = true;
+ final dlVideo:Promise<String> = ytDlp.downloadAsync(url, {
+ format: formatIds,
+ output: '${cache.cacheDir}/$inVideoName',
+ remuxVideo: "mp4",
+ cookies: useCookies ? getCookiesPathOrNull() : null,
+ onProgress: p -> {
+ final isFinished = p.status == "finished";
+ var ratio = if (isFinished) {
+ 1;
+ } else {
+ (p.downloaded / p.total).clamp(0, 1);
+ }
+ if (isVideoFormatDownloading) {
+ ratio = ratio * videoSizeRatio;
+ } else {
+ ratio = videoSizeRatio + ratio * audioSizeRatio;
+ }
+ if (isFinished) isVideoFormatDownloading = false;
+ main.sendByName(clientName, {
+ type: Progress,
+ progress: {
+ type: Downloading,
+ ratio: ratio.toFixed(4)
+ }
+ });
+ }
+ }).catchError(err -> {
+ final err = "Error during video download: " + err;
+ cache.logWithAdmins(client, err);
removeInputFiles();
- cancelProgress(client);
+ cancelProgress(clientName);
});
- final dlAudio:Readable<Dynamic> = ytdl(url, {
- format: audioFormat,
- agent: agent,
- });
- dlAudio.pipe(Fs.createWriteStream('${cache.cacheDir}/$inAudioName'));
- dlAudio.on("error", err -> {
- log(client, "Error during audio download: " + err);
+ dlVideo.then((v) -> {
+ final name = cache.findFile(n -> n.startsWith(inVideoName) && n.endsWith(".mp4")) ?? {
+ final err = 'Error: cannot find downloaded file with prefix $inVideoName';
+ cache.logWithAdmins(client, err);
+ return;
+ };
+ FileSystem.rename('${cache.cacheDir}/$name', '${cache.cacheDir}/$outName');
removeInputFiles();
- cancelProgress(client);
+ cache.add(outName);
+ callback(outName);
});
+ }
- var count = 0;
- function onComplete(type:String):Void {
- count++;
- trace('$type track downloaded ($count/2)');
- if (count < 2) return;
- if (!cache.isFileExists(inVideoName) || !cache.isFileExists(inAudioName)) {
- log(client, "Input files not found for making final video");
- removeInputFiles();
- cancelProgress(client);
- return;
- }
- var size = FileSystem.stat('${cache.cacheDir}/$inVideoName').size;
- size += FileSystem.stat('${cache.cacheDir}/$inAudioName').size;
- // clean some space for full mp4
- if (!checkEnoughSpace(size)) return;
-
- final args = '-y -i ./$inVideoName -i ./$inAudioName -c copy -map 0:v -map 1:a ./$outName'.split(" ");
- final process = ChildProcess.spawn("ffmpeg", args, {
- cwd: cache.cacheDir,
- // stdio: "ignore"
- });
- final outputData:Array<Buffer> = [];
- process.stderr.on("data", (data) -> outputData.push(data));
- process.on("close", (code:Int) -> {
- removeInputFiles();
- if (code != 0) {
- cancelProgress(client);
- final errCodeMsg = 'Error: ffmpeg closed with code $code';
- final admins = main.clients.filter(client -> client.isAdmin);
- for (client in admins) {
- log(client, Buffer.concat(outputData).toString());
- log(client, errCodeMsg);
- }
- if (!admins.contains(client)) log(client, errCodeMsg);
- return;
- }
- cache.add(outName);
-
- callback(outName);
- });
- }
- dlVideo.on("finish", () -> onComplete("Video"));
- dlAudio.on("finish", () -> onComplete("Audio"));
- dlVideo.on("progress", (chunkLength:Int, downloaded:Int, contentLength:Int) -> {
- final ratio = (downloaded / contentLength).clamp(0, 1);
- main.send(client, {
- type: Progress,
- progress: {
- type: Downloading,
- ratio: ratio
- }
- });
+ getInfoAsync(url, useCookies).then(onGetInfo).catchError(err -> {
+ trace(err);
+ useCookies = true;
+ getInfoAsync(url, useCookies).then(onGetInfo).catchError(err -> {
+ removeInputFiles();
+ cancelProgress(clientName);
+ log(clientName, "" + err);
});
- }).catchError(err -> {
- removeInputFiles();
- cancelProgress(client);
- log(client, "" + err);
});
}
- function getBestYoutubeVideoFormat(formats:Array<YoutubeVideoFormat>, ?ignoreQuality:String):Null<YoutubeVideoFormat> {
+ function getInfoAsync(url:String, useCookies = false):Promise<VideoInfo> {
+ return ytDlp.execAsync(url, {
+ dumpSingleJson: true,
+ quiet: true,
+ cookies: useCookies ? getCookiesPathOrNull() : null,
+ }).then(data -> Json.parse(data));
+ }
+
+ function getCookiesPathOrNull():Null<String> {
+ final cookiesPath = '${main.userDir}/cookies.txt';
+ return FileSystem.exists(cookiesPath) ? cookiesPath : null;
+ }
+
+ function getBestYoutubeVideoFormat(formats:Array<VideoFormat>, ?ignoreQualities:Array<Int>):Null<VideoFormat> {
final qPriority = [1080, 720, 480, 360, 240, 144];
+ if (ignoreQualities != null) {
+ for (q in ignoreQualities) qPriority.remove(q);
+ }
+ final format60 = findFormat(formats, qPriority, true);
+ return format60 ?? findFormat(formats, qPriority, false);
+ }
+
+ function findFormat(formats:Array<VideoFormat>, qPriority:Array<Int>, is60fps:Bool):Null<VideoFormat> {
for (q in qPriority) {
- final quality = '${q}p';
- if (quality == ignoreQuality) continue;
+ final quality = '${q}p' + (is60fps ? "60" : "");
for (format in formats) {
- if (format.videoCodec == null) continue;
- if (format.qualityLabel == quality) return format;
+ final height = format.height ?? continue;
+ if (height > q) continue;
+ final format_note = formatVideoQuality(format);
+ if (format_note == quality) return format;
}
}
return null;
}
- function log(client:Client, msg:String):Void {
- cache.log(client, msg);
+ function formatVideoQuality(format:VideoFormat):Null<String> {
+ final height = format.height ?? return null;
+ // when there is 720p and 720p60 formats
+ return format.format_note ?? '${height}p';
+ }
+
+ function log(clientName:String, msg:String):Void {
+ cache.logByName(clientName, msg);
}
- function cancelProgress(client:Client):Void {
- main.send(client, {
+ function cancelProgress(clientName:String):Void {
+ main.sendByName(clientName, {
type: Progress,
progress: {
type: Canceled,
send patches to the email below
yukais@pinapelz.com
include the subject [PATCH repo_name]
pinapelz.com
homepage