Cache on server feature

Server will download video from supported players and add as raw video to playlist (only youtube is supported for now).

Cache for YT player is available after installing optional dependencies, see readme. For cache size see `cacheStorageLimitGiB ` in config.

There is also minor ux improvement, latest checkbox states will be keeped in local storage now.

4 files changed, 202 insertions, 163 deletions

diff --git a/src/server/Cache.hx b/src/server/Cache.hx
new file mode 100644
index 0000000..8348476
--- /dev/null
+++ b/src/server/Cache.hx
@@ -0,0 +1,165 @@
+package server;
+
+import js.lib.Promise;
+import js.node.ChildProcess;
+import js.node.Fs.Fs;
+import js.node.stream.Readable;
+import sys.FileSystem;
+import utils.YoutubeUtils;
+
+class Cache {
+	final main:Main;
+	final cacheDir:String;
+
+	public final cachedFiles:Array<String> = [];
+
+	public final isYtReady = false;
+
+	/** In bytes **/
+	public var storageLimit = 3 * 1024 * 1024 * 1024;
+
+	public function new(main:Main, cacheDir:String) {
+		this.main = main;
+		this.cacheDir = cacheDir;
+		Utils.ensureDir(cacheDir);
+		isYtReady = checkYtDeps();
+	}
+
+	function checkYtDeps():Bool {
+		final ytdl = try {
+			untyped require("@distube/ytdl-core");
+		} catch (e) {
+			return false;
+		}
+		try {
+			ChildProcess.execSync("ffmpeg -version", {stdio: "ignore", timeout: 3000});
+			return true;
+		} catch (e) {
+			return false;
+		}
+	}
+
+	function log(client:Client, msg:String):Void {
+		main.serverMessage(client, msg);
+		trace(msg);
+	}
+
+	public function cacheYoutubeVideo(client:Client, url:String, callback:(name:String) -> Void) {
+		if (!isYtReady) {
+			trace("Do `npm i @distube/ytdl-core@latest` to use cache feature (you also need to install `ffmpeg` to build mp4 from downloaded audio/video tracks).");
+			return;
+		}
+		final videoId = YoutubeUtils.extractVideoId(url);
+		if (videoId == "") {
+			log(client, 'Error: youtube video id not found in url: $url');
+			return;
+		}
+		final outName = videoId + ".mp4";
+		if (cachedFiles.contains(outName)) {
+			callback(outName);
+			return;
+		}
+		final ytdl:Dynamic = untyped require("@distube/ytdl-core");
+		log(client, 'Caching $url to $outName...');
+		final opts = {playerClients: ["IOS", "WEB_CREATOR"]};
+		final promise:Promise<YouTubeVideoInfo> = ytdl.getInfo(url, opts);
+		promise.then(info -> {
+			// trace(info.formats.filter(item -> item.audioCodec != null));
+			trace('Get info with ${info.formats.length} formats');
+			final audioFormat:YoutubeVideoFormat = try {
+				ytdl.chooseFormat(info.formats.filter(item -> {
+					return item.audioCodec?.startsWith("mp4a");
+				}), {quality: "highestaudio"});
+			} catch (e) {
+				log(client, "Error: audio format not found");
+				trace(e);
+				trace(info.formats);
+				return;
+			}
+			final videoFormat = getBestYoutubeVideoFormat(info.formats) ?? {
+				log(client, "Error: video format not found");
+				trace(info.formats);
+				return;
+			}
+			trace("Picked audio and video formats");
+
+			final dlVideo:Readable<Dynamic> = ytdl(url, {
+				format: videoFormat,
+				playerClients: opts.playerClients
+			});
+			dlVideo.pipe(Fs.createWriteStream('$cacheDir/input-video'));
+			dlVideo.on("error", err -> log(client, "Error during video download: " + err));
+
+			final dlAudio:Readable<Dynamic> = ytdl(url, {
+				format: audioFormat,
+				playerClients: opts.playerClients
+			});
+			dlAudio.pipe(Fs.createWriteStream('$cacheDir/input-audio'));
+			dlAudio.on("error", err -> log(client, "Error during audio download: " + err));
+
+			var count = 0;
+			function onComplete(type:String):Void {
+				count++;
+				log(client, '$type track downloaded ($count/2)');
+				if (count < 2) return;
+				final args = '-y -i input-video -i input-audio -c copy -map 0:v -map 1:a $outName'.split(" ");
+				final process = ChildProcess.spawn("ffmpeg", args, {
+					cwd: cacheDir,
+					stdio: "ignore"
+				});
+				process.on("close", (code:Int) -> {
+					if (code != 0) {
+						log(client, 'Error: ffmpeg closed with code $code');
+						return;
+					}
+					final inVideo = '$cacheDir/input-video';
+					final inAudio = '$cacheDir/input-audio';
+					if (FileSystem.exists(inVideo)) FileSystem.deleteFile(inVideo);
+					if (FileSystem.exists(inAudio)) FileSystem.deleteFile(inAudio);
+
+					cachedFiles.push(outName);
+					removeOlderCache();
+
+					callback(outName);
+				});
+			}
+			dlVideo.on("finish", () -> onComplete("Video"));
+			dlAudio.on("finish", () -> onComplete("Audio"));
+			// dlVideo.on('progress', (c, d, t) -> {
+			// 	final progress = Std.int((d / t * 100) * 10) / 10;
+			// 	trace(progress);
+			// });
+		}).catchError(err -> {
+			log(client, "" + err);
+		});
+	}
+
+	function removeOlderCache():Void {
+		while (getUsedSpace() > storageLimit) {
+			final name = cachedFiles.shift();
+			final path = '$cacheDir/$name';
+			if (FileSystem.exists(path)) FileSystem.deleteFile(path);
+		}
+	}
+
+	function getUsedSpace():Int {
+		var total = 0;
+		for (name in cachedFiles) {
+			final path = '$cacheDir/$name';
+			total += FileSystem.stat(path).size;
+		}
+		return total;
+	}
+
+	function getBestYoutubeVideoFormat(formats:Array<YoutubeVideoFormat>):Null<YoutubeVideoFormat> {
+		final qPriority = [1080, 720, 480, 360, 240];
+		for (q in qPriority) {
+			final quality = '${q}p';
+			for (format in formats) {
+				if (format.videoCodec == null) continue;
+				if (format.qualityLabel == quality) return format;
+			}
+		}
+		return null;
+	}
+}
diff --git a/src/server/Main.hx b/src/server/Main.hx
index 9c2f6e2..b257e91 100644
--- a/src/server/Main.hx
+++ b/src/server/Main.hx
@@ -5,6 +5,7 @@ import Types.Config;
 import Types.FlashbackItem;
 import Types.Message;
 import Types.Permission;
+import Types.PlayerType;
 import Types.UserList;
 import Types.VideoItem;
 import Types.WsEvent;
@@ -48,12 +49,14 @@ class Main {
 	var wss:WSServer;
 	final localIp:String;
 	var globalIp:String;
+	final playersCacheSupport:Array<PlayerType> = [];
 	var port:Int;
 	final userList:UserList;
 	final clients:Array<Client> = [];
 	final freeIds:Array<Int> = [];
 	final wsEventParser = new JsonParser<WsEvent>();
 	final consoleInput:ConsoleInput;
+	final cache:Cache;
 	final videoList = new VideoList();
 	final videoTimer = new VideoTimer();
 	final messages:Array<Message> = [];
@@ -99,9 +102,12 @@ class Main {
 		logger = new Logger(logsDir, 10, verbose);
 		consoleInput = new ConsoleInput(this);
 		consoleInput.initConsoleInput();
+		cache = new Cache(this, '$rootDir/user/res/cache');
+		if (cache.isYtReady) playersCacheSupport.push(YoutubeType);
 		initIntergationHandlers();
 		loadState();
 		config = loadUserConfig();
+		cache.storageLimit = cast config.cacheStorageLimitGiB * 1024 * 1024 * 1024;
 		userList = loadUsers();
 		config.isVerbose = verbose;
 		config.salt = generateConfigSalt();
@@ -276,7 +282,8 @@ class Main {
 				time: videoTimer.getTime(),
 				paused: videoTimer.isPaused()
 			},
-			flashbacks: flashbacks
+			flashbacks: flashbacks,
+			cachedFiles: cache.cachedFiles
 		}
 	}
 
@@ -285,6 +292,9 @@ class Main {
 		if (!FileSystem.exists(statePath)) return;
 		trace("Loading state...");
 		final state:ServerState = Json.parse(File.getContent(statePath));
+		state.flashbacks ??= [];
+		state.cachedFiles ??= [];
+
 		videoList.setItems(state.videoList);
 		videoList.isOpen = state.isPlaylistOpen;
 		videoList.setPos(state.itemPos);
@@ -293,7 +303,10 @@ class Main {
 		for (message in state.messages) messages.push(message);
 
 		flashbacks.resize(0);
-		for (flashback in state.flashbacks ?? []) flashbacks.push(flashback);
+		for (flashback in state.flashbacks) flashbacks.push(flashback);
+
+		cache.cachedFiles.resize(0);
+		for (name in state.cachedFiles) cache.cachedFiles.push(name);
 
 		videoTimer.start();
 		videoTimer.setTime(state.timer.time);
@@ -479,7 +492,8 @@ class Main {
 						videoList: videoList.getItems(),
 						isPlaylistOpen: videoList.isOpen,
 						itemPos: videoList.pos,
-						globalIp: globalIp
+						globalIp: globalIp,
+						playersCacheSupport: playersCacheSupport,
 					}
 				});
 				sendClientListExcept(client);
@@ -623,17 +637,6 @@ class Main {
 				broadcast(data);
 
 			case ServerMessage:
-			case GetYoutubeVideoInfo:
-				final url = data.getYoutubeVideoInfo.url;
-				YoutubeFallback.getInfo(url, info -> {
-					send(client, {
-						type: data.type,
-						getYoutubeVideoInfo: {
-							url: url,
-							response: info
-						}
-					});
-				});
 			case AddVideo:
 				if (isPlaylistLockedFor(client)) return;
 				if (!checkPermission(client, AddVideoPerm)) return;
@@ -660,11 +663,23 @@ class Main {
 					serverMessage(client, "videoAlreadyExistsError");
 					return;
 				}
-				data.addVideo.item = item;
-				videoList.addItem(item, data.addVideo.atEnd);
-				broadcast(data);
-				// Initial timer start if VideoLoaded is not happen
-				if (videoList.length == 1) restartWaitTimer();
+
+				inline function addVideo():Void {
+					data.addVideo.item = item;
+					videoList.addItem(item, data.addVideo.atEnd);
+					broadcast(data);
+					// Initial timer start if VideoLoaded is not happen
+					if (videoList.length == 1) restartWaitTimer();
+				}
+				if (!item.doCache) {
+					addVideo();
+				} else {
+					cache.cacheYoutubeVideo(client, item.url, (name) -> {
+						item = item.withUrl('/cache/$name');
+						if (item.duration > 1) item.duration -= 1;
+						addVideo();
+					});
+				}
 
 			case VideoLoaded:
 				// Called if client loads next video and can play it
@@ -939,7 +954,7 @@ class Main {
 		});
 	}
 
-	function serverMessage(client:Client, textId:String):Void {
+	public function serverMessage(client:Client, textId:String):Void {
 		send(client, {
 			type: ServerMessage,
 			serverMessage: {
diff --git a/src/server/ServerState.hx b/src/server/ServerState.hx
index 0369353..41f86ba 100644
--- a/src/server/ServerState.hx
+++ b/src/server/ServerState.hx
@@ -12,5 +12,6 @@ typedef ServerState = {
 	timer:{
 		time:Float, paused:Bool
 	},
-	?flashbacks:Array<FlashbackItem>
+	?flashbacks:Array<FlashbackItem>,
+	?cachedFiles:Array<String>,
 }
diff --git a/src/server/YoutubeFallback.hx b/src/server/YoutubeFallback.hx
deleted file mode 100644
index 5360283..0000000
--- a/src/server/YoutubeFallback.hx
+++ /dev/null
@@ -1,142 +0,0 @@
-package server;
-
-import haxe.Json;
-import js.lib.Function;
-import js.lib.Object;
-import js.lib.Promise;
-import js.node.Https.Https;
-import js.node.Https.HttpsRequestOptions;
-import js.node.url.URLSearchParams;
-import utils.YoutubeUtils;
-
-class YoutubeFallback {
-	static function httpsGet(
-		url:String,
-		?options:HttpsRequestOptions,
-		?callback:(status:Int, data:String) -> Void
-	):Void {
-		final request = Https.get(url, options, res -> {
-			var data = "";
-			res.on("data", chunk -> data += chunk.toString());
-			res.on("end", () -> callback(res.statusCode, data));
-		});
-		request.on("error", err -> {
-			trace(url);
-			trace("request error: ", err);
-		});
-	}
-
-	public static function resolvePlayerResponse(watchHtml:String):String {
-		if (watchHtml == null) return "";
-		final resReg = ~/ytInitialPlayerResponse = (.*)}}};/;
-		var matches = resReg.match(watchHtml);
-		return matches ? resReg.matched(1) + "}}}" : "";
-	}
-
-	public static function resoleM3U8Link(watchHtml:String):Null<String> {
-		if (watchHtml == null) return null;
-		final hlsReg = ~/hlsManifestUrl":"(.*\/file\/index\.m3u8)/;
-		return hlsReg.match(watchHtml) ? hlsReg.matched(1) : null;
-	}
-
-	public static function buildDecoder(watchHtml:String, callback:(decoder:(cipher:String) -> String) -> Void):Void {
-		if (watchHtml == null) return callback(null);
-
-		final jsFileUrlReg = ~/\/s\/player\/[A-Za-z0-9]+\/[A-Za-z0-9_.]+\/[A-Za-z0-9_]+\/base\.js/;
-		if (!jsFileUrlReg.match(watchHtml)) return callback(null);
-
-		final url = "https://www.youtube.com" + jsFileUrlReg.matched(0);
-		httpsGet(url, {}, (status, jsFileContent) -> {
-			final funcReg = ~/function.*\.split\(""\).*\.join\(""\)}/;
-			if (!funcReg.match(jsFileContent)) return callback(null);
-
-			final decodeFunction = funcReg.matched(0);
-			final varNameReg = ~/\.split\(""\);([a-zA-Z0-9]+)\./i;
-			if (!varNameReg.match(decodeFunction)) return callback(null);
-
-			final varStartIndex = jsFileContent.indexOf("var " + varNameReg.matched(1) + "={");
-			if (varStartIndex < 0) return callback(null);
-
-			final varEndIndex = jsFileContent.indexOf("}};", varStartIndex);
-			if (varEndIndex < 0) return callback(null);
-
-			final varDeclares = jsFileContent.substring(varStartIndex, varEndIndex + 3);
-			if (varDeclares.length == 0) return callback(null);
-
-			callback(signatureCipher -> {
-				final params = new URLSearchParams(signatureCipher);
-				final obj = Object.fromEntries(params);
-				final signature = obj.s;
-				final signatureParam = obj.sp ?? "signature";
-				final url = obj.url;
-				final decodedSignature = new Function('
-					"use strict";
-					$varDeclares
-					return ($decodeFunction)("$signature");
-				').call(null);
-				return '$url&$signatureParam=${untyped encodeURIComponent(decodedSignature)}';
-			});
-		});
-	}
-
-	public static function getInfo(url:String, callback:(info:Null<YouTubeVideoInfo>) -> Void):Void {
-		final videoId = YoutubeUtils.extractVideoId(url);
-		if (videoId.length == 0) {
-			trace("youtube videoId is not found");
-			return callback(null);
-		}
-
-		final url = 'https://www.youtube.com/watch?v=$videoId';
-		httpsGet(url, {}, (status, data) -> {
-			if (status != 200 || data.length == 0) {
-				trace("Cannot get youtube video response");
-				return callback(null);
-			}
-
-			final ytInitialPlayerResponse = resolvePlayerResponse(data);
-			final parsedResponse = Json.parse(ytInitialPlayerResponse);
-			final streamingData:YouTubeVideoInfo = parsedResponse.streamingData ?? cast {};
-			streamingData.formats ??= [];
-			streamingData.adaptiveFormats ??= [];
-			var formats:Array<YoutubeVideoFormat> = streamingData.formats.concat(streamingData.adaptiveFormats);
-
-			final promises:Array<Promise<Any>> = [];
-
-			final isEncryptedVideo = formats.exists(it -> it.signatureCipher != null);
-			if (isEncryptedVideo) {
-				final promise = new Promise((resolve, reject) -> {
-					buildDecoder(data, decoder -> {
-						if (decoder != null) {
-							formats = formats.map(item -> {
-								if (item.url != null || item.signatureCipher == null) return item;
-
-								item.url = decoder(item.signatureCipher);
-								item.signatureCipher = null;
-								return item;
-							});
-						}
-						resolve(null);
-					});
-				});
-				promises.push(promise);
-			}
-
-			Promise.all(promises).then(_ -> {
-				final result:YouTubeVideoInfo = {
-					videoDetails: parsedResponse.videoDetails ?? cast {},
-					formats: formats.filter(format -> format.url != null)
-				};
-				if (result.videoDetails.isLiveContent) {
-					final m3u8Link = resoleM3U8Link(data);
-					try {
-						result.liveData = {
-							manifestUrl: m3u8Link,
-							// data: m3u8Parser.getResult()
-						};
-					}
-				}
-				callback(result);
-			});
-		});
-	}
-}