aboutsummaryrefslogtreecommitdiffstats
path: root/src/server
diff options
context:
space:
mode:
Diffstat (limited to 'src/server')
-rw-r--r--src/server/ConsoleInput.hx2
-rw-r--r--src/server/HttpServer.hx1
-rw-r--r--src/server/Main.hx16
-rw-r--r--src/server/cache/Cache.hx171
-rw-r--r--src/server/cache/RawCache.hx469
-rw-r--r--src/server/cache/YoutubeCache.hx (renamed from src/server/Cache.hx)221
6 files changed, 700 insertions, 180 deletions
diff --git a/src/server/ConsoleInput.hx b/src/server/ConsoleInput.hx
index cf88df6..e0712f5 100644
--- a/src/server/ConsoleInput.hx
+++ b/src/server/ConsoleInput.hx
@@ -107,7 +107,7 @@ class ConsoleInput {
case AddAdmin:
final name = args[0];
final password = args[1];
- if (main.badNickName(name)) {
+ if (main.isBadClientName(name)) {
final error = Lang.get("usernameError")
.replace("$MAX", '${main.config.maxLoginLength}');
trace(error);
diff --git a/src/server/HttpServer.hx b/src/server/HttpServer.hx
index 4734815..dd8c178 100644
--- a/src/server/HttpServer.hx
+++ b/src/server/HttpServer.hx
@@ -12,6 +12,7 @@ import js.node.http.ClientRequest;
import js.node.http.IncomingMessage;
import js.node.http.ServerResponse;
import js.node.url.URL;
+import server.cache.Cache;
import sys.FileSystem;
@:structInit
diff --git a/src/server/Main.hx b/src/server/Main.hx
index 86e619e..d935157 100644
--- a/src/server/Main.hx
+++ b/src/server/Main.hx
@@ -22,6 +22,7 @@ import js.npm.ws.Server as WSServer;
import js.npm.ws.WebSocket;
import json2object.ErrorUtils;
import json2object.JsonParser;
+import server.cache.Cache;
import sys.FileSystem;
import sys.io.File;
@@ -48,7 +49,7 @@ class Main {
var wss:WSServer;
final localIp:String;
var globalIp:String;
- final playersCacheSupport:Array<PlayerType> = [];
+ final playersCacheSupport:Array<PlayerType> = [RawType];
var port:Int;
final userList:UserList;
@@ -577,7 +578,7 @@ class Main {
case Login:
final name = data.login.clientName.trim();
final lcName = name.toLowerCase();
- if (badNickName(lcName)) {
+ if (isBadClientName(lcName)) {
serverMessage(client, "usernameError");
send(client, {type: LoginError});
return;
@@ -686,6 +687,11 @@ class Main {
addVideo();
} else {
switch item.playerType {
+ case RawType:
+ cache.cacheRawVideo(client, item.url, (name) -> {
+ item = item.withUrl(cache.getFileUrl(name));
+ addVideo();
+ });
case YoutubeType:
cache.cacheYoutubeVideo(client, item.url, (name) -> {
item = item.withUrl(cache.getFileUrl(name));
@@ -1065,7 +1071,7 @@ class Main {
final matchHtmlChars = ~/[&^<>'"]/;
final matchGuestName = ~/guest [0-9]+/;
- public function badNickName(name:String):Bool {
+ public function isBadClientName(name:String):Bool {
if (name.length > config.maxLoginLength) return true;
if (name.length == 0) return true;
if (matchHtmlChars.match(name)) return true;
@@ -1147,4 +1153,8 @@ class Main {
}
return false;
}
+
+ public function hasPlaylistUrl(url:String):Bool {
+ return videoList.exists(item -> item.url == url);
+ }
}
diff --git a/src/server/cache/Cache.hx b/src/server/cache/Cache.hx
new file mode 100644
index 0000000..f71b465
--- /dev/null
+++ b/src/server/cache/Cache.hx
@@ -0,0 +1,171 @@
+package server.cache;
+
+import haxe.io.Path;
+import js.node.Fs.Fs;
+import sys.FileSystem;
+
+class Cache {
+ public final notEnoughSpaceErrorText = "Error: Not enough free space on server or file size is out of cache storage limit.";
+
+ public final isYtReady = false;
+
+ /** In bytes **/
+ public var storageLimit(default, null) = 3 * 1024 * 1024 * 1024;
+
+ final main:Main;
+
+ public final cacheDir:String;
+ public final freeSpaceBlock = 10 * 1024 * 1024; // 10MB
+
+ final cachedFiles:Array<String> = [];
+ var youtubeCache:YoutubeCache;
+ var rawCache:RawCache;
+
+ public function new(main:Main, cacheDir:String) {
+ this.main = main;
+ this.cacheDir = cacheDir;
+ Utils.ensureDir(cacheDir);
+ youtubeCache = new YoutubeCache(main, this);
+ rawCache = new RawCache(main, this);
+ isYtReady = youtubeCache.checkYtDeps();
+ if (isYtReady) youtubeCache.cleanYtInputFiles();
+ }
+
+ public function getCachedFiles():Array<String> {
+ return cachedFiles;
+ }
+
+ public function setCachedFiles(names:Array<String>) {
+ cachedFiles.resize(0);
+ for (name in names) cachedFiles.push(name);
+
+ final names = FileSystem.readDirectory(cacheDir);
+ for (name in names) {
+ if (name.startsWith(".")) continue;
+ if (FileSystem.isDirectory('$cacheDir/$name')) continue;
+ if (cachedFiles.contains(name)) continue;
+ trace('Remove non-tracked cache $name');
+ remove(name);
+ }
+ }
+
+ public function log(client:Client, msg:String):Void {
+ main.serverMessage(client, msg);
+ trace(msg);
+ }
+
+ public function cacheYoutubeVideo(client:Client, url:String, callback:(name:String) -> Void) {
+ youtubeCache.cacheYoutubeVideo(client, url, callback);
+ }
+
+ public function cacheRawVideo(client:Client, url:String, callback:(name:String) -> Void) {
+ rawCache.cacheRawVideo(client, url, callback);
+ }
+
+ public function setStorageLimit(bytes:Int) {
+ storageLimit = cast bytes;
+ storageLimit = storageLimit.limitMin(0);
+ getFreeDiskSpace(availSpace -> {
+ final availSpace = (availSpace - freeSpaceBlock).limitMin(0);
+ removeOlderCache();
+ final freeSpace = getFreeSpace();
+ if (availSpace < freeSpace) {
+ // shrink limit lower than disk space
+ storageLimit += availSpace - freeSpace;
+ storageLimit = storageLimit.limitMin(0);
+ removeOlderCache();
+ }
+ });
+ }
+
+ public function getFreeDiskSpace(callback:(availSpace:Int) -> Void):Void {
+ final statfs = (Fs : Dynamic).statfs ?? {
+ trace("Warning: no fs.statfs support in current nodejs version (needs v18+)");
+ callback(storageLimit);
+ return;
+ }
+ statfs("/", (err, stats) -> {
+ if (err != null) {
+ trace(err);
+ callback(storageLimit);
+ return;
+ }
+ callback(stats.bsize * stats.bavail);
+ });
+ }
+
+ public function add(name:String) {
+ if (!cachedFiles.contains(name)) {
+ cachedFiles.unshift(name);
+ }
+ }
+
+ public function remove(name:String):Void {
+ cachedFiles.remove(name);
+ removeFile(name);
+ }
+
+ public function exists(name:String):Bool {
+ return cachedFiles.contains(name) && isFileExists(name);
+ }
+
+ /** Returns `true` if there is enough space to save `addFileSize` bytes. **/
+ public function removeOlderCache(addFileSize = 0):Bool {
+ var space = getUsedSpace(addFileSize);
+ for (name in cachedFiles.reversed()) {
+ if (space <= storageLimit) break;
+ // do not remove cached items that are in playlist
+ if (main.hasPlaylistUrl(getFileUrl(name))) continue;
+ remove(name);
+ space = getUsedSpace(addFileSize);
+ }
+ return space < storageLimit;
+ }
+
+ function removeFile(name:String):Void {
+ final path = getFilePath(name);
+ if (FileSystem.exists(path)) FileSystem.deleteFile(path);
+ }
+
+ public function getFreeFileName(fullName = "video.mp4"):String {
+ final baseName = Path.withoutDirectory(Path.withoutExtension(fullName));
+ final ext = Path.extension(fullName);
+ var i = 1;
+ while (true) {
+ final n = i == 1 ? "" : '$i';
+ final name = '$baseName$n.$ext';
+ if (!isFileExists(name)) return name;
+ i++;
+ }
+ }
+
+ public function getFilePath(name:String):String {
+ return '$cacheDir/$name';
+ }
+
+ public function getFileUrl(name:String):String {
+ final folder = Path.withoutDirectory(cacheDir);
+ return '/$folder/$name';
+ }
+
+ public function isFileExists(name:String):Bool {
+ return FileSystem.exists(getFilePath(name));
+ }
+
+ public function getFreeSpace():Int {
+ return storageLimit - getUsedSpace();
+ }
+
+ public function getUsedSpace(addFileSize = 0):Int {
+ var total = addFileSize.limitMin(0);
+ for (name in cachedFiles.reversed()) {
+ final path = getFilePath(name);
+ if (!FileSystem.exists(path)) {
+ cachedFiles.remove(name);
+ continue;
+ }
+ total += FileSystem.stat(path).size;
+ }
+ return total;
+ }
+}
diff --git a/src/server/cache/RawCache.hx b/src/server/cache/RawCache.hx
new file mode 100644
index 0000000..ed8679c
--- /dev/null
+++ b/src/server/cache/RawCache.hx
@@ -0,0 +1,469 @@
+package server.cache;
+
+import js.node.Buffer;
+import js.node.ChildProcess;
+import js.node.Fs.Fs;
+import js.node.Http;
+import js.node.Https;
+import js.node.http.ClientRequest;
+import js.node.http.IncomingMessage;
+import sys.FileSystem;
+import sys.io.File;
+
+typedef Segment = {
+ i:Int,
+ url:String,
+ started:Bool,
+ completed:Bool,
+ name:String
+}
+
+class RawCache {
+ final main:Main;
+ final cache:Cache;
+
+ public function new(main:Main, cache:Cache):Void {
+ this.main = main;
+ this.cache = cache;
+ }
+
+ public function cacheRawVideo(client:Client, url:String, callback:(name:String) -> Void) {
+ final isM3U8 = url.contains(".m3u8");
+ final ext = isM3U8 ? "m3u8" : "mp4";
+
+ final matchName = ~/^([^:.]+)\.(.+)/;
+ final decodedUrl = try url.urlDecode() catch (e) url;
+ final lastPart = decodedUrl.substr(decodedUrl.lastIndexOf("/") + 1);
+ var outName = matchName.match(lastPart) ? matchName.matched(1)
+ + '.$ext' : 'video.$ext';
+ outName = cache.getFreeFileName(outName);
+
+ if (cache.exists(outName)) {
+ callback(outName);
+ return;
+ }
+
+ trace('Caching $url to $outName...');
+ main.send(client, {
+ type: Progress,
+ progress: {
+ type: Caching,
+ ratio: 0,
+ data: outName
+ }
+ });
+
+ if (isM3U8) {
+ handleM3u8(client, url, outName, callback);
+ } else {
+ handleMp4(client, url, outName, callback);
+ }
+ }
+
+ function handleMp4(client:Client, url:String, outName:String, callback:(name:String) -> Void) {
+ downloadFile(client, url, outName, (downloaded, total) -> {
+ main.send(client, {
+ type: Progress,
+ progress: {
+ type: Downloading,
+ ratio: (downloaded / total).clamp(0, 1)
+ }
+ });
+ }, () -> {
+ cache.add(outName);
+ callback(outName);
+ }, (err) -> {
+ log(client, 'Mp4 download failed: $err');
+ cancelProgress(client);
+ });
+ }
+
+ function handleM3u8(client:Client, url:String, outName:String, callback:(name:String) -> Void):Void {
+ final useProxy = true;
+ downloadM3u8Playlist(client, url, useProxy, (playlist, totalSize, segments) -> {
+ // only playlist file donwloaded
+ if (useProxy) totalSize = playlist.length;
+
+ if (!cache.removeOlderCache(totalSize + cache.freeSpaceBlock)) {
+ log(client, cache.notEnoughSpaceErrorText);
+ cancelProgress(client);
+ return;
+ }
+
+ if (useProxy) {
+ main.send(client, {
+ type: Progress,
+ progress: {
+ type: Caching,
+ ratio: 1,
+ data: outName
+ }
+ });
+ File.saveContent('${cache.cacheDir}/$outName', playlist);
+ cache.add(outName);
+ callback(outName);
+ return;
+ }
+
+ var activeDownloads = 0;
+ final maxParallelDownloads = 10;
+ var downloaded = 0;
+
+ function downloadNextBatch():Void {
+ for (segment in segments) {
+ if (activeDownloads >= maxParallelDownloads) break;
+ if (segment.started) continue;
+ segment.started = true;
+ activeDownloads++;
+ trace("download segment", segment.i);
+
+ downloadFile(client, segment.url, segment.name,
+ (downloadedBytes, totalBytes) -> {},
+
+ () -> {
+ activeDownloads--;
+ segment.completed = true;
+ downloaded++;
+
+ final progress = downloaded / segments.length;
+ main.send(client, {
+ type: Progress,
+ progress: {
+ type: Downloading,
+ ratio: progress.clamp(0, 1)
+ }
+ });
+
+ if (downloaded == segments.length) {
+ trace('All ${downloaded}/${segments.length} segments downloaded');
+
+ File.saveContent('${cache.cacheDir}/$outName', playlist);
+ cache.add(outName);
+ callback(outName);
+ // buildTsFiles(
+ // segments.map(item -> item.name),
+ // outName,
+ // client,
+ // callback
+ // );
+ } else {
+ downloadNextBatch();
+ }
+ },
+
+ (err) -> {
+ activeDownloads--;
+ downloaded++;
+ log(client, 'TS segment ${segment.i} download failed: $err');
+ cancelProgress(client);
+ cleanupFiles(segments.map(item -> item.name));
+ }
+ );
+ }
+ }
+
+ // Start the initial batch of downloads
+ downloadNextBatch();
+ }, (err) -> {
+ log(client, 'M3U8 processing failed: $err');
+ cancelProgress(client);
+ });
+ }
+
+ function request(url:String, ?options:Null<HttpsRequestOptions>, ?callback:Null<IncomingMessage->
+ Void>):ClientRequest {
+ final httpsOptions:HttpsRequestOptions = options ?? {};
+ // Allow self-signed certificates
+ httpsOptions.rejectUnauthorized = false;
+ httpsOptions.headers ??= {};
+
+ if (url.startsWith("https:")) {
+ return Https.request(url, httpsOptions, callback);
+ } else {
+ return Http.request(url, httpsOptions, callback);
+ }
+ }
+
+ function downloadM3u8Playlist(
+ client:Client,
+ url:String,
+ useProxy:Bool,
+ onSuccess:(playlist:String, totalSize:Int, segments:Array<Segment>) -> Void,
+ onError:(err:String) -> Void
+ ) {
+ final options:HttpsRequestOptions = {
+ headers: {
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+ "Accept": "*/*",
+ }
+ };
+
+ final req = request(url, options, (res:IncomingMessage) -> {
+ if (res.statusCode >= 300 && res.statusCode < 400) {
+ final redirectUrl = res.headers.get("location");
+ if (redirectUrl != null) {
+ downloadM3u8Playlist(client, redirectUrl, useProxy, onSuccess, onError);
+ return;
+ }
+ }
+
+ final body:Array<Any> = [];
+ res.on("data", chunk -> body.push(chunk));
+ res.on("end", () -> {
+ try {
+ final buffer = Buffer.concat(body);
+ final content = buffer.toString();
+ if (!~/^#EXTM3U/.match(content)) {
+ onError("Invalid M3U8 playlist");
+ return;
+ }
+
+ final baseUrl = url.substring(0, url.lastIndexOf("/") + 1);
+ final segments:Array<Segment> = [];
+
+ final lines = content.split("\n");
+ for (lineI => line in lines) {
+ final line = line.trim();
+ if (line.length == 0) continue;
+ if (line.startsWith("#")) continue;
+ final segmentUrl = !line.contains("://") ? baseUrl + line : line;
+ final i = segments.length;
+ final segment:Segment = {
+ i: i,
+ url: segmentUrl,
+ started: false,
+ completed: false,
+ name: 'segment$i.ts',
+ }
+ segments.push(segment);
+ lines[lineI] = './${segment.name}';
+ if (useProxy) {
+ lines[lineI] = '/proxy?url=$segmentUrl';
+ }
+ }
+
+ // Head request can return full stream size, so lets do loose assumption
+ final req = request(segments[0].url, {method: Get}, (res:IncomingMessage) -> {
+ final contentLength = Std.parseInt(res.headers["content-length"]) ?? 0;
+ final totalSize = contentLength * (segments.length + 1);
+ if (totalSize == 0) {
+ onError("Failed to get segment sizes: no content-length");
+ return;
+ }
+ onSuccess(lines.join("\n"), totalSize, segments);
+ });
+ req.on("error", (err) -> {
+ onError("Request error: failed to get segment sizes");
+ });
+ req.end();
+ } catch (e) {
+ onError('Playlist processing error: $e');
+ }
+ });
+ });
+
+ req.on("error", onError);
+ req.end();
+ }
+
+ function downloadFile(
+ client:Client, url:String, fileName:String,
+ onProgress:(downloaded:Int, total:Int) -> Void,
+ onComplete:() -> Void,
+ onError:(err:String) -> Void
+ ):Void {
+ final outPath = '${cache.cacheDir}/$fileName';
+ final file = Fs.createWriteStream(outPath);
+ final options:HttpsRequestOptions = {
+ method: Get,
+ headers: {
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537." +
+ Std.random(100),
+ "Accept": "*/*",
+ }
+ };
+ final req = request(url, options, (res:IncomingMessage) -> {
+ final total = Std.parseInt(res.headers["content-length"]) ?? 0;
+ var downloaded = 0;
+
+ // Handle response data chunks
+ res.on("data", (chunk) -> {
+ downloaded += chunk.length;
+ onProgress(downloaded, total);
+
+ // Handle backpressure
+ if (!file.write(chunk)) {
+ res.pause();
+ file.once("drain", () -> res.resume());
+ }
+ });
+
+ // Handle response completion
+ res.on("end", () -> file.end());
+
+ // Handle response errors
+ res.on("error", (err) -> {
+ file.destroy();
+ onError('Response error: $err');
+ });
+ });
+
+ // Handle file write completion
+ file.on("finish", onComplete);
+
+ // Handle file system errors
+ file.on("error", (err) -> {
+ req.destroy();
+ onError('File error: $err');
+ });
+
+ // Handle request errors
+ req.on("error", (err) -> {
+ file.destroy();
+ onError('Request failed: $err');
+ });
+
+ req.end();
+ }
+
+ function buildTsFiles(tempFiles:Array<String>, outName:String, client:Client, callback:String->Void) {
+ final missingFiles = tempFiles.filter(f ->
+ !FileSystem.exists('${cache.cacheDir}/$f'));
+ if (missingFiles.length > 0) {
+ log(client, 'Concatenation failed: ${missingFiles.length} segments are missing');
+ main.send(client, {
+ type: Progress,
+ progress: {
+ type: Canceled,
+ ratio: 1
+ }
+ });
+ cleanupFiles(tempFiles);
+ return;
+ }
+
+ // Create concat file with absolute paths and proper escaping
+ final concatFile = 'concat_list.txt';
+ final concatContent = tempFiles.map(f -> {
+ final path = './$f';
+ return 'file \'${path}\'';
+ }).join("\n");
+
+ File.saveContent('${cache.cacheDir}/$concatFile', concatContent);
+
+ // Prepare FFmpeg args with proper bitstream filters for TS files
+ final args = [
+ "-y", // Overwrite output without asking
+ "-f", "concat", // Use concat format
+ "-safe", "0", // Allow absolute paths
+ "-i", concatFile, // Input file list
+ "-c", "copy", // Copy streams without re-encoding
+ "-bsf:a", "aac_adtstoasc", // Fix AAC audio streams from TS files
+ "-movflags", "+faststart", // Optimize for web streaming
+ outName // Output filename
+ ];
+
+ trace('Executing FFmpeg with args: ${args.join(" ")}');
+
+ // Create process with proper error capturing
+ final process = ChildProcess.spawn("ffmpeg", args, {
+ cwd: cache.cacheDir,
+ // stderr: "pipe" // Capture stderr for error reporting
+ });
+
+ final errorOutput:Array<Buffer> = [];
+ process.stderr.on("data", (data) -> errorOutput.push(data));
+
+ // Set a reasonable timeout
+ final timeout = 5 * 60 * 1000; // 5 minutes
+ final timeoutId = js.Node.setTimeout(() -> {
+ process.kill();
+ log(client, 'FFmpeg process timed out after ${timeout / 1000} seconds');
+ main.send(client, {
+ type: Progress,
+ progress: {
+ type: Canceled,
+ ratio: 1
+ }
+ });
+ cleanupFiles(tempFiles.concat([concatFile]));
+ }, timeout);
+
+ process.on("close", (code:Int) -> {
+ js.Node.clearTimeout(timeoutId);
+
+ if (code != 0) {
+ final errorMsg = Buffer.concat(errorOutput).toString();
+ log(client, 'FFmpeg concatenation failed with code $code');
+ trace('FFmpeg error output: $errorMsg');
+
+ // Log detailed error to admins
+ final admins = main.clients.filter(client -> client.isAdmin);
+ for (admin in admins) {
+ log(admin, 'FFmpeg error: $errorMsg');
+ }
+
+ main.send(client, {
+ type: Progress,
+ progress: {
+ type: Canceled,
+ ratio: 1
+ }
+ });
+ } else {
+ // Verify the output file exists and has content
+ if (FileSystem.exists('${cache.cacheDir}/$outName')
+ && FileSystem.stat('${cache.cacheDir}/$outName').size > 0) {
+ cache.add(outName);
+ callback(outName);
+ } else {
+ log(client, 'FFmpeg process completed but output file is missing or empty');
+ main.send(client, {
+ type: Progress,
+ progress: {
+ type: Canceled,
+ ratio: 1
+ }
+ });
+ }
+ }
+
+ // Clean up temporary files after everything is done
+ cleanupFiles(tempFiles.concat([concatFile]));
+ });
+
+ // Handle process errors (like if FFmpeg isn't found)
+ process.on("error", (err) -> {
+ js.Node.clearTimeout(timeoutId);
+ log(client, 'Failed to start FFmpeg: $err');
+ main.send(client, {
+ type: Progress,
+ progress: {
+ type: Canceled,
+ ratio: 1
+ }
+ });
+ cleanupFiles(tempFiles.concat([concatFile]));
+ });
+ }
+
+ function cleanupFiles(files:Array<String>):Void {
+ for (file in files) {
+ if (FileSystem.exists(file)) FileSystem.deleteFile(file);
+ }
+ }
+
+ function log(client:Client, msg:String):Void {
+ cache.log(client, msg);
+ }
+
+ function cancelProgress(client:Client):Void {
+ main.send(client, {
+ type: Progress,
+ progress: {
+ type: Canceled,
+ ratio: 0
+ }
+ });
+ }
+}
diff --git a/src/server/Cache.hx b/src/server/cache/YoutubeCache.hx
index ef74517..c0a5c4c 100644
--- a/src/server/Cache.hx
+++ b/src/server/cache/YoutubeCache.hx
@@ -1,7 +1,6 @@
-package server;
+package server.cache;
import haxe.Json;
-import haxe.io.Path;
import js.lib.Promise;
import js.node.Buffer;
import js.node.ChildProcess;
@@ -11,28 +10,16 @@ import sys.FileSystem;
import sys.io.File;
import utils.YoutubeUtils;
-class Cache {
- public final notEnoughSpaceErrorText = "Error: Not enough free space on server or file size is out of cache storage limit.";
-
- public final isYtReady = false;
-
- /** In bytes **/
- public var storageLimit(default, null) = 3 * 1024 * 1024 * 1024;
-
+class YoutubeCache {
final main:Main;
- final cacheDir:String;
- final cachedFiles:Array<String> = [];
- final freeSpaceBlock = 10 * 1024 * 1024; // 10MB
+ final cache:Cache;
- public function new(main:Main, cacheDir:String) {
+ public function new(main:Main, cache:Cache):Void {
this.main = main;
- this.cacheDir = cacheDir;
- Utils.ensureDir(cacheDir);
- isYtReady = checkYtDeps();
- if (isYtReady) cleanYtInputFiles();
+ this.cache = cache;
}
- function checkYtDeps():Bool {
+ public function checkYtDeps():Bool {
final ytdl = try {
untyped require("@distube/ytdl-core");
} catch (e) {
@@ -46,39 +33,16 @@ class Cache {
}
}
- function cleanYtInputFiles():Void {
- final names = FileSystem.readDirectory(cacheDir);
+ public function cleanYtInputFiles():Void {
+ final names = FileSystem.readDirectory(cache.cacheDir);
for (name in names) {
if (!name.startsWith("__tmp")) continue;
- remove(name);
- }
- }
-
- public function getCachedFiles():Array<String> {
- return cachedFiles;
- }
-
- public function setCachedFiles(names:Array<String>) {
- cachedFiles.resize(0);
- for (name in names) cachedFiles.push(name);
-
- final names = FileSystem.readDirectory(cacheDir);
- for (name in names) {
- if (name.startsWith(".")) continue;
- if (FileSystem.isDirectory('$cacheDir/$name')) continue;
- if (cachedFiles.contains(name)) continue;
- trace('Remove non-tracked cache $name');
- remove(name);
+ cache.remove(name);
}
}
- function log(client:Client, msg:String):Void {
- main.serverMessage(client, msg);
- trace(msg);
- }
-
public function cacheYoutubeVideo(client:Client, url:String, callback:(name:String) -> Void) {
- if (!isYtReady) {
+ if (!cache.isYtReady) {
trace("Do `npm i @distube/ytdl-core@latest` to use cache feature (you also need to install `ffmpeg` to build mp4 from downloaded audio/video tracks).");
return;
}
@@ -88,36 +52,27 @@ class Cache {
return;
}
final outName = videoId + ".mp4";
- if (cachedFiles.contains(outName) && isFileExists(outName)) {
+ if (cache.exists(outName)) {
callback(outName);
return;
}
final inVideoName = '__tmp-video-$videoId';
final inAudioName = '__tmp-audio-$videoId';
inline function removeInputFiles():Void {
- remove(inVideoName);
- remove(inAudioName);
- }
- inline function cancelProgress():Void {
- main.send(client, {
- type: Progress,
- progress: {
- type: Canceled,
- ratio: 1
- }
- });
+ cache.remove(inVideoName);
+ cache.remove(inAudioName);
}
inline function checkEnoughSpace(contentLength:Int):Bool {
- final hasSpace = removeOlderCache(contentLength + freeSpaceBlock);
+ final hasSpace = cache.removeOlderCache(contentLength + cache.freeSpaceBlock);
if (!hasSpace) {
removeInputFiles();
- cancelProgress();
- log(client, notEnoughSpaceErrorText);
+ cancelProgress(client);
+ log(client, cache.notEnoughSpaceErrorText);
}
return hasSpace;
}
- if (isFileExists(inVideoName)) {
+ if (cache.isFileExists(inVideoName)) {
log(client, 'Caching $outName already in progress');
return;
}
@@ -162,7 +117,8 @@ class Cache {
return videoSize + audioSize;
}
// check if we have space for formats and video build
- final hasSpace = removeOlderCache(getTotalFormatsSize() * 2 + freeSpaceBlock);
+ final hasSpace = cache.removeOlderCache(getTotalFormatsSize() * 2
+ + cache.freeSpaceBlock);
if (!hasSpace) {
// try fallback to worse video quality
videoFormat = getBestYoutubeVideoFormat(info.formats, videoFormat.qualityLabel);
@@ -173,22 +129,22 @@ class Cache {
format: videoFormat,
agent: agent,
});
- dlVideo.pipe(Fs.createWriteStream('$cacheDir/$inVideoName'));
+ dlVideo.pipe(Fs.createWriteStream('${cache.cacheDir}/$inVideoName'));
dlVideo.on("error", err -> {
log(client, "Error during video download: " + err);
removeInputFiles();
- cancelProgress();
+ cancelProgress(client);
});
final dlAudio:Readable<Dynamic> = ytdl(url, {
format: audioFormat,
agent: agent,
});
- dlAudio.pipe(Fs.createWriteStream('$cacheDir/$inAudioName'));
+ dlAudio.pipe(Fs.createWriteStream('${cache.cacheDir}/$inAudioName'));
dlAudio.on("error", err -> {
log(client, "Error during audio download: " + err);
removeInputFiles();
- cancelProgress();
+ cancelProgress(client);
});
var count = 0;
@@ -196,20 +152,20 @@ class Cache {
count++;
trace('$type track downloaded ($count/2)');
if (count < 2) return;
- if (!isFileExists(inVideoName) || !isFileExists(inAudioName)) {
+ if (!cache.isFileExists(inVideoName) || !cache.isFileExists(inAudioName)) {
log(client, "Input files not found for making final video");
removeInputFiles();
- cancelProgress();
+ cancelProgress(client);
return;
}
- var size = FileSystem.stat('$cacheDir/$inVideoName').size;
- size += FileSystem.stat('$cacheDir/$inAudioName').size;
+ var size = FileSystem.stat('${cache.cacheDir}/$inVideoName').size;
+ size += FileSystem.stat('${cache.cacheDir}/$inAudioName').size;
// clean some space for full mp4
if (!checkEnoughSpace(size)) return;
final args = '-y -i ./$inVideoName -i ./$inAudioName -c copy -map 0:v -map 1:a ./$outName'.split(" ");
final process = ChildProcess.spawn("ffmpeg", args, {
- cwd: cacheDir,
+ cwd: cache.cacheDir,
// stdio: "ignore"
});
final outputData:Array<Buffer> = [];
@@ -217,7 +173,7 @@ class Cache {
process.on("close", (code:Int) -> {
removeInputFiles();
if (code != 0) {
- cancelProgress();
+ cancelProgress(client);
final errCodeMsg = 'Error: ffmpeg closed with code $code';
final admins = main.clients.filter(client -> client.isAdmin);
for (client in admins) {
@@ -227,7 +183,7 @@ class Cache {
if (!admins.contains(client)) log(client, errCodeMsg);
return;
}
- add(outName);
+ cache.add(outName);
callback(outName);
});
@@ -246,112 +202,11 @@ class Cache {
});
}).catchError(err -> {
removeInputFiles();
- cancelProgress();
+ cancelProgress(client);
log(client, "" + err);
});
}
- public function setStorageLimit(bytes:Int) {
- storageLimit = cast bytes;
- storageLimit = storageLimit.limitMin(0);
- getFreeDiskSpace(availSpace -> {
- final availSpace = (availSpace - freeSpaceBlock).limitMin(0);
- removeOlderCache();
- final freeSpace = getFreeSpace();
- if (availSpace < freeSpace) {
- // shrink limit lower than disk space
- storageLimit += availSpace - freeSpace;
- storageLimit = storageLimit.limitMin(0);
- removeOlderCache();
- }
- });
- }
-
- public function getFreeDiskSpace(callback:(availSpace:Int) -> Void):Void {
- final statfs = (Fs : Dynamic).statfs ?? {
- trace("Warning: no fs.statfs support in current nodejs version (needs v18+)");
- callback(storageLimit);
- return;
- }
- statfs("/", (err, stats) -> {
- if (err != null) {
- trace(err);
- callback(storageLimit);
- return;
- }
- callback(stats.bsize * stats.bavail);
- });
- }
-
- public function add(name:String) {
- if (!cachedFiles.contains(name)) {
- cachedFiles.unshift(name);
- }
- }
-
- public function remove(name:String):Void {
- cachedFiles.remove(name);
- removeFile(name);
- }
-
- /** Returns `true` if there is enough space to save `addFileSize` bytes. **/
- public function removeOlderCache(addFileSize = 0):Bool {
- var space = getUsedSpace(addFileSize);
- while (space > storageLimit) {
- final name = cachedFiles.pop() ?? break;
- removeFile(name);
- space = getUsedSpace(addFileSize);
- }
- return space < storageLimit;
- }
-
- function removeFile(name:String):Void {
- final path = getFilePath(name);
- if (FileSystem.exists(path)) FileSystem.deleteFile(path);
- }
-
- public function getFreeFileName(fullName = "video.mp4"):String {
- final baseName = Path.withoutDirectory(Path.withoutExtension(fullName));
- final ext = Path.extension(fullName);
- var i = 1;
- while (true) {
- final n = i == 1 ? "" : '$i';
- final name = '$baseName$n.$ext';
- if (!isFileExists(name)) return name;
- i++;
- }
- }
-
- public function getFilePath(name:String):String {
- return '$cacheDir/$name';
- }
-
- public function getFileUrl(name:String):String {
- final folder = Path.withoutDirectory(cacheDir);
- return '/$folder/$name';
- }
-
- public function isFileExists(name:String):Bool {
- return FileSystem.exists(getFilePath(name));
- }
-
- public function getFreeSpace():Int {
- return storageLimit - getUsedSpace();
- }
-
- public function getUsedSpace(addFileSize = 0):Int {
- var total = addFileSize.limitMin(0);
- for (name in cachedFiles.reversed()) {
- final path = getFilePath(name);
- if (!FileSystem.exists(path)) {
- cachedFiles.remove(name);
- continue;
- }
- total += FileSystem.stat(path).size;
- }
- return total;
- }
-
function getBestYoutubeVideoFormat(formats:Array<YoutubeVideoFormat>, ?ignoreQuality:String):Null<YoutubeVideoFormat> {
final qPriority = [1080, 720, 480, 360, 240, 144];
for (q in qPriority) {
@@ -364,4 +219,18 @@ class Cache {
}
return null;
}
+
+ function log(client:Client, msg:String):Void {
+ cache.log(client, msg);
+ }
+
+ function cancelProgress(client:Client):Void {
+ main.send(client, {
+ type: Progress,
+ progress: {
+ type: Canceled,
+ ratio: 0
+ }
+ });
+ }
}
send patches to the email below
yukais@pinapelz.com
include the subject [PATCH repo_name]
pinapelz.com
homepage