From 82a6c65d46e2583883b1b01d706145386308d19e Mon Sep 17 00:00:00 2001 From: RblSb Date: Sun, 9 Feb 2025 04:25:18 +0300 Subject: Improve cache limit handling --- build/server.js | 166 ++++++++++++++++++++++++++++++++++++----------- res/client.js | 7 +- res/css/des.css | 2 + src/Types.hx | 1 + src/client/Main.hx | 6 +- src/server/Cache.hx | 108 ++++++++++++++++++++++++------ src/server/HttpServer.hx | 5 +- src/server/Logger.hx | 4 +- src/server/Main.hx | 6 +- 9 files changed, 235 insertions(+), 70 deletions(-) diff --git a/build/server.js b/build/server.js index d2062b0..8a95604 100644 --- a/build/server.js +++ b/build/server.js @@ -1617,7 +1617,7 @@ JsonParser_$57.prototype = $extend(json2object_reader_BaseParser.prototype,{ this.value = null; } ,loadJsonString: function(s,pos,variable) { - this.value = this.loadString(s,pos,variable,["Caching","Downloading","Uploading"],"Caching"); + this.value = this.loadString(s,pos,variable,["Caching","Downloading","Uploading","Canceled"],"Caching"); } ,__class__: JsonParser_$57 }); @@ -3681,9 +3681,10 @@ json2object_PositionUtils.prototype = { }; var server_Cache = function(main,cacheDir) { this.freeSpaceBlock = 10485760; + this.cachedFiles = []; this.storageLimit = 3145728 * 1024; this.isYtReady = false; - this.cachedFiles = []; + this.notEnoughSpaceErrorText = "Error: Not enough free space on server or file size is out of cache storage limit."; this.main = main; this.cacheDir = cacheDir; server_Utils.ensureDir(cacheDir); @@ -3720,14 +3721,39 @@ server_Cache.prototype = { this.remove(name); } } + ,getCachedFiles: function() { + return this.cachedFiles; + } + ,setCachedFiles: function(names) { + this.cachedFiles.length = 0; + var _g = 0; + while(_g < names.length) this.cachedFiles.push(names[_g++]); + var names = js_node_Fs.readdirSync(this.cacheDir); + var _g = 0; + while(_g < names.length) { + var name = names[_g]; + ++_g; + if(StringTools.startsWith(name,".")) { + continue; + } + if(sys_FileSystem.isDirectory("" + this.cacheDir + "/" + name)) { + continue; + } + if(this.cachedFiles.indexOf(name) != -1) { + continue; + } + haxe_Log.trace("Remove non-tracked cache " + name,{ fileName : "src/server/Cache.hx", lineNumber : 68, className : "server.Cache", methodName : "setCachedFiles"}); + this.remove(name); + } + } ,log: function(client,msg) { this.main.serverMessage(client,msg); - haxe_Log.trace(msg,{ fileName : "src/server/Cache.hx", lineNumber : 56, className : "server.Cache", methodName : "log"}); + haxe_Log.trace(msg,{ fileName : "src/server/Cache.hx", lineNumber : 75, className : "server.Cache", methodName : "log"}); } ,cacheYoutubeVideo: function(client,url,callback) { var _gthis = this; if(!this.isYtReady) { - haxe_Log.trace("Do `npm i @distube/ytdl-core@latest` to use cache feature (you also need to install `ffmpeg` to build mp4 from downloaded audio/video tracks).",{ fileName : "src/server/Cache.hx", lineNumber : 61, className : "server.Cache", methodName : "cacheYoutubeVideo"}); + haxe_Log.trace("Do `npm i @distube/ytdl-core@latest` to use cache feature (you also need to install `ffmpeg` to build mp4 from downloaded audio/video tracks).",{ fileName : "src/server/Cache.hx", lineNumber : 80, className : "server.Cache", methodName : "cacheYoutubeVideo"}); return; } var videoId = utils_YoutubeUtils.extractVideoId(url); @@ -3747,11 +3773,11 @@ server_Cache.prototype = { return; } var ytdl = require("@distube/ytdl-core"); - haxe_Log.trace("Caching " + url + " to " + outName + "...",{ fileName : "src/server/Cache.hx", lineNumber : 85, className : "server.Cache", methodName : "cacheYoutubeVideo"}); + haxe_Log.trace("Caching " + url + " to " + outName + "...",{ fileName : "src/server/Cache.hx", lineNumber : 113, className : "server.Cache", methodName : "cacheYoutubeVideo"}); this.main.send(client,{ type : "Progress", progress : { type : "Caching", ratio : 0, data : outName}}); var promise = ytdl.getInfo(url); promise.then(function(info) { - haxe_Log.trace("Get info with " + info.formats.length + " formats",{ fileName : "src/server/Cache.hx", lineNumber : 96, className : "server.Cache", methodName : "cacheYoutubeVideo"}); + haxe_Log.trace("Get info with " + info.formats.length + " formats",{ fileName : "src/server/Cache.hx", lineNumber : 124, className : "server.Cache", methodName : "cacheYoutubeVideo"}); var audioFormat; try { var ytdl1 = ytdl.chooseFormat; @@ -3770,7 +3796,7 @@ server_Cache.prototype = { } catch( _g ) { var e = haxe_Exception.caught(_g); _gthis.log(client,"Error: audio format not found"); - haxe_Log.trace(e,{ fileName : "src/server/Cache.hx", lineNumber : 103, className : "server.Cache", methodName : "cacheYoutubeVideo"}); + haxe_Log.trace(e,{ fileName : "src/server/Cache.hx", lineNumber : 131, className : "server.Cache", methodName : "cacheYoutubeVideo"}); var _g1 = []; var _g2 = 0; var _g3 = info.formats; @@ -3781,7 +3807,7 @@ server_Cache.prototype = { _g1.push(v); } } - haxe_Log.trace(_g1,{ fileName : "src/server/Cache.hx", lineNumber : 104, className : "server.Cache", methodName : "cacheYoutubeVideo"}); + haxe_Log.trace(_g1,{ fileName : "src/server/Cache.hx", lineNumber : 132, className : "server.Cache", methodName : "cacheYoutubeVideo"}); return; } var videoFormat; @@ -3800,7 +3826,7 @@ server_Cache.prototype = { _g.push(v); } } - haxe_Log.trace(_g,{ fileName : "src/server/Cache.hx", lineNumber : 109, className : "server.Cache", methodName : "cacheYoutubeVideo"}); + haxe_Log.trace(_g,{ fileName : "src/server/Cache.hx", lineNumber : 137, className : "server.Cache", methodName : "cacheYoutubeVideo"}); return; } var dlVideo = ytdl(url,{ format : videoFormat}); @@ -3809,6 +3835,7 @@ server_Cache.prototype = { _gthis.log(client,"Error during video download: " + err); _gthis.remove(inVideoName); _gthis.remove(inAudioName); + _gthis.main.send(client,{ type : "Progress", progress : { type : "Canceled", ratio : 1}}); }); var dlAudio = ytdl(url,{ format : audioFormat}); dlAudio.pipe(js_node_Fs.createWriteStream("" + _gthis.cacheDir + "/" + inAudioName)); @@ -3816,29 +3843,65 @@ server_Cache.prototype = { _gthis.log(client,"Error during audio download: " + err); _gthis.remove(inVideoName); _gthis.remove(inAudioName); + _gthis.main.send(client,{ type : "Progress", progress : { type : "Canceled", ratio : 1}}); }); var count = 0; var onComplete = function(type) { count += 1; - haxe_Log.trace("" + type + " track downloaded (" + count + "/2)",{ fileName : "src/server/Cache.hx", lineNumber : 134, className : "server.Cache", methodName : "cacheYoutubeVideo"}); + haxe_Log.trace("" + type + " track downloaded (" + count + "/2)",{ fileName : "src/server/Cache.hx", lineNumber : 164, className : "server.Cache", methodName : "cacheYoutubeVideo"}); if(count < 2) { return; } if(!_gthis.isFileExists(inVideoName) || !_gthis.isFileExists(inAudioName)) { + _gthis.log(client,"Input files not found for making final video"); _gthis.remove(inVideoName); _gthis.remove(inAudioName); + _gthis.main.send(client,{ type : "Progress", progress : { type : "Canceled", ratio : 1}}); return; } var size = js_node_Fs.statSync("" + _gthis.cacheDir + "/" + inVideoName).size; size += js_node_Fs.statSync("" + _gthis.cacheDir + "/" + inAudioName).size; - _gthis.removeOlderCache(size + _gthis.freeSpaceBlock); + var hasSpace = _gthis.removeOlderCache(size + _gthis.freeSpaceBlock); + if(!hasSpace) { + _gthis.remove(inVideoName); + _gthis.remove(inAudioName); + _gthis.main.send(client,{ type : "Progress", progress : { type : "Canceled", ratio : 1}}); + _gthis.log(client,_gthis.notEnoughSpaceErrorText); + return; + } var args = ("-y -i ./" + inVideoName + " -i ./" + inAudioName + " -c copy -map 0:v -map 1:a ./" + outName).split(" "); - var $process = js_node_ChildProcess.spawn("ffmpeg",args,{ cwd : _gthis.cacheDir, stdio : "ignore"}); + var $process = js_node_ChildProcess.spawn("ffmpeg",args,{ cwd : _gthis.cacheDir}); + var outputData = []; + $process.stderr.on("data",function(data) { + return outputData.push(data); + }); $process.on("close",function(code) { _gthis.remove(inVideoName); _gthis.remove(inAudioName); if(code != 0) { - _gthis.log(client,"Error: ffmpeg closed with code " + code); + _gthis.main.send(client,{ type : "Progress", progress : { type : "Canceled", ratio : 1}}); + var errCodeMsg = "Error: ffmpeg closed with code " + code; + var _g = []; + var _g1 = 0; + var _g2 = _gthis.main.clients; + while(_g1 < _g2.length) { + var v = _g2[_g1]; + ++_g1; + if((v.group & 1 << ClientGroup.Admin._hx_index) != 0) { + _g.push(v); + } + } + var admins = _g; + var _g = 0; + while(_g < admins.length) { + var client1 = admins[_g]; + ++_g; + _gthis.log(client1,js_node_buffer_Buffer.concat(outputData).toString()); + _gthis.log(client1,errCodeMsg); + } + if(admins.indexOf(client) == -1) { + _gthis.log(client,errCodeMsg); + } return; } _gthis.add(outName); @@ -3855,14 +3918,30 @@ server_Cache.prototype = { dlAudio.on("progress",function(chunkLength,downloaded,contentLength) { if(isAudioStart) { isAudioStart = false; - _gthis.removeOlderCache(contentLength); + var hasSpace = _gthis.removeOlderCache(contentLength + _gthis.freeSpaceBlock); + if(!hasSpace) { + dlVideo.destroy(); + dlAudio.destroy(); + _gthis.remove(inVideoName); + _gthis.remove(inAudioName); + _gthis.main.send(client,{ type : "Progress", progress : { type : "Canceled", ratio : 1}}); + _gthis.main.serverMessage(client,_gthis.notEnoughSpaceErrorText); + } } }); var isVideoStart = true; dlVideo.on("progress",function(chunkLength,downloaded,contentLength) { if(isVideoStart) { isVideoStart = false; - _gthis.removeOlderCache(contentLength); + var hasSpace = _gthis.removeOlderCache(contentLength + _gthis.freeSpaceBlock); + if(!hasSpace) { + dlVideo.destroy(); + dlAudio.destroy(); + _gthis.remove(inVideoName); + _gthis.remove(inAudioName); + _gthis.main.send(client,{ type : "Progress", progress : { type : "Canceled", ratio : 1}}); + _gthis.main.serverMessage(client,_gthis.notEnoughSpaceErrorText); + } } var v = downloaded / contentLength; var ratio = v < 0 ? 0 : v > 1 ? 1 : v; @@ -3871,6 +3950,7 @@ server_Cache.prototype = { }).catch(function(err) { _gthis.remove(inVideoName); _gthis.remove(inAudioName); + _gthis.main.send(client,{ type : "Progress", progress : { type : "Canceled", ratio : 1}}); _gthis.log(client,"" + err); }); } @@ -3879,16 +3959,8 @@ server_Cache.prototype = { this.storageLimit = bytes; var a = this.storageLimit; this.storageLimit = a < 0 ? 0 : a; - var tmp = js_node_Fs.statfs; - if(tmp == null) { - return; - } - tmp("/",function(err,stats) { - if(err != null) { - haxe_Log.trace(err,{ fileName : "src/server/Cache.hx", lineNumber : 200, className : "server.Cache", methodName : "setStorageLimit"}); - return; - } - var a = stats.bsize * stats.bavail - _gthis.freeSpaceBlock; + this.getFreeDiskSpace(function(availSpace) { + var a = availSpace - _gthis.freeSpaceBlock; var availSpace = a < 0 ? 0 : a; _gthis.removeOlderCache(); var freeSpace = _gthis.getFreeSpace(); @@ -3899,6 +3971,23 @@ server_Cache.prototype = { } }); } + ,getFreeDiskSpace: function(callback) { + var _gthis = this; + var tmp = js_node_Fs.statfs; + if(tmp == null) { + haxe_Log.trace("Warning: no fs.statfs support in current nodejs version (needs v18+)",{ fileName : "src/server/Cache.hx", lineNumber : 267, className : "server.Cache", methodName : "getFreeDiskSpace"}); + callback(this.storageLimit); + return; + } + tmp("/",function(err,stats) { + if(err != null) { + haxe_Log.trace(err,{ fileName : "src/server/Cache.hx", lineNumber : 273, className : "server.Cache", methodName : "getFreeDiskSpace"}); + callback(_gthis.storageLimit); + return; + } + callback(stats.bsize * stats.bavail); + }); + } ,add: function(name) { if(this.cachedFiles.indexOf(name) == -1) { this.cachedFiles.unshift(name); @@ -3921,6 +4010,7 @@ server_Cache.prototype = { this.removeFile(tmp); space = this.getUsedSpace(addFileSize); } + return space < this.storageLimit; } ,removeFile: function(name) { var path = this.getFilePath(name); @@ -4285,7 +4375,7 @@ server_HttpServer.prototype = { } if(this.cache.getFreeSpace() < tmp) { res.statusCode = 413; - res.end(JSON.stringify({ info : "Error: Not enough free space on server or file size is out of cache storage limit.", errorId : "freeSpace"})); + res.end(JSON.stringify({ info : this.cache.notEnoughSpaceErrorText, errorId : "freeSpace"})); var _this = _gthis.uploadingFilesSizes; if(Object.prototype.hasOwnProperty.call(_this.h,filePath)) { delete(_this.h[filePath]); @@ -4300,7 +4390,7 @@ server_HttpServer.prototype = { if(tmp1 == null) { return; } - this.main.serverMessage(tmp1,"Error: Not enough free space on server or file size is out of cache storage limit."); + this.main.serverMessage(tmp1,this.cache.notEnoughSpaceErrorText); return; } var stream = js_node_Fs.createWriteStream(filePath); @@ -4320,7 +4410,7 @@ server_HttpServer.prototype = { } }); stream.on("error",function(err) { - haxe_Log.trace(err,{ fileName : "src/server/HttpServer.hx", lineNumber : 197, className : "server.HttpServer", methodName : "uploadFile"}); + haxe_Log.trace(err,{ fileName : "src/server/HttpServer.hx", lineNumber : 196, className : "server.HttpServer", methodName : "uploadFile"}); res.statusCode = 500; res.end(JSON.stringify({ info : "File write stream error."})); var _this = _gthis.uploadingFilesSizes; @@ -4334,7 +4424,7 @@ server_HttpServer.prototype = { _gthis.cache.remove(name); }); req.on("error",function(err) { - haxe_Log.trace("Request Error:",{ fileName : "src/server/HttpServer.hx", lineNumber : 204, className : "server.HttpServer", methodName : "uploadFile", customParams : [err]}); + haxe_Log.trace("Request Error:",{ fileName : "src/server/HttpServer.hx", lineNumber : 203, className : "server.HttpServer", methodName : "uploadFile", customParams : [err]}); stream.destroy(); res.statusCode = 500; res.end(JSON.stringify({ info : "File request error."})); @@ -4559,22 +4649,22 @@ server_Logger.prototype = { return; } server_Utils.ensureDir(this.folder); - this.removeOldestLog(this.folder); + this.removeOldestLog(); var name = DateTools.format(new Date(),"%Y-%m-%d_%H_%M_%S"); js_node_Fs.writeFileSync("" + this.folder + "/" + name + ".json",server_Main.jsonStringify(this.getLogs(),"\t")); } ,getLogs: function() { return this.logs; } - ,removeOldestLog: function(folder) { + ,removeOldestLog: function() { var _gthis = this; - var _this = js_node_Fs.readdirSync(folder); + var _this = js_node_Fs.readdirSync(this.folder); var _g = []; var _g1 = 0; while(_g1 < _this.length) { var v = _this[_g1]; ++_g1; - if(sys_FileSystem.isDirectory("" + folder + "/" + v) ? false : StringTools.startsWith(v,".") ? false : StringTools.endsWith(v,".json")) { + if(sys_FileSystem.isDirectory("" + _gthis.folder + "/" + v) ? false : StringTools.startsWith(v,".") ? false : StringTools.endsWith(v,".json")) { _g.push(v); } } @@ -4598,7 +4688,7 @@ server_Logger.prototype = { if(fileName == null) { return; } - js_node_Fs.unlinkSync("" + folder + "/" + fileName); + js_node_Fs.unlinkSync("" + this.folder + "/" + fileName); } ,extractFileDate: function(name) { name = haxe_io_Path.withoutExtension(name); @@ -4865,7 +4955,7 @@ server_Main.prototype = { this.writeUsers(this.userList); } ,getCurrentState: function() { - return { videoList : this.videoList.items, isPlaylistOpen : this.videoList.isOpen, itemPos : this.videoList.pos, messages : this.messages, timer : { time : this.videoTimer.getTime(), paused : this.videoTimer.isPaused()}, flashbacks : this.flashbacks, cachedFiles : this.cache.cachedFiles}; + return { videoList : this.videoList.items, isPlaylistOpen : this.videoList.isOpen, itemPos : this.videoList.pos, messages : this.messages, timer : { time : this.videoTimer.getTime(), paused : this.videoTimer.isPaused()}, flashbacks : this.flashbacks, cachedFiles : this.cache.getCachedFiles()}; } ,loadState: function() { if(this.isNoState) { @@ -4889,15 +4979,13 @@ server_Main.prototype = { var _g = 0; var _g1 = state.flashbacks; while(_g < _g1.length) this.flashbacks.push(_g1[_g++]); - this.cache.cachedFiles.length = 0; - var _g = 0; - var _g1 = state.cachedFiles; - while(_g < _g1.length) this.cache.cachedFiles.push(_g1[_g++]); + this.cache.setCachedFiles(state.cachedFiles); this.videoTimer.start(); this.videoTimer.setTime(state.timer.time); this.videoTimer.pause(); } ,logError: function(type,data) { + this.cache.removeOlderCache(1048576); haxe_Log.trace(type,{ fileName : "src/server/Main.hx", lineNumber : 327, className : "server.Main", methodName : "logError", customParams : [data]}); var crashesFolder = "" + this.rootDir + "/user/crashes"; server_Utils.ensureDir(crashesFolder); diff --git a/res/client.js b/res/client.js index a7560d6..e818b1f 100644 --- a/res/client.js +++ b/res/client.js @@ -1596,7 +1596,7 @@ client_Main.prototype = { if(StringTools.startsWith(url,"/")) { var host = $global.location.hostname; var port = $global.location.port; - url = "" + protocol + "//" + host + ":" + port + url; + url = "" + protocol + "//" + host + (port.length > 0 ? ":" + port : port) + url; } if(!StringTools.startsWith(url,"http")) { url = "" + protocol + "//" + url; @@ -1677,7 +1677,7 @@ client_Main.prototype = { var data = JSON.parse(e.data); if(this.config != null && this.config.isVerbose) { var t = data.type; - haxe_Log.trace("Event: " + data.type,{ fileName : "src/client/Main.hx", lineNumber : 456, className : "client.Main", methodName : "onMessage", customParams : [Reflect.field(data,t.charAt(0).toLowerCase() + HxOverrides.substr(t,1,null))]}); + haxe_Log.trace("Event: " + data.type,{ fileName : "src/client/Main.hx", lineNumber : 457, className : "client.Main", methodName : "onMessage", customParams : [Reflect.field(data,t.charAt(0).toLowerCase() + HxOverrides.substr(t,1,null))]}); } client_JsApi.fireEvents(data); switch(data.type) { @@ -1915,6 +1915,9 @@ client_Main.prototype = { case "Caching": text = "" + Lang.get("caching") + " " + data1.data; break; + case "Canceled": + this.hideDynamicChin(); + return; case "Downloading": text = Lang.get("downloading"); break; diff --git a/res/css/des.css b/res/css/des.css index f047714..0a88856 100644 --- a/res/css/des.css +++ b/res/css/des.css @@ -800,6 +800,7 @@ footer#footer { font-size: .8em; font-style: normal; color: var(--midground); + flex-shrink: 0; } #messagebuffer .text { @@ -808,6 +809,7 @@ footer#footer { .server-whisper { font-style: italic; + word-break: break-word; } #scroll-to-chat-end { diff --git a/src/Types.hx b/src/Types.hx index ac2599c..2dfe899 100644 --- a/src/Types.hx +++ b/src/Types.hx @@ -117,6 +117,7 @@ enum abstract ProgressType(String) { var Caching; var Downloading; var Uploading; + var Canceled; } @:using(Types.VideoItemTools) diff --git a/src/client/Main.hx b/src/client/Main.hx index 5c4b28d..72a619a 100644 --- a/src/client/Main.hx +++ b/src/client/Main.hx @@ -327,7 +327,8 @@ class Main { if (url.startsWith("/")) { final host = Browser.location.hostname; final port = Browser.location.port; - url = '$protocol//$host:$port$url'; + final colonPort = port.length > 0 ? ':$port' : port; + url = '$protocol//$host$colonPort$url'; } if (!url.startsWith("http")) url = '$protocol//$url'; @@ -667,6 +668,9 @@ class Main { '$caching $name'; case Downloading: Lang.get("downloading"); case Uploading: Lang.get("uploading"); + case Canceled: + hideDynamicChin(); + return; } final percent = (data.ratio * 100).toFixed(1); var text = '$text...'; diff --git a/src/server/Cache.hx b/src/server/Cache.hx index d772648..cfd5d88 100644 --- a/src/server/Cache.hx +++ b/src/server/Cache.hx @@ -2,6 +2,7 @@ package server; import haxe.io.Path; import js.lib.Promise; +import js.node.Buffer; import js.node.ChildProcess; import js.node.Fs.Fs; import js.node.stream.Readable; @@ -9,16 +10,16 @@ import sys.FileSystem; import utils.YoutubeUtils; class Cache { - final main:Main; - final cacheDir:String; - - public final cachedFiles:Array = []; + public final notEnoughSpaceErrorText = "Error: Not enough free space on server or file size is out of cache storage limit."; public final isYtReady = false; /** In bytes **/ public var storageLimit(default, null) = 3 * 1024 * 1024 * 1024; + final main:Main; + final cacheDir:String; + final cachedFiles:Array = []; final freeSpaceBlock = 10 * 1024 * 1024; // 10MB public function new(main:Main, cacheDir:String) { @@ -51,6 +52,24 @@ class Cache { } } + public function getCachedFiles():Array { + return cachedFiles; + } + + public function setCachedFiles(names:Array) { + cachedFiles.resize(0); + for (name in names) cachedFiles.push(name); + + final names = FileSystem.readDirectory(cacheDir); + for (name in names) { + if (name.startsWith(".")) continue; + if (FileSystem.isDirectory('$cacheDir/$name')) continue; + if (cachedFiles.contains(name)) continue; + trace('Remove non-tracked cache $name'); + remove(name); + } + } + function log(client:Client, msg:String):Void { main.serverMessage(client, msg); trace(msg); @@ -77,6 +96,15 @@ class Cache { remove(inVideoName); remove(inAudioName); } + inline function cancelProgress():Void { + main.send(client, { + type: Progress, + progress: { + type: Canceled, + ratio: 1 + } + }); + } if (isFileExists(inVideoName)) { log(client, 'Caching $outName already in progress'); return; @@ -117,6 +145,7 @@ class Cache { dlVideo.on("error", err -> { log(client, "Error during video download: " + err); removeInputFiles(); + cancelProgress(); }); final dlAudio:Readable = ytdl(url, { @@ -126,6 +155,7 @@ class Cache { dlAudio.on("error", err -> { log(client, "Error during audio download: " + err); removeInputFiles(); + cancelProgress(); }); var count = 0; @@ -134,26 +164,40 @@ class Cache { trace('$type track downloaded ($count/2)'); if (count < 2) return; if (!isFileExists(inVideoName) || !isFileExists(inAudioName)) { + log(client, "Input files not found for making final video"); removeInputFiles(); + cancelProgress(); return; } var size = FileSystem.stat('$cacheDir/$inVideoName').size; size += FileSystem.stat('$cacheDir/$inAudioName').size; // clean some space for full mp4 - removeOlderCache(size + freeSpaceBlock); + final hasSpace = removeOlderCache(size + freeSpaceBlock); + if (!hasSpace) { + removeInputFiles(); + cancelProgress(); + log(client, notEnoughSpaceErrorText); + return; + } final args = '-y -i ./$inVideoName -i ./$inAudioName -c copy -map 0:v -map 1:a ./$outName'.split(" "); final process = ChildProcess.spawn("ffmpeg", args, { cwd: cacheDir, - stdio: "ignore" + // stdio: "ignore" }); - // process.stderr.on('data', (data) -> { - // trace('FFmpeg stderr: ${data}'); - // }); + final outputData:Array = []; + process.stderr.on("data", (data) -> outputData.push(data)); process.on("close", (code:Int) -> { removeInputFiles(); if (code != 0) { - log(client, 'Error: ffmpeg closed with code $code'); + cancelProgress(); + final errCodeMsg = 'Error: ffmpeg closed with code $code'; + final admins = main.clients.filter(client -> client.isAdmin); + for (client in admins) { + log(client, Buffer.concat(outputData).toString()); + log(client, errCodeMsg); + } + if (!admins.contains(client)) log(client, errCodeMsg); return; } add(outName); @@ -163,18 +207,28 @@ class Cache { } dlVideo.on("finish", () -> onComplete("Video")); dlAudio.on("finish", () -> onComplete("Audio")); + inline function checkEnoughSpace(contentLength:Int):Void { + final hasSpace = removeOlderCache(contentLength + freeSpaceBlock); + if (!hasSpace) { + dlVideo.destroy(); + dlAudio.destroy(); + removeInputFiles(); + cancelProgress(); + main.serverMessage(client, notEnoughSpaceErrorText); + } + } var isAudioStart = true; dlAudio.on("progress", (chunkLength:Int, downloaded:Int, contentLength:Int) -> { if (isAudioStart) { isAudioStart = false; - removeOlderCache(contentLength); + checkEnoughSpace(contentLength); } }); var isVideoStart = true; dlVideo.on("progress", (chunkLength:Int, downloaded:Int, contentLength:Int) -> { if (isVideoStart) { isVideoStart = false; - removeOlderCache(contentLength); + checkEnoughSpace(contentLength); } final ratio = (downloaded / contentLength).clamp(0, 1); main.send(client, { @@ -187,6 +241,7 @@ class Cache { }); }).catchError(err -> { removeInputFiles(); + cancelProgress(); log(client, "" + err); }); } @@ -194,13 +249,8 @@ class Cache { public function setStorageLimit(bytes:Int) { storageLimit = cast bytes; storageLimit = storageLimit.limitMin(0); - final statfs = (Fs : Dynamic).statfs ?? return; - statfs("/", (err, stats) -> { - if (err != null) { - trace(err); - return; - } - final availSpace = (stats.bsize * stats.bavail - freeSpaceBlock).limitMin(0); + getFreeDiskSpace(availSpace -> { + final availSpace = (availSpace - freeSpaceBlock).limitMin(0); removeOlderCache(); final freeSpace = getFreeSpace(); if (availSpace < freeSpace) { @@ -212,6 +262,22 @@ class Cache { }); } + public function getFreeDiskSpace(callback:(availSpace:Int) -> Void):Void { + final statfs = (Fs : Dynamic).statfs ?? { + trace("Warning: no fs.statfs support in current nodejs version (needs v18+)"); + callback(storageLimit); + return; + } + statfs("/", (err, stats) -> { + if (err != null) { + trace(err); + callback(storageLimit); + return; + } + callback(stats.bsize * stats.bavail); + }); + } + public function add(name:String) { if (!cachedFiles.contains(name)) { cachedFiles.unshift(name); @@ -223,13 +289,15 @@ class Cache { removeFile(name); } - public function removeOlderCache(addFileSize = 0):Void { + /** Returns `true` if there is enough space to save `addFileSize` bytes. **/ + public function removeOlderCache(addFileSize = 0):Bool { var space = getUsedSpace(addFileSize); while (space > storageLimit) { final name = cachedFiles.pop() ?? break; removeFile(name); space = getUsedSpace(addFileSize); } + return space < storageLimit; } function removeFile(name:String):Void { diff --git a/src/server/HttpServer.hx b/src/server/HttpServer.hx index 346aac1..8f0b56e 100644 --- a/src/server/HttpServer.hx +++ b/src/server/HttpServer.hx @@ -170,15 +170,14 @@ class HttpServer { cache.removeOlderCache(size); } if (cache.getFreeSpace() < size) { - final errText = "Error: Not enough free space on server or file size is out of cache storage limit."; end(413, { // Payload Too Large - info: errText, + info: cache.notEnoughSpaceErrorText, errorId: "freeSpace", }); cache.remove(name); req.destroy(); final client = main.clients.getByName(name) ?? return; - main.serverMessage(client, errText); + main.serverMessage(client, cache.notEnoughSpaceErrorText); return; } diff --git a/src/server/Logger.hx b/src/server/Logger.hx index 42d382d..f22c9d9 100644 --- a/src/server/Logger.hx +++ b/src/server/Logger.hx @@ -38,7 +38,7 @@ class Logger { public function saveLog():Void { if (logs.length == 0) return; Utils.ensureDir(folder); - removeOldestLog(folder); + removeOldestLog(); final name = DateTools.format(Date.now(), "%Y-%m-%d_%H_%M_%S"); File.saveContent('$folder/$name.json', Main.jsonStringify(getLogs(), "\t")); } @@ -47,7 +47,7 @@ class Logger { return logs; } - function removeOldestLog(folder:String):Void { + public function removeOldestLog():Void { final names = FileSystem.readDirectory(folder).filter(name -> { if (FileSystem.isDirectory('$folder/$name')) return false; if (name.startsWith(".")) return false; diff --git a/src/server/Main.hx b/src/server/Main.hx index 644a8b1..7d78879 100644 --- a/src/server/Main.hx +++ b/src/server/Main.hx @@ -293,7 +293,7 @@ class Main { paused: videoTimer.isPaused() }, flashbacks: flashbacks, - cachedFiles: cache.cachedFiles + cachedFiles: cache.getCachedFiles() } } @@ -315,8 +315,7 @@ class Main { flashbacks.resize(0); for (flashback in state.flashbacks) flashbacks.push(flashback); - cache.cachedFiles.resize(0); - for (name in state.cachedFiles) cache.cachedFiles.push(name); + cache.setCachedFiles(state.cachedFiles); videoTimer.start(); videoTimer.setTime(state.timer.time); @@ -324,6 +323,7 @@ class Main { } function logError(type:String, data:Dynamic):Void { + cache.removeOlderCache(1024 * 1024); trace(type, data); final crashesFolder = '$rootDir/user/crashes'; Utils.ensureDir(crashesFolder); -- cgit v1.2.3