diff options
| author | Pinapelz <yukais@pinapelz.com> | 2026-11-01 12:18:12 -0700 |
|---|---|---|
| committer | Pinapelz <yukais@pinapelz.com> | 2026-11-01 12:18:16 -0700 |
| commit | 69b7b1cde8b144401c8eab05e6a80605c38f5952 (patch) | |
| tree | 6e17d7d3e4a94fc085626c42dfbdd6ca3321df98 | |
| parent | 29ff6ad6bcf6e7bfc9fece59a3bd849a786022a1 (diff) | |
add rnnoise for voice
| -rw-r--r-- | __pycache__/app.cpython-313.pyc | bin | 25241 -> 27259 bytes | |||
| -rw-r--r-- | app.py | 37 | ||||
| -rw-r--r-- | cache.json | 2 | ||||
| -rw-r--r-- | index.css | 46 | ||||
| -rw-r--r-- | static/mic-worklet.js | 172 | ||||
| -rw-r--r-- | static/voice.js | 485 |
6 files changed, 729 insertions, 13 deletions
diff --git a/__pycache__/app.cpython-313.pyc b/__pycache__/app.cpython-313.pyc Binary files differindex 8f15542..011fed6 100644 --- a/__pycache__/app.cpython-313.pyc +++ b/__pycache__/app.cpython-313.pyc @@ -342,7 +342,10 @@ PAGE_TEMPLATE = """ <section class="voice-topbar" id="voice-topbar" aria-label="Browser voice setup"> <div class="voice-topbar-bar"> <span class="voice-topbar-identity" id="voice-topbar-identity">[ not logged in ]</span> - <button type="button" class="voice-login-btn" id="voice-login-btn">browser connect</button> + <div class="voice-topbar-actions"> + <button type="button" class="btn-voice btn-voice-rnnoise voice-topbar-btn" id="voice-wasm-open-btn" aria-pressed="true" title="Open RNNoise WASM settings">rnnoise: on</button> + <button type="button" class="voice-login-btn" id="voice-login-btn">browser connect</button> + </div> </div> <p class="voice-topbar-hint" id="voice-message" role="status" aria-live="polite" hidden></p> </section> @@ -411,6 +414,36 @@ PAGE_TEMPLATE = """ </form> </dialog> +<dialog class="voice-dialog" id="voice-wasm-dialog" aria-labelledby="voice-wasm-dialog-title"> + <div class="voice-dialog-form"> + <header class="voice-dialog-header"> + <h2 id="voice-wasm-dialog-title">RNNoise WASM</h2> + <span class="voice-dialog-target voice-wasm-status" id="voice-wasm-status">not cached</span> + </header> + <div class="voice-dialog-body"> + <div class="voice-field"> + <span class="voice-field-tag">[ background noise suppression ]</span> + <p class="voice-wasm-desc">Filters microphone input through RNNoise (WebAssembly) before audio is sent to the voice gateway.</p> + <div class="voice-wasm-controls"> + <button type="button" class="btn-voice btn-voice-rnnoise" id="voice-rnnoise-toggle" aria-pressed="true">rnnoise: on</button> + </div> + <p class="voice-wasm-desc">Processing: <span class="voice-wasm-status" id="voice-wasm-active" data-state="idle">not in a call</span></p> + </div> + <div class="voice-field"> + <span class="voice-field-tag">[ wasm model cache ]</span> + <div class="voice-wasm-controls"> + <button type="button" class="btn-voice" id="voice-wasm-redownload">re-download</button> + <button type="button" class="btn-voice btn-voice-danger" id="voice-wasm-delete">delete</button> + </div> + </div> + <p class="voice-dialog-message" id="voice-wasm-message" role="status" aria-live="polite" hidden></p> + </div> + <footer class="voice-dialog-actions"> + <button type="button" class="btn-voice" id="voice-wasm-close">close</button> + </footer> + </div> +</dialog> + <section class="voice-bar" id="voice-bar" aria-label="Voice controls" hidden> <div class="voice-meter" aria-hidden="true"> <div class="voice-meter-fill" id="voice-meter"></div> @@ -505,7 +538,7 @@ PAGE_TEMPLATE = """ }); {% endif %} </script> -{% if voice_available %}<script src="/static/voice.js" defer></script>{% endif %} +{% if voice_available %}<script src="/static/voice.js?v=rnnoise-2" defer></script>{% endif %} </body> </html> """ @@ -1 +1 @@ -{"channels": [{"channel_name": "[Cspacer]INFORMATION", "channel_needed_subscribe_power": "0", "channel_order": "0", "cid": "13", "pid": "0", "total_clients": "0"}, {"channel_name": "[Cspacer]GAME SERVERS", "channel_needed_subscribe_power": "0", "channel_order": "13", "cid": "17", "pid": "0", "total_clients": "0"}, {"channel_name": "[Cspacer]\ud83d\udd07 AFK NPCs \ud83d\udd07", "channel_needed_subscribe_power": "0", "channel_order": "17", "cid": "4", "pid": "0", "total_clients": "1"}, {"channel_name": "General", "channel_needed_subscribe_power": "0", "channel_order": "4", "cid": "1", "pid": "0", "total_clients": "0"}, {"channel_name": "CRASHING OUT", "channel_needed_subscribe_power": "0", "channel_order": "1", "cid": "9", "pid": "0", "total_clients": "0"}, {"channel_name": "Wonhee HQ", "channel_needed_subscribe_power": "0", "channel_order": "9", "cid": "10", "pid": "0", "total_clients": "1"}], "clients": [{"cid": "4", "clid": "1433", "client_channel_group_id": "8", "client_channel_group_inherited_channel_id": "4", "client_database_id": "1", "client_nickname": "serveradmin", "client_servergroups": "2", "client_type": "1"}, {"cid": "10", "clid": "104", "client_channel_group_id": "5", "client_channel_group_inherited_channel_id": "10", "client_database_id": "29", "client_nickname": "The Real Wonhee", "client_servergroups": "8", "client_type": "0", "afk_text": "", "is_afk": false, "is_streaming": false, "stream_text": ""}]}
\ No newline at end of file +{"channels": [{"channel_name": "[Cspacer]INFORMATION", "channel_needed_subscribe_power": "0", "channel_order": "0", "cid": "13", "pid": "0", "total_clients": "0"}, {"channel_name": "[Cspacer]GAME SERVERS", "channel_needed_subscribe_power": "0", "channel_order": "13", "cid": "17", "pid": "0", "total_clients": "0"}, {"channel_name": "[Cspacer]\ud83d\udd07 AFK NPCs \ud83d\udd07", "channel_needed_subscribe_power": "0", "channel_order": "17", "cid": "4", "pid": "0", "total_clients": "1"}, {"channel_name": "General", "channel_needed_subscribe_power": "0", "channel_order": "4", "cid": "1", "pid": "0", "total_clients": "0"}, {"channel_name": "CRASHING OUT", "channel_needed_subscribe_power": "0", "channel_order": "1", "cid": "9", "pid": "0", "total_clients": "0"}, {"channel_name": "Wonhee HQ", "channel_needed_subscribe_power": "0", "channel_order": "9", "cid": "10", "pid": "0", "total_clients": "1"}], "clients": [{"cid": "4", "clid": "1893", "client_channel_group_id": "8", "client_channel_group_inherited_channel_id": "4", "client_database_id": "1", "client_nickname": "serveradmin", "client_servergroups": "2", "client_type": "1"}, {"cid": "10", "clid": "104", "client_channel_group_id": "5", "client_channel_group_inherited_channel_id": "10", "client_database_id": "29", "client_nickname": "The Real Wonhee", "client_servergroups": "8", "client_type": "0", "afk_text": "", "is_afk": false, "is_streaming": false, "stream_text": ""}]}
\ No newline at end of file @@ -411,6 +411,18 @@ body.voice-live { color: var(--cl-green-6); } +.voice-topbar-actions { + display: flex; + align-items: center; + gap: 0.6rem; + flex-shrink: 0; +} + +.voice-topbar-btn { + padding: 4px 10px; + background-color: var(--cl-gray-0); +} + .voice-login-btn { padding: 4px 10px; color: var(--cl-orange-6); @@ -900,6 +912,11 @@ body.voice-live { border-color: var(--cl-red-6); } +.btn-voice.btn-voice-rnnoise[aria-pressed="true"] { + color: var(--cl-green-6); + border-color: var(--cl-green-6); +} + .btn-voice[aria-expanded="true"] { color: var(--cl-orange-6); border-color: var(--cl-orange-6); @@ -963,6 +980,35 @@ body.voice-live { align-items: flex-start; } +.voice-wasm-desc { + margin: 0; + color: var(--cl-gray-7); + font-size: 0.85rem; + line-height: 1.45; +} + +.voice-wasm-controls { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 0.55rem; +} + +.voice-wasm-status[data-state="cached"] { + color: var(--cl-green-6); + border-color: var(--cl-green-6); +} + +.voice-wasm-status[data-state="loading"] { + color: var(--cl-orange-6); + border-color: var(--cl-orange-6); +} + +.voice-wasm-status[data-state="error"] { + color: var(--cl-red-6); + border-color: var(--cl-red-6); +} + .voice-setting input[type="range"] { width: 100%; margin: 0.35rem 0; diff --git a/static/mic-worklet.js b/static/mic-worklet.js index 814bf6a..3276488 100644 --- a/static/mic-worklet.js +++ b/static/mic-worklet.js @@ -1,30 +1,196 @@ -// Microphone capture worklet: resamples the context rate to 48 kHz mono and +// Microphone capture worklet: resamples the context rate to 48 kHz mono, +// optionally runs RNNoise (WASM) on 10 ms (480-sample) sub-frames, and // posts 20 ms frames (960 Float32 samples) together with their RMS level. const OUT_RATE = 48000; const FRAME_SAMPLES = 960; +const RNNOISE_FRAME_SAMPLES = 480; +const PCM16_SCALE = 32768; + +function instantiateRnnoise(wasmBinary) { + let memory = null; + const resizeHeap = (requestedSize) => { + if (!memory) return 0; + const pages = ((requestedSize >>> 0) - memory.buffer.byteLength + 65535) >>> 16; + try { + memory.grow(pages); + return 1; + } catch { + return 0; + } + }; + const memcpyBig = (dest, src, num) => { + if (!memory) return; + new Uint8Array(memory.buffer).copyWithin(dest >>> 0, src >>> 0, (src + num) >>> 0); + }; + const imports = { + a: { a: resizeHeap, b: memcpyBig }, + env: { + emscripten_resize_heap: resizeHeap, + emscripten_memcpy_big: memcpyBig, + __assert_fail: () => { + throw new Error("RNNoise WASM assertion failed"); + }, + }, + }; + + const buildDenoiser = (instance) => { + const exp = instance.exports; + memory = exp.memory || exp.c; + if (typeof exp.emscripten_stack_init === "function") exp.emscripten_stack_init(); + const initRuntime = exp.__wasm_call_ctors || exp.d; + if (typeof initRuntime === "function") initRuntime(); + + const rnnoiseInit = exp.rnnoise_init || exp._rnnoise_init || exp.e; + const rnnoiseCreate = exp.rnnoise_create || exp._rnnoise_create || exp.f; + const malloc = exp.malloc || exp._malloc || exp.g; + const rnnoiseDestroy = exp.rnnoise_destroy || exp._rnnoise_destroy || exp.h; + const free = exp.free || exp._free || exp.i; + const rnnoiseProcessFrame = exp.rnnoise_process_frame || exp._rnnoise_process_frame || exp.j; + + const state = rnnoiseCreate(0); + const ptrIn = malloc(RNNOISE_FRAME_SAMPLES * 4); + const ptrOut = malloc(RNNOISE_FRAME_SAMPLES * 4); + if (!state || !ptrIn || !ptrOut) { + if (state) rnnoiseDestroy(state); + if (ptrIn) free(ptrIn); + if (ptrOut) free(ptrOut); + throw new Error("Failed to allocate RNNoise WASM state"); + } + + let heapF32 = new Float32Array(memory.buffer); + let destroyed = false; + + return { + processFrame480(frame, offset) { + if (destroyed) return 0; + if (heapF32.buffer !== memory.buffer) { + heapF32 = new Float32Array(memory.buffer); + } + const inBase = ptrIn >> 2; + const outBase = ptrOut >> 2; + for (let i = 0; i < RNNOISE_FRAME_SAMPLES; i++) { + heapF32[inBase + i] = frame[offset + i] * PCM16_SCALE; + } + const vad = rnnoiseProcessFrame(state, ptrOut, ptrIn); + if (heapF32.buffer !== memory.buffer) { + heapF32 = new Float32Array(memory.buffer); + } + for (let i = 0; i < RNNOISE_FRAME_SAMPLES; i++) { + const s = heapF32[outBase + i] / PCM16_SCALE; + frame[offset + i] = s < -1 ? -1 : s > 1 ? 1 : s; + } + return vad; + }, + reset() { + if (destroyed) return; + if (typeof rnnoiseInit === "function") { + rnnoiseInit(state, 0); + } + }, + destroy() { + if (destroyed) return; + destroyed = true; + try { rnnoiseDestroy(state); } catch { /* ignore */ } + try { free(ptrIn); } catch { /* ignore */ } + try { free(ptrOut); } catch { /* ignore */ } + }, + }; + }; + + return WebAssembly.instantiate(wasmBinary, imports).then((res) => buildDenoiser(res.instance || res)); +} class Ts6MicCapture extends AudioWorkletProcessor { - constructor() { + constructor(options) { super(); this.ratio = sampleRate / OUT_RATE; // input samples per output sample this.t = 0; // fractional read position; -1 refers to `last` this.last = 0; this.frame = new Float32Array(FRAME_SAMPLES); this.offset = 0; + + const opts = (options && options.processorOptions) || {}; + this.rnnoiseEnabled = Boolean(opts.rnnoiseEnabled); + this.rnnoise = null; + this.disposed = false; + + this.port.onmessage = (e) => this.onMessage(e.data); + if (opts.wasmBinary) { + this.initRnnoise(opts.wasmBinary); + } + } + + onMessage(msg) { + if (!msg || typeof msg !== "object") return; + if (msg.type === "rnnoise-init" && msg.wasmBinary) { + if (typeof msg.enabled === "boolean") this.rnnoiseEnabled = msg.enabled; + this.initRnnoise(msg.wasmBinary); + } else if (msg.type === "rnnoise-enable") { + const next = Boolean(msg.enabled); + if (next && !this.rnnoiseEnabled && this.rnnoise) { + this.rnnoise.reset(); + } + this.rnnoiseEnabled = next; + } else if (msg.type === "rnnoise-clear") { + this.rnnoiseEnabled = false; + if (this.rnnoise) { + this.rnnoise.destroy(); + this.rnnoise = null; + } + this.port.postMessage({ type: "rnnoise-status", ready: false }); + } else if (msg.type === "dispose") { + this.disposed = true; + if (this.rnnoise) { + this.rnnoise.destroy(); + this.rnnoise = null; + } + } + } + + initRnnoise(wasmBinary) { + instantiateRnnoise(wasmBinary) + .then((denoiser) => { + if (this.disposed) { + denoiser.destroy(); + return; + } + if (this.rnnoise) this.rnnoise.destroy(); + this.rnnoise = denoiser; + this.port.postMessage({ type: "rnnoise-status", ready: true }); + }) + .catch((err) => { + this.port.postMessage({ + type: "rnnoise-status", + ready: false, + error: err && err.message ? err.message : String(err), + }); + }); } push(sample) { this.frame[this.offset++] = sample; if (this.offset < FRAME_SAMPLES) return; + + let denoised = false; + if (this.rnnoiseEnabled && this.rnnoise) { + this.rnnoise.processFrame480(this.frame, 0); + this.rnnoise.processFrame480(this.frame, RNNOISE_FRAME_SAMPLES); + denoised = true; + } + let sum = 0; for (let i = 0; i < FRAME_SAMPLES; i++) sum += this.frame[i] * this.frame[i]; const frame = this.frame; this.frame = new Float32Array(FRAME_SAMPLES); this.offset = 0; - this.port.postMessage({ samples: frame, rms: Math.sqrt(sum / FRAME_SAMPLES) }, [frame.buffer]); + this.port.postMessage( + { samples: frame, rms: Math.sqrt(sum / FRAME_SAMPLES), denoised }, + [frame.buffer] + ); } process(inputs, outputs) { + if (this.disposed) return false; const output = outputs[0] && outputs[0][0]; if (output) output.fill(0); const channels = inputs[0]; diff --git a/static/voice.js b/static/voice.js index 9cfa74b..6116b99 100644 --- a/static/voice.js +++ b/static/voice.js @@ -44,6 +44,15 @@ threshold: $("voice-threshold"), mute: $("voice-mute"), deafen: $("voice-deafen"), + wasmOpenBtn: $("voice-wasm-open-btn"), + wasmDialog: $("voice-wasm-dialog"), + wasmClose: $("voice-wasm-close"), + wasmMessage: $("voice-wasm-message"), + rnnoiseToggle: $("voice-rnnoise-toggle"), + wasmRedownload: $("voice-wasm-redownload"), + wasmDelete: $("voice-wasm-delete"), + wasmStatus: $("voice-wasm-status"), + wasmActive: $("voice-wasm-active"), settingsToggle: $("voice-settings-toggle"), settings: $("voice-settings"), mode: $("voice-mode"), @@ -61,6 +70,7 @@ sensitivity: "ts6voice.sensitivity", volume: "ts6voice.volume", mode: "ts6voice.mode", + rnnoise: "ts6voice.rnnoise", userVolumes: "ts6voice.userVolumes", }; const store = { @@ -76,6 +86,7 @@ const MAX_AHEAD_S = 0.25; const MAX_SEND_BUFFER = 16 * 1024; const MAX_CHAT_HISTORY = 100; + const MIC_WORKLET_VERSION = "rnnoise-2"; let session = null; // active session, see startJoin() let pendingJoin = null; // undefined = save only; null = join default channel; {cid,name} = join specific channel @@ -199,6 +210,22 @@ if (e.target === ui.dialog && (!session || session.ready)) closeLoginModal(); }); + if (ui.wasmOpenBtn && ui.wasmDialog) { + ui.wasmOpenBtn.addEventListener("click", () => { + setText(ui.wasmMessage, ""); + updateRnnoiseActiveUi(session); + if (!ui.wasmDialog.open) ui.wasmDialog.showModal(); + }); + if (ui.wasmClose) { + ui.wasmClose.addEventListener("click", () => { + if (ui.wasmDialog.open) ui.wasmDialog.close(); + }); + } + ui.wasmDialog.addEventListener("click", (e) => { + if (e.target === ui.wasmDialog) ui.wasmDialog.close(); + }); + } + // ---- preferences -------------------------------------------------------------- ui.nickname.value = store.get(STORE.nickname) || ""; @@ -206,6 +233,7 @@ if (store.get(STORE.sensitivity) !== null) ui.sensitivity.value = store.get(STORE.sensitivity); if (store.get(STORE.volume) !== null) ui.volume.value = store.get(STORE.volume); let mode = store.get(STORE.mode) === "ptt" ? "ptt" : "vad"; + let rnnoiseEnabled = store.get(STORE.rnnoise) !== "0"; updateTopbarSummary(); ui.nickname.addEventListener("input", () => { @@ -230,8 +258,17 @@ ui.ptt.hidden = mode !== "ptt"; ui.threshold.hidden = mode === "ptt"; } + function applyRnnoiseUi() { + const label = rnnoiseEnabled ? "rnnoise: on" : "rnnoise: off"; + for (const btn of [ui.wasmOpenBtn, ui.rnnoiseToggle]) { + if (!btn) continue; + btn.setAttribute("aria-pressed", String(rnnoiseEnabled)); + btn.textContent = label; + } + } updateThresholdMark(); applyModeUi(); + applyRnnoiseUi(); // ---- live channel list & chat (channel + individual DMs) -------------------- @@ -749,7 +786,304 @@ } } - // ---- microphone --------------------------------------------------------------- + // ---- microphone & RNNoise (WASM) ---------------------------------------------- + + const RNNOISE_WASM_URLS = [ + "https://cdn.jsdelivr.net/npm/@jitsi/rnnoise-wasm@0.2.1/dist/rnnoise.wasm", + "https://unpkg.com/@jitsi/rnnoise-wasm@0.2.1/dist/rnnoise.wasm", + ]; + const WASM_CACHE_NAME = "ts6voice-rnnoise-v1"; + const WASM_CACHE_KEY = "https://ts6voice.local/rnnoise.wasm"; + const RNNOISE_FRAME_SAMPLES = 480; + const PCM16_SCALE = 32768; + let rnnoiseWasmBuffer = null; + let rnnoiseWasmPromise = null; + + function setWasmStatus(text, state = "idle") { + if (!ui.wasmStatus) return; + ui.wasmStatus.textContent = text; + ui.wasmStatus.dataset.state = state; + } + + function setWasmButtonsBusy(busy) { + for (const btn of [ui.wasmRedownload, ui.wasmDelete]) { + if (btn) btn.disabled = busy; + } + } + + function sayWasm(text, kind = "info") { + if (ui.wasmDialog && ui.wasmDialog.open) setText(ui.wasmMessage, text, kind); + else say(text, kind); + } + + function formatKb(byteLength) { + return `${Math.max(1, Math.round(byteLength / 1024))} KB`; + } + + async function readWasmFromCache() { + if (typeof caches === "undefined") return null; + try { + const cache = await caches.open(WASM_CACHE_NAME); + const match = await cache.match(WASM_CACHE_KEY); + if (!match) return null; + const buf = await match.arrayBuffer(); + return buf && buf.byteLength > 0 ? buf : null; + } catch { + return null; + } + } + + async function writeWasmToCache(buf) { + if (typeof caches === "undefined" || !buf) return; + try { + const cache = await caches.open(WASM_CACHE_NAME); + await cache.put( + WASM_CACHE_KEY, + new Response(buf.slice(0), { headers: { "content-type": "application/wasm" } }) + ); + } catch { /* ignore quota or private mode errors */ } + } + + async function clearWasmCache() { + if (typeof caches === "undefined") return; + try { + await caches.delete(WASM_CACHE_NAME); + } catch { /* ignore */ } + } + + function fetchRnnoiseWasm({ forceReload = false } = {}) { + if (!forceReload && rnnoiseWasmBuffer) { + return Promise.resolve(rnnoiseWasmBuffer); + } + if (!forceReload && rnnoiseWasmPromise) { + return rnnoiseWasmPromise; + } + setWasmButtonsBusy(true); + setWasmStatus("downloading...", "loading"); + + rnnoiseWasmPromise = (async () => { + if (!forceReload) { + const cached = await readWasmFromCache(); + if (cached) { + rnnoiseWasmBuffer = cached; + setWasmStatus(`cached (${formatKb(cached.byteLength)})`, "cached"); + return cached; + } + } else { + await clearWasmCache(); + } + + let lastError = null; + for (const url of RNNOISE_WASM_URLS) { + try { + const res = await fetch(url, { + mode: "cors", + credentials: "omit", + cache: forceReload ? "reload" : "default", + }); + if (!res.ok) throw new Error(`HTTP ${res.status} from ${url}`); + const buf = await res.arrayBuffer(); + if (!buf || buf.byteLength === 0) throw new Error("Empty WASM response"); + rnnoiseWasmBuffer = buf; + await writeWasmToCache(buf); + setWasmStatus(`cached (${formatKb(buf.byteLength)})`, "cached"); + return buf; + } catch (err) { + lastError = err; + } + } + throw lastError || new Error("Failed to download rnnoise.wasm"); + })() + .catch((err) => { + rnnoiseWasmBuffer = null; + rnnoiseWasmPromise = null; + setWasmStatus("download failed", "error"); + throw err; + }) + .finally(() => { + setWasmButtonsBusy(false); + }); + + return rnnoiseWasmPromise; + } + + async function deleteRnnoiseWasm() { + setWasmButtonsBusy(true); + try { + await clearWasmCache(); + rnnoiseWasmBuffer = null; + rnnoiseWasmPromise = null; + // Keep the user's on/off preference; with no model cached, RNNoise simply + // stays inactive until the model is downloaded again. + + if (session) { + session.rnnoiseLoaded = false; + session.rnnoiseWorkletReady = false; + if (session.rnnoiseFallback) { + session.rnnoiseFallback.destroy(); + session.rnnoiseFallback = null; + } + if (session.worklet) { + try { session.worklet.port.postMessage({ type: "rnnoise-clear" }); } catch { /* ignore */ } + } + applyBrowserNoiseSuppression(session); + updateRnnoiseActiveUi(session); + } + setWasmStatus("deleted (not cached)", "idle"); + sayWasm("RNNoise WASM model deleted from browser storage.", "info"); + } finally { + setWasmButtonsBusy(false); + } + } + + async function redownloadRnnoiseWasm() { + try { + setText(ui.wasmMessage, ""); + rnnoiseWasmBuffer = null; + rnnoiseWasmPromise = null; + const wasmBinary = await fetchRnnoiseWasm({ forceReload: true }); + if (session && !session.closed) { + session.rnnoiseLoaded = true; + if (session.worklet) { + session.worklet.port.postMessage({ + type: "rnnoise-init", + wasmBinary: wasmBinary.slice(0), + enabled: rnnoiseEnabled, + }); + } + const nextFallback = await createMainThreadRnnoise(wasmBinary); + if (session && !session.closed) { + if (session.rnnoiseFallback) session.rnnoiseFallback.destroy(); + session.rnnoiseFallback = nextFallback; + applyBrowserNoiseSuppression(session); + } else { + nextFallback.destroy(); + } + } + sayWasm("RNNoise WASM model downloaded and ready.", "info"); + } catch (err) { + sayWasm(`Failed to re-download RNNoise WASM: ${err && err.message ? err.message : err}`, "error"); + } + } + + void readWasmFromCache().then((cached) => { + if (cached) { + rnnoiseWasmBuffer = cached; + setWasmStatus(`cached (${formatKb(cached.byteLength)})`, "cached"); + } else { + setWasmStatus("not cached", "idle"); + } + }); + + async function createMainThreadRnnoise(wasmBinary) { + let memory = null; + const resizeHeap = (requestedSize) => { + if (!memory) return 0; + const pages = ((requestedSize >>> 0) - memory.buffer.byteLength + 65535) >>> 16; + try { memory.grow(pages); return 1; } catch { return 0; } + }; + const memcpyBig = (dest, src, num) => { + if (!memory) return; + new Uint8Array(memory.buffer).copyWithin(dest >>> 0, src >>> 0, (src + num) >>> 0); + }; + const res = await WebAssembly.instantiate(wasmBinary, { + a: { a: resizeHeap, b: memcpyBig }, + env: { + emscripten_resize_heap: resizeHeap, + emscripten_memcpy_big: memcpyBig, + __assert_fail: () => { throw new Error("RNNoise WASM assertion failed"); }, + }, + }); + const inst = res.instance || res; + const exp = inst.exports; + memory = exp.memory || exp.c; + if (typeof exp.emscripten_stack_init === "function") exp.emscripten_stack_init(); + const initRuntime = exp.__wasm_call_ctors || exp.d; + if (typeof initRuntime === "function") initRuntime(); + + const rnnoiseInit = exp.rnnoise_init || exp._rnnoise_init || exp.e; + const rnnoiseCreate = exp.rnnoise_create || exp._rnnoise_create || exp.f; + const malloc = exp.malloc || exp._malloc || exp.g; + const rnnoiseDestroy = exp.rnnoise_destroy || exp._rnnoise_destroy || exp.h; + const free = exp.free || exp._free || exp.i; + const rnnoiseProcessFrame = exp.rnnoise_process_frame || exp._rnnoise_process_frame || exp.j; + + const state = rnnoiseCreate(0); + const ptrIn = malloc(RNNOISE_FRAME_SAMPLES * 4); + const ptrOut = malloc(RNNOISE_FRAME_SAMPLES * 4); + if (!state || !ptrIn || !ptrOut) { + if (state) rnnoiseDestroy(state); + if (ptrIn) free(ptrIn); + if (ptrOut) free(ptrOut); + throw new Error("Failed to allocate RNNoise WASM buffers"); + } + + let heapF32 = new Float32Array(memory.buffer); + let destroyed = false; + + const processSubFrame = (samples, offset) => { + if (heapF32.buffer !== memory.buffer) heapF32 = new Float32Array(memory.buffer); + const inBase = ptrIn >> 2; + const outBase = ptrOut >> 2; + for (let i = 0; i < RNNOISE_FRAME_SAMPLES; i++) { + heapF32[inBase + i] = samples[offset + i] * PCM16_SCALE; + } + rnnoiseProcessFrame(state, ptrOut, ptrIn); + if (heapF32.buffer !== memory.buffer) heapF32 = new Float32Array(memory.buffer); + for (let i = 0; i < RNNOISE_FRAME_SAMPLES; i++) { + const v = heapF32[outBase + i] / PCM16_SCALE; + samples[offset + i] = v < -1 ? -1 : v > 1 ? 1 : v; + } + }; + + return { + processFrame960(samples) { + if (destroyed || samples.length < RNNOISE_FRAME_SAMPLES * 2) return; + processSubFrame(samples, 0); + processSubFrame(samples, RNNOISE_FRAME_SAMPLES); + }, + reset() { + if (!destroyed && typeof rnnoiseInit === "function") rnnoiseInit(state, 0); + }, + destroy() { + if (destroyed) return; + destroyed = true; + try { rnnoiseDestroy(state); } catch { /* ignore */ } + try { free(ptrIn); } catch { /* ignore */ } + try { free(ptrOut); } catch { /* ignore */ } + }, + }; + } + + async function ensureSessionRnnoise(s) { + if (!s || s.closed || s.rnnoiseLoading || s.rnnoiseLoaded) return; + s.rnnoiseLoading = true; + try { + const wasmBinary = await fetchRnnoiseWasm(); + if (s.closed || session !== s) return; + s.rnnoiseLoaded = true; + if (s.worklet) { + s.worklet.port.postMessage({ + type: "rnnoise-init", + wasmBinary: wasmBinary.slice(0), + enabled: rnnoiseEnabled, + }); + } + s.rnnoiseFallback = await createMainThreadRnnoise(wasmBinary); + if (s.closed || session !== s) { + s.rnnoiseFallback.destroy(); + s.rnnoiseFallback = null; + } else { + applyBrowserNoiseSuppression(s); + } + } catch (err) { + if (session === s && rnnoiseEnabled) { + say(`RNNoise WASM failed to load: ${err && err.message ? err.message : err}`, "error"); + } + } finally { + s.rnnoiseLoading = false; + } + } function gateOpen(s, rms) { if (s.muted) return false; @@ -759,8 +1093,80 @@ return now < s.vadUntil; } - function onMicFrame(s, { samples, rms }) { - if (session !== s) return; + /** True once a working RNNoise instance exists (worklet or main thread). */ + function rnnoiseAvailable(s) { + return Boolean(s && (s.rnnoiseWorkletReady || s.rnnoiseFallback)); + } + + /** + * The browser's built-in noise suppression runs before our pipeline and + * already removes most steady background noise, which makes RNNoise look + * like it does nothing. Turn it off while RNNoise is actually processing, + * and back on otherwise so users are never left without any suppression. + */ + function applyBrowserNoiseSuppression(s) { + if (!s || !s.stream) return; + const wantBrowserNs = !(rnnoiseEnabled && rnnoiseAvailable(s)); + if (s.browserNs === wantBrowserNs) return; + s.browserNs = wantBrowserNs; + for (const track of s.stream.getAudioTracks()) { + track.applyConstraints({ noiseSuppression: wantBrowserNs }).catch(() => undefined); + } + } + + function updateRnnoiseActiveUi(s) { + if (!ui.wasmActive) return; + let text = "not in a call"; + let state = "idle"; + if (s && !s.closed) { + if (!rnnoiseEnabled) { + text = "off (browser noise suppression)"; + } else if (s.rnnoiseVia) { + text = `active (${s.rnnoiseVia})`; + state = "cached"; + } else if (s.rnnoiseError) { + text = `failed: ${s.rnnoiseError}`; + state = "error"; + } else { + text = "waiting for model..."; + state = "loading"; + } + } + if (ui.wasmActive.textContent !== text) ui.wasmActive.textContent = text; + ui.wasmActive.dataset.state = state; + } + + function onMicMessage(s, data) { + if (session !== s || !data) return; + if (data.type === "rnnoise-status") { + s.rnnoiseWorkletReady = Boolean(data.ready); + if (data.error) { + s.rnnoiseError = data.error; + console.warn("[rnnoise] worklet init failed, using main-thread fallback:", data.error); + } + applyBrowserNoiseSuppression(s); + updateRnnoiseActiveUi(s); + return; + } + if (!data.samples) return; + + const samples = data.samples; + let rms = data.rms; + let via = data.denoised ? "audio worklet" : ""; + // If RNNoise is enabled and the worklet did not denoise this frame yet, + // run the main-thread WASM denoiser before metering, VAD gating, and gateway send. + if (rnnoiseEnabled && !data.denoised && s.rnnoiseFallback) { + s.rnnoiseFallback.processFrame960(samples); + let sum = 0; + for (let i = 0; i < samples.length; i++) sum += samples[i] * samples[i]; + rms = Math.sqrt(sum / samples.length); + via = "main thread"; + } + if (s.rnnoiseVia !== via) { + s.rnnoiseVia = via; + updateRnnoiseActiveUi(s); + } + ui.meter.style.width = `${levelFromRms(rms) * 100}%`; const open = s.ready && gateOpen(s, rms); ui.meter.classList.toggle("is-open", open); @@ -808,6 +1214,7 @@ const s = { nickname, target: targetChannel, ws: null, ctx: null, stream: null, worklet: null, playback: null, output: null, ready: false, muted: false, deafened: false, pttDown: false, vadUntil: 0, wasTalking: false, + rnnoiseLoaded: false, rnnoiseLoading: false, rnnoiseWorkletReady: false, rnnoiseFallback: null, selfId: 0, snapshot: null, talking: new Set(), talkTimers: new Map(), userVolumes: new Map(), chatByChannel: new Map(), dmByUser: new Map(), activeChatTab: { type: "channel" }, chatDrafts: new Map(), @@ -821,19 +1228,51 @@ // The AudioContext must be created inside the click handler's turn. s.ctx = new AudioContext({ latencyHint: "interactive" }); const resume = s.ctx.resume(); + const wasmPreload = rnnoiseEnabled ? fetchRnnoiseWasm().catch(() => null) : Promise.resolve(null); + // Browser NS starts on; applyBrowserNoiseSuppression() turns it off once + // RNNoise is confirmed to be running. s.stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true, channelCount: 1 }, }); + s.browserNs = true; await resume; - await s.ctx.audioWorklet.addModule("/static/mic-worklet.js"); + // Versioned URL so browsers never run a stale cached worklet without RNNoise support. + await s.ctx.audioWorklet.addModule(`/static/mic-worklet.js?v=${MIC_WORKLET_VERSION}`); + const wasmBinary = await wasmPreload; if (s.closed) return; const source = s.ctx.createMediaStreamSource(s.stream); - s.worklet = new AudioWorkletNode(s.ctx, "ts6-mic-capture", { numberOfInputs: 1, numberOfOutputs: 1, outputChannelCount: [1] }); + s.worklet = new AudioWorkletNode(s.ctx, "ts6-mic-capture", { + numberOfInputs: 1, + numberOfOutputs: 1, + outputChannelCount: [1], + processorOptions: { + rnnoiseEnabled, + wasmBinary: wasmBinary ? wasmBinary.slice(0) : null, + }, + }); + if (wasmBinary) { + s.rnnoiseLoaded = true; + createMainThreadRnnoise(wasmBinary) + .then((fallback) => { + if (s.closed || session !== s) fallback.destroy(); + else { + s.rnnoiseFallback = fallback; + applyBrowserNoiseSuppression(s); + updateRnnoiseActiveUi(s); + } + }) + .catch((err) => { + s.rnnoiseError = err && err.message ? err.message : String(err); + updateRnnoiseActiveUi(s); + }); + } else if (rnnoiseEnabled) { + void ensureSessionRnnoise(s); + } const sink = s.ctx.createGain(); sink.gain.value = 0; source.connect(s.worklet).connect(sink).connect(s.ctx.destination); - s.worklet.port.onmessage = (e) => onMicFrame(s, e.data); + s.worklet.port.onmessage = (e) => onMicMessage(s, e.data); s.output = s.ctx.createGain(); s.output.gain.value = Number(ui.volume.value) / 100; @@ -958,7 +1397,15 @@ } catch { /* closed */ } } if (s.stream) for (const track of s.stream.getTracks()) track.stop(); - if (s.worklet) s.worklet.port.onmessage = null; + if (s.worklet) { + try { s.worklet.port.postMessage({ type: "dispose" }); } catch { /* ignore */ } + s.worklet.port.onmessage = null; + } + if (s.rnnoiseFallback) { + s.rnnoiseFallback.destroy(); + s.rnnoiseFallback = null; + } + updateRnnoiseActiveUi(null); if (s.playback) s.playback.close(); if (s.ctx) s.ctx.close().catch(() => undefined); @@ -1116,6 +1563,30 @@ applyModeUi(); }); + function toggleRnnoise() { + rnnoiseEnabled = !rnnoiseEnabled; + store.set(STORE.rnnoise, rnnoiseEnabled ? "1" : "0"); + applyRnnoiseUi(); + if (session) { + if (rnnoiseEnabled && !session.rnnoiseLoaded) { + void ensureSessionRnnoise(session); + } + if (rnnoiseEnabled && session.rnnoiseFallback) { + session.rnnoiseFallback.reset(); + } + if (session.worklet) { + try { + session.worklet.port.postMessage({ type: "rnnoise-enable", enabled: rnnoiseEnabled }); + } catch { /* ignore */ } + } + applyBrowserNoiseSuppression(session); + updateRnnoiseActiveUi(session); + } + } + if (ui.rnnoiseToggle) ui.rnnoiseToggle.addEventListener("click", toggleRnnoise); + if (ui.wasmRedownload) ui.wasmRedownload.addEventListener("click", () => void redownloadRnnoiseWasm()); + if (ui.wasmDelete) ui.wasmDelete.addEventListener("click", () => void deleteRnnoiseWasm()); + ui.sensitivity.addEventListener("input", () => { store.set(STORE.sensitivity, ui.sensitivity.value); updateThresholdMark(); |
