diff options
Diffstat (limited to 'static/voice.js')
| -rw-r--r-- | static/voice.js | 485 |
1 files changed, 478 insertions, 7 deletions
diff --git a/static/voice.js b/static/voice.js index 9cfa74b..6116b99 100644 --- a/static/voice.js +++ b/static/voice.js @@ -44,6 +44,15 @@ threshold: $("voice-threshold"), mute: $("voice-mute"), deafen: $("voice-deafen"), + wasmOpenBtn: $("voice-wasm-open-btn"), + wasmDialog: $("voice-wasm-dialog"), + wasmClose: $("voice-wasm-close"), + wasmMessage: $("voice-wasm-message"), + rnnoiseToggle: $("voice-rnnoise-toggle"), + wasmRedownload: $("voice-wasm-redownload"), + wasmDelete: $("voice-wasm-delete"), + wasmStatus: $("voice-wasm-status"), + wasmActive: $("voice-wasm-active"), settingsToggle: $("voice-settings-toggle"), settings: $("voice-settings"), mode: $("voice-mode"), @@ -61,6 +70,7 @@ sensitivity: "ts6voice.sensitivity", volume: "ts6voice.volume", mode: "ts6voice.mode", + rnnoise: "ts6voice.rnnoise", userVolumes: "ts6voice.userVolumes", }; const store = { @@ -76,6 +86,7 @@ const MAX_AHEAD_S = 0.25; const MAX_SEND_BUFFER = 16 * 1024; const MAX_CHAT_HISTORY = 100; + const MIC_WORKLET_VERSION = "rnnoise-2"; let session = null; // active session, see startJoin() let pendingJoin = null; // undefined = save only; null = join default channel; {cid,name} = join specific channel @@ -199,6 +210,22 @@ if (e.target === ui.dialog && (!session || session.ready)) closeLoginModal(); }); + if (ui.wasmOpenBtn && ui.wasmDialog) { + ui.wasmOpenBtn.addEventListener("click", () => { + setText(ui.wasmMessage, ""); + updateRnnoiseActiveUi(session); + if (!ui.wasmDialog.open) ui.wasmDialog.showModal(); + }); + if (ui.wasmClose) { + ui.wasmClose.addEventListener("click", () => { + if (ui.wasmDialog.open) ui.wasmDialog.close(); + }); + } + ui.wasmDialog.addEventListener("click", (e) => { + if (e.target === ui.wasmDialog) ui.wasmDialog.close(); + }); + } + // ---- preferences -------------------------------------------------------------- ui.nickname.value = store.get(STORE.nickname) || ""; @@ -206,6 +233,7 @@ if (store.get(STORE.sensitivity) !== null) ui.sensitivity.value = store.get(STORE.sensitivity); if (store.get(STORE.volume) !== null) ui.volume.value = store.get(STORE.volume); let mode = store.get(STORE.mode) === "ptt" ? "ptt" : "vad"; + let rnnoiseEnabled = store.get(STORE.rnnoise) !== "0"; updateTopbarSummary(); ui.nickname.addEventListener("input", () => { @@ -230,8 +258,17 @@ ui.ptt.hidden = mode !== "ptt"; ui.threshold.hidden = mode === "ptt"; } + function applyRnnoiseUi() { + const label = rnnoiseEnabled ? "rnnoise: on" : "rnnoise: off"; + for (const btn of [ui.wasmOpenBtn, ui.rnnoiseToggle]) { + if (!btn) continue; + btn.setAttribute("aria-pressed", String(rnnoiseEnabled)); + btn.textContent = label; + } + } updateThresholdMark(); applyModeUi(); + applyRnnoiseUi(); // ---- live channel list & chat (channel + individual DMs) -------------------- @@ -749,7 +786,304 @@ } } - // ---- microphone --------------------------------------------------------------- + // ---- microphone & RNNoise (WASM) ---------------------------------------------- + + const RNNOISE_WASM_URLS = [ + "https://cdn.jsdelivr.net/npm/@jitsi/rnnoise-wasm@0.2.1/dist/rnnoise.wasm", + "https://unpkg.com/@jitsi/rnnoise-wasm@0.2.1/dist/rnnoise.wasm", + ]; + const WASM_CACHE_NAME = "ts6voice-rnnoise-v1"; + const WASM_CACHE_KEY = "https://ts6voice.local/rnnoise.wasm"; + const RNNOISE_FRAME_SAMPLES = 480; + const PCM16_SCALE = 32768; + let rnnoiseWasmBuffer = null; + let rnnoiseWasmPromise = null; + + function setWasmStatus(text, state = "idle") { + if (!ui.wasmStatus) return; + ui.wasmStatus.textContent = text; + ui.wasmStatus.dataset.state = state; + } + + function setWasmButtonsBusy(busy) { + for (const btn of [ui.wasmRedownload, ui.wasmDelete]) { + if (btn) btn.disabled = busy; + } + } + + function sayWasm(text, kind = "info") { + if (ui.wasmDialog && ui.wasmDialog.open) setText(ui.wasmMessage, text, kind); + else say(text, kind); + } + + function formatKb(byteLength) { + return `${Math.max(1, Math.round(byteLength / 1024))} KB`; + } + + async function readWasmFromCache() { + if (typeof caches === "undefined") return null; + try { + const cache = await caches.open(WASM_CACHE_NAME); + const match = await cache.match(WASM_CACHE_KEY); + if (!match) return null; + const buf = await match.arrayBuffer(); + return buf && buf.byteLength > 0 ? buf : null; + } catch { + return null; + } + } + + async function writeWasmToCache(buf) { + if (typeof caches === "undefined" || !buf) return; + try { + const cache = await caches.open(WASM_CACHE_NAME); + await cache.put( + WASM_CACHE_KEY, + new Response(buf.slice(0), { headers: { "content-type": "application/wasm" } }) + ); + } catch { /* ignore quota or private mode errors */ } + } + + async function clearWasmCache() { + if (typeof caches === "undefined") return; + try { + await caches.delete(WASM_CACHE_NAME); + } catch { /* ignore */ } + } + + function fetchRnnoiseWasm({ forceReload = false } = {}) { + if (!forceReload && rnnoiseWasmBuffer) { + return Promise.resolve(rnnoiseWasmBuffer); + } + if (!forceReload && rnnoiseWasmPromise) { + return rnnoiseWasmPromise; + } + setWasmButtonsBusy(true); + setWasmStatus("downloading...", "loading"); + + rnnoiseWasmPromise = (async () => { + if (!forceReload) { + const cached = await readWasmFromCache(); + if (cached) { + rnnoiseWasmBuffer = cached; + setWasmStatus(`cached (${formatKb(cached.byteLength)})`, "cached"); + return cached; + } + } else { + await clearWasmCache(); + } + + let lastError = null; + for (const url of RNNOISE_WASM_URLS) { + try { + const res = await fetch(url, { + mode: "cors", + credentials: "omit", + cache: forceReload ? "reload" : "default", + }); + if (!res.ok) throw new Error(`HTTP ${res.status} from ${url}`); + const buf = await res.arrayBuffer(); + if (!buf || buf.byteLength === 0) throw new Error("Empty WASM response"); + rnnoiseWasmBuffer = buf; + await writeWasmToCache(buf); + setWasmStatus(`cached (${formatKb(buf.byteLength)})`, "cached"); + return buf; + } catch (err) { + lastError = err; + } + } + throw lastError || new Error("Failed to download rnnoise.wasm"); + })() + .catch((err) => { + rnnoiseWasmBuffer = null; + rnnoiseWasmPromise = null; + setWasmStatus("download failed", "error"); + throw err; + }) + .finally(() => { + setWasmButtonsBusy(false); + }); + + return rnnoiseWasmPromise; + } + + async function deleteRnnoiseWasm() { + setWasmButtonsBusy(true); + try { + await clearWasmCache(); + rnnoiseWasmBuffer = null; + rnnoiseWasmPromise = null; + // Keep the user's on/off preference; with no model cached, RNNoise simply + // stays inactive until the model is downloaded again. + + if (session) { + session.rnnoiseLoaded = false; + session.rnnoiseWorkletReady = false; + if (session.rnnoiseFallback) { + session.rnnoiseFallback.destroy(); + session.rnnoiseFallback = null; + } + if (session.worklet) { + try { session.worklet.port.postMessage({ type: "rnnoise-clear" }); } catch { /* ignore */ } + } + applyBrowserNoiseSuppression(session); + updateRnnoiseActiveUi(session); + } + setWasmStatus("deleted (not cached)", "idle"); + sayWasm("RNNoise WASM model deleted from browser storage.", "info"); + } finally { + setWasmButtonsBusy(false); + } + } + + async function redownloadRnnoiseWasm() { + try { + setText(ui.wasmMessage, ""); + rnnoiseWasmBuffer = null; + rnnoiseWasmPromise = null; + const wasmBinary = await fetchRnnoiseWasm({ forceReload: true }); + if (session && !session.closed) { + session.rnnoiseLoaded = true; + if (session.worklet) { + session.worklet.port.postMessage({ + type: "rnnoise-init", + wasmBinary: wasmBinary.slice(0), + enabled: rnnoiseEnabled, + }); + } + const nextFallback = await createMainThreadRnnoise(wasmBinary); + if (session && !session.closed) { + if (session.rnnoiseFallback) session.rnnoiseFallback.destroy(); + session.rnnoiseFallback = nextFallback; + applyBrowserNoiseSuppression(session); + } else { + nextFallback.destroy(); + } + } + sayWasm("RNNoise WASM model downloaded and ready.", "info"); + } catch (err) { + sayWasm(`Failed to re-download RNNoise WASM: ${err && err.message ? err.message : err}`, "error"); + } + } + + void readWasmFromCache().then((cached) => { + if (cached) { + rnnoiseWasmBuffer = cached; + setWasmStatus(`cached (${formatKb(cached.byteLength)})`, "cached"); + } else { + setWasmStatus("not cached", "idle"); + } + }); + + async function createMainThreadRnnoise(wasmBinary) { + let memory = null; + const resizeHeap = (requestedSize) => { + if (!memory) return 0; + const pages = ((requestedSize >>> 0) - memory.buffer.byteLength + 65535) >>> 16; + try { memory.grow(pages); return 1; } catch { return 0; } + }; + const memcpyBig = (dest, src, num) => { + if (!memory) return; + new Uint8Array(memory.buffer).copyWithin(dest >>> 0, src >>> 0, (src + num) >>> 0); + }; + const res = await WebAssembly.instantiate(wasmBinary, { + a: { a: resizeHeap, b: memcpyBig }, + env: { + emscripten_resize_heap: resizeHeap, + emscripten_memcpy_big: memcpyBig, + __assert_fail: () => { throw new Error("RNNoise WASM assertion failed"); }, + }, + }); + const inst = res.instance || res; + const exp = inst.exports; + memory = exp.memory || exp.c; + if (typeof exp.emscripten_stack_init === "function") exp.emscripten_stack_init(); + const initRuntime = exp.__wasm_call_ctors || exp.d; + if (typeof initRuntime === "function") initRuntime(); + + const rnnoiseInit = exp.rnnoise_init || exp._rnnoise_init || exp.e; + const rnnoiseCreate = exp.rnnoise_create || exp._rnnoise_create || exp.f; + const malloc = exp.malloc || exp._malloc || exp.g; + const rnnoiseDestroy = exp.rnnoise_destroy || exp._rnnoise_destroy || exp.h; + const free = exp.free || exp._free || exp.i; + const rnnoiseProcessFrame = exp.rnnoise_process_frame || exp._rnnoise_process_frame || exp.j; + + const state = rnnoiseCreate(0); + const ptrIn = malloc(RNNOISE_FRAME_SAMPLES * 4); + const ptrOut = malloc(RNNOISE_FRAME_SAMPLES * 4); + if (!state || !ptrIn || !ptrOut) { + if (state) rnnoiseDestroy(state); + if (ptrIn) free(ptrIn); + if (ptrOut) free(ptrOut); + throw new Error("Failed to allocate RNNoise WASM buffers"); + } + + let heapF32 = new Float32Array(memory.buffer); + let destroyed = false; + + const processSubFrame = (samples, offset) => { + if (heapF32.buffer !== memory.buffer) heapF32 = new Float32Array(memory.buffer); + const inBase = ptrIn >> 2; + const outBase = ptrOut >> 2; + for (let i = 0; i < RNNOISE_FRAME_SAMPLES; i++) { + heapF32[inBase + i] = samples[offset + i] * PCM16_SCALE; + } + rnnoiseProcessFrame(state, ptrOut, ptrIn); + if (heapF32.buffer !== memory.buffer) heapF32 = new Float32Array(memory.buffer); + for (let i = 0; i < RNNOISE_FRAME_SAMPLES; i++) { + const v = heapF32[outBase + i] / PCM16_SCALE; + samples[offset + i] = v < -1 ? -1 : v > 1 ? 1 : v; + } + }; + + return { + processFrame960(samples) { + if (destroyed || samples.length < RNNOISE_FRAME_SAMPLES * 2) return; + processSubFrame(samples, 0); + processSubFrame(samples, RNNOISE_FRAME_SAMPLES); + }, + reset() { + if (!destroyed && typeof rnnoiseInit === "function") rnnoiseInit(state, 0); + }, + destroy() { + if (destroyed) return; + destroyed = true; + try { rnnoiseDestroy(state); } catch { /* ignore */ } + try { free(ptrIn); } catch { /* ignore */ } + try { free(ptrOut); } catch { /* ignore */ } + }, + }; + } + + async function ensureSessionRnnoise(s) { + if (!s || s.closed || s.rnnoiseLoading || s.rnnoiseLoaded) return; + s.rnnoiseLoading = true; + try { + const wasmBinary = await fetchRnnoiseWasm(); + if (s.closed || session !== s) return; + s.rnnoiseLoaded = true; + if (s.worklet) { + s.worklet.port.postMessage({ + type: "rnnoise-init", + wasmBinary: wasmBinary.slice(0), + enabled: rnnoiseEnabled, + }); + } + s.rnnoiseFallback = await createMainThreadRnnoise(wasmBinary); + if (s.closed || session !== s) { + s.rnnoiseFallback.destroy(); + s.rnnoiseFallback = null; + } else { + applyBrowserNoiseSuppression(s); + } + } catch (err) { + if (session === s && rnnoiseEnabled) { + say(`RNNoise WASM failed to load: ${err && err.message ? err.message : err}`, "error"); + } + } finally { + s.rnnoiseLoading = false; + } + } function gateOpen(s, rms) { if (s.muted) return false; @@ -759,8 +1093,80 @@ return now < s.vadUntil; } - function onMicFrame(s, { samples, rms }) { - if (session !== s) return; + /** True once a working RNNoise instance exists (worklet or main thread). */ + function rnnoiseAvailable(s) { + return Boolean(s && (s.rnnoiseWorkletReady || s.rnnoiseFallback)); + } + + /** + * The browser's built-in noise suppression runs before our pipeline and + * already removes most steady background noise, which makes RNNoise look + * like it does nothing. Turn it off while RNNoise is actually processing, + * and back on otherwise so users are never left without any suppression. + */ + function applyBrowserNoiseSuppression(s) { + if (!s || !s.stream) return; + const wantBrowserNs = !(rnnoiseEnabled && rnnoiseAvailable(s)); + if (s.browserNs === wantBrowserNs) return; + s.browserNs = wantBrowserNs; + for (const track of s.stream.getAudioTracks()) { + track.applyConstraints({ noiseSuppression: wantBrowserNs }).catch(() => undefined); + } + } + + function updateRnnoiseActiveUi(s) { + if (!ui.wasmActive) return; + let text = "not in a call"; + let state = "idle"; + if (s && !s.closed) { + if (!rnnoiseEnabled) { + text = "off (browser noise suppression)"; + } else if (s.rnnoiseVia) { + text = `active (${s.rnnoiseVia})`; + state = "cached"; + } else if (s.rnnoiseError) { + text = `failed: ${s.rnnoiseError}`; + state = "error"; + } else { + text = "waiting for model..."; + state = "loading"; + } + } + if (ui.wasmActive.textContent !== text) ui.wasmActive.textContent = text; + ui.wasmActive.dataset.state = state; + } + + function onMicMessage(s, data) { + if (session !== s || !data) return; + if (data.type === "rnnoise-status") { + s.rnnoiseWorkletReady = Boolean(data.ready); + if (data.error) { + s.rnnoiseError = data.error; + console.warn("[rnnoise] worklet init failed, using main-thread fallback:", data.error); + } + applyBrowserNoiseSuppression(s); + updateRnnoiseActiveUi(s); + return; + } + if (!data.samples) return; + + const samples = data.samples; + let rms = data.rms; + let via = data.denoised ? "audio worklet" : ""; + // If RNNoise is enabled and the worklet did not denoise this frame yet, + // run the main-thread WASM denoiser before metering, VAD gating, and gateway send. + if (rnnoiseEnabled && !data.denoised && s.rnnoiseFallback) { + s.rnnoiseFallback.processFrame960(samples); + let sum = 0; + for (let i = 0; i < samples.length; i++) sum += samples[i] * samples[i]; + rms = Math.sqrt(sum / samples.length); + via = "main thread"; + } + if (s.rnnoiseVia !== via) { + s.rnnoiseVia = via; + updateRnnoiseActiveUi(s); + } + ui.meter.style.width = `${levelFromRms(rms) * 100}%`; const open = s.ready && gateOpen(s, rms); ui.meter.classList.toggle("is-open", open); @@ -808,6 +1214,7 @@ const s = { nickname, target: targetChannel, ws: null, ctx: null, stream: null, worklet: null, playback: null, output: null, ready: false, muted: false, deafened: false, pttDown: false, vadUntil: 0, wasTalking: false, + rnnoiseLoaded: false, rnnoiseLoading: false, rnnoiseWorkletReady: false, rnnoiseFallback: null, selfId: 0, snapshot: null, talking: new Set(), talkTimers: new Map(), userVolumes: new Map(), chatByChannel: new Map(), dmByUser: new Map(), activeChatTab: { type: "channel" }, chatDrafts: new Map(), @@ -821,19 +1228,51 @@ // The AudioContext must be created inside the click handler's turn. s.ctx = new AudioContext({ latencyHint: "interactive" }); const resume = s.ctx.resume(); + const wasmPreload = rnnoiseEnabled ? fetchRnnoiseWasm().catch(() => null) : Promise.resolve(null); + // Browser NS starts on; applyBrowserNoiseSuppression() turns it off once + // RNNoise is confirmed to be running. s.stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true, channelCount: 1 }, }); + s.browserNs = true; await resume; - await s.ctx.audioWorklet.addModule("/static/mic-worklet.js"); + // Versioned URL so browsers never run a stale cached worklet without RNNoise support. + await s.ctx.audioWorklet.addModule(`/static/mic-worklet.js?v=${MIC_WORKLET_VERSION}`); + const wasmBinary = await wasmPreload; if (s.closed) return; const source = s.ctx.createMediaStreamSource(s.stream); - s.worklet = new AudioWorkletNode(s.ctx, "ts6-mic-capture", { numberOfInputs: 1, numberOfOutputs: 1, outputChannelCount: [1] }); + s.worklet = new AudioWorkletNode(s.ctx, "ts6-mic-capture", { + numberOfInputs: 1, + numberOfOutputs: 1, + outputChannelCount: [1], + processorOptions: { + rnnoiseEnabled, + wasmBinary: wasmBinary ? wasmBinary.slice(0) : null, + }, + }); + if (wasmBinary) { + s.rnnoiseLoaded = true; + createMainThreadRnnoise(wasmBinary) + .then((fallback) => { + if (s.closed || session !== s) fallback.destroy(); + else { + s.rnnoiseFallback = fallback; + applyBrowserNoiseSuppression(s); + updateRnnoiseActiveUi(s); + } + }) + .catch((err) => { + s.rnnoiseError = err && err.message ? err.message : String(err); + updateRnnoiseActiveUi(s); + }); + } else if (rnnoiseEnabled) { + void ensureSessionRnnoise(s); + } const sink = s.ctx.createGain(); sink.gain.value = 0; source.connect(s.worklet).connect(sink).connect(s.ctx.destination); - s.worklet.port.onmessage = (e) => onMicFrame(s, e.data); + s.worklet.port.onmessage = (e) => onMicMessage(s, e.data); s.output = s.ctx.createGain(); s.output.gain.value = Number(ui.volume.value) / 100; @@ -958,7 +1397,15 @@ } catch { /* closed */ } } if (s.stream) for (const track of s.stream.getTracks()) track.stop(); - if (s.worklet) s.worklet.port.onmessage = null; + if (s.worklet) { + try { s.worklet.port.postMessage({ type: "dispose" }); } catch { /* ignore */ } + s.worklet.port.onmessage = null; + } + if (s.rnnoiseFallback) { + s.rnnoiseFallback.destroy(); + s.rnnoiseFallback = null; + } + updateRnnoiseActiveUi(null); if (s.playback) s.playback.close(); if (s.ctx) s.ctx.close().catch(() => undefined); @@ -1116,6 +1563,30 @@ applyModeUi(); }); + function toggleRnnoise() { + rnnoiseEnabled = !rnnoiseEnabled; + store.set(STORE.rnnoise, rnnoiseEnabled ? "1" : "0"); + applyRnnoiseUi(); + if (session) { + if (rnnoiseEnabled && !session.rnnoiseLoaded) { + void ensureSessionRnnoise(session); + } + if (rnnoiseEnabled && session.rnnoiseFallback) { + session.rnnoiseFallback.reset(); + } + if (session.worklet) { + try { + session.worklet.port.postMessage({ type: "rnnoise-enable", enabled: rnnoiseEnabled }); + } catch { /* ignore */ } + } + applyBrowserNoiseSuppression(session); + updateRnnoiseActiveUi(session); + } + } + if (ui.rnnoiseToggle) ui.rnnoiseToggle.addEventListener("click", toggleRnnoise); + if (ui.wasmRedownload) ui.wasmRedownload.addEventListener("click", () => void redownloadRnnoiseWasm()); + if (ui.wasmDelete) ui.wasmDelete.addEventListener("click", () => void deleteRnnoiseWasm()); + ui.sensitivity.addEventListener("input", () => { store.set(STORE.sensitivity, ui.sensitivity.value); updateThresholdMark(); |