feat: move to virtual sink and loopback audio device as default methodHEADmain

- creates a virtual audio sink on startup and a loopback
- enables listening on loopback
- provide finegrain control over what you want the AI to hear

1 files changed, 115 insertions, 42 deletions

diff --git a/server.py b/server.py
index fefe61e..fa12783 100644
--- a/server.py
+++ b/server.py
@@ -3,6 +3,7 @@ import threading
 import json
 import queue
 import os
+import atexit
 from collections import Counter, deque
 import re
 from typing import Any, Deque, Dict, Optional, Set, List, Iterator, Callable
@@ -20,6 +21,7 @@ from gui.gui import select_settings, prompt_input_sample_rate, run_runtime_dashb
 from openai_realtime import OpenAIRealtimeTranslator
 from routes import register_routes
 from config import _SYSTEM_PROMPT, _LLM_EMPTY_SENTINELS, _HALLUCINATION_PHRASES
+import linux_audio
 
 TARGET_SAMPLE_RATE: int = 16000
 CAPTURE_SAMPLE_RATE: int = 0
@@ -32,6 +34,7 @@ SSE_KEEPALIVE_SECONDS: int = 15
 RUNTIME_SUBTITLE_LINES_MAX: int = 120
 RUNTIME_LOG_LINES_MAX: int = 300
 
+
 USE_OLLAMA_CLEANUP: bool = True
 USE_OPENAI_REALTIME_TRANSLATE: bool = False
 OLLAMA_MODEL: str = "qwen2.5:7b-instruct"
@@ -113,6 +116,8 @@ _raw_batch_lock: threading.Lock = threading.Lock()
 
 openai_realtime_client: Optional[OpenAIRealtimeTranslator] = None
 
+_virtual_sink_cleanup_registered: bool = False
+
 def resample_audio(audio_np: np.ndarray, src_rate: int, dst_rate: int) -> np.ndarray:
     """
     Resamples audio to TARGET_SAMPLE_RATE (default is 16000hz), speeds up inference time, fetched as a nd array
@@ -131,6 +136,7 @@ def resample_audio(audio_np: np.ndarray, src_rate: int, dst_rate: int) -> np.nda
 
 
 
+
 def load_settings() -> Dict[str, Any]:
     if not os.path.exists(SETTINGS_PATH):
         return DEFAULT_SETTINGS.copy()
@@ -174,7 +180,7 @@ def cleanup_subtitle_with_ollama(raw_text: str, context: List[str]) -> Optional[
             ],
             options=OLLAMA_OPTIONS,
         )
-        return response.message.content.strip()
+        return (response.message.content or "").strip()
     except Exception as exc:
         print(f"⚠️  OLLAMA cleanup error: {exc}")
         return None
@@ -197,7 +203,7 @@ def ensure_ollama_ready(status_callback: Optional[Callable[[str], None]] = None)
             f"Cannot reach Ollama — is the server running?  ({exc})"
         ) from exc
 
-    model_names: List[str] = [m.model for m in local.models]
+    model_names: List[str] = [str(m.model) for m in local.models if m.model]
     if not any(name.startswith(OLLAMA_MODEL) for name in model_names):
         report(f"Model '{OLLAMA_MODEL}' not found locally. Pulling now (this can take a while)...")
         try:
@@ -478,14 +484,7 @@ def publish_audio_activity(chunk_rms: float) -> None:
     broadcast_event(SSE_EVENT_AUDIO_ACTIVITY, payload)
 
 
-def audio_callback(indata: np.ndarray, frames: int, time_info: Any, status: Any) -> None:
-    """
-    Callback definition for audio sink. Sends audio to local Whisper buffer or OpenAI realtime queue.
-    """
-    if status:
-        print(f"Audio status: {status}")
-    # Take first channel
-    chunk: np.ndarray = indata[:, 0].copy()
+def _handle_audio_chunk(chunk: np.ndarray) -> None:
     chunk_rms: float = float(np.sqrt(np.mean(np.square(chunk)))) if len(chunk) > 0 else 0.0
     publish_audio_activity(chunk_rms)
 
@@ -501,6 +500,18 @@ def audio_callback(indata: np.ndarray, frames: int, time_info: Any, status: Any)
             audio_buffer = audio_buffer[-MAX_SAMPLES:]
 
 
+def audio_callback(indata: np.ndarray, frames: int, time_info: Any, status: Any) -> None:
+    """
+    Callback definition for audio sink. Sends audio to local Whisper buffer or OpenAI realtime queue.
+    """
+    if status:
+        print(f"Audio status: {status}")
+    if indata is None or len(indata) == 0:
+        return
+    chunk: np.ndarray = indata[:, 0].copy()
+    _handle_audio_chunk(chunk)
+
+
 def is_silent(audio_16k: Optional[np.ndarray]) -> bool:
     """
     Basic rudimentary silence detection, do not run whisper if rms value isn't reached
@@ -547,19 +558,31 @@ def select_input_sample_rate(device_index: int, preferred_rate: int) -> int:
     return prompt_input_sample_rate(device_index, common_rates)
 
 
+
 def main() -> None:
     global CAPTURE_SAMPLE_RATE, MAX_SAMPLES, model, WHISPER_TASK, WHISPER_BEAM_SIZE, WHISPER_LANGUAGE
     global BUFFER_SECONDS, PROCESS_INTERVAL_SECONDS, USE_OLLAMA_CLEANUP, USE_OPENAI_REALTIME_TRANSLATE
     global OLLAMA_MODEL, OLLAMA_CONTEXT_WINDOW, RAW_BATCH_SIZE, subtitle_context
     global AUDIO_ACTIVITY_THRESHOLD, last_audio_activity_payload, _audio_active_until, _audio_last_emit
     global OPENAI_API_KEY, OPENAI_OUTPUT_LANGUAGE, OPENAI_REALTIME_MODEL, OPENAI_SAFETY_IDENTIFIER
-    global openai_realtime_client
+    global openai_realtime_client, _virtual_sink_cleanup_registered
 
     start_subtitle_server()
 
+    linux_audio.ensure_virtual_input_sink()
+    if linux_audio.is_linux() and not _virtual_sink_cleanup_registered:
+        atexit.register(linux_audio.teardown_virtual_input_sink)
+        _virtual_sink_cleanup_registered = True
+
     settings: Dict[str, Any] = load_settings()
     devices = sd.query_devices()
     input_devices = [(idx, dev) for idx, dev in enumerate(devices) if dev["max_input_channels"] > 0]
+    if linux_audio.is_linux():
+        has_virtual_input = any(
+            linux_audio.is_virtual_input_name(str(dev.get("name", "")))
+            for _idx, dev in input_devices
+        )
+        linux_audio.append_pipewire_direct_device_option(input_devices, has_virtual_input)
     settings = select_settings(
         settings,
         input_devices,
@@ -597,14 +620,20 @@ def main() -> None:
         llm_thread.start()
 
     device_name: str = settings.get("audio_device_name", "")
+    use_pipewire_direct_capture = (
+        linux_audio.is_linux()
+        and device_name == linux_audio.LINUX_VIRTUAL_DIRECT_DEVICE_NAME
+    )
+
     matched_index: Optional[int] = None
-    for idx, dev in enumerate(devices):
-        if dev.get("name") == device_name and dev.get("max_input_channels", 0) > 0:
-            matched_index = idx
-            break
-    if matched_index is None:
-        raise RuntimeError("Saved audio device not found. Please reselect in the settings window.")
-    device_index: int = matched_index
+    if not use_pipewire_direct_capture:
+        for idx, dev in enumerate(devices):
+            if dev.get("name") == device_name and dev.get("max_input_channels", 0) > 0:
+                matched_index = idx
+                break
+        if matched_index is None:
+            raise RuntimeError("Saved audio device not found. Please reselect in the settings window.")
+    device_index: Optional[int] = matched_index
 
     model_name: str = settings["model_name"]
     whisper_device: str = settings["device"]
@@ -631,11 +660,20 @@ def main() -> None:
     _audio_last_emit = 0.0
     broadcast_event(SSE_EVENT_AUDIO_ACTIVITY, last_audio_activity_payload)
 
-    device_info = sd.query_devices(device_index)
-    preferred_rate: int = int(device_info["default_samplerate"])
-    if preferred_rate <= 0:
-        preferred_rate = 48000
-    CAPTURE_SAMPLE_RATE = select_input_sample_rate(device_index, preferred_rate)
+    if use_pipewire_direct_capture:
+        device_info = {
+            "name": linux_audio.LINUX_VIRTUAL_DIRECT_DEVICE_NAME,
+            "default_samplerate": float(linux_audio.LINUX_VIRTUAL_DIRECT_CAPTURE_RATE),
+        }
+        CAPTURE_SAMPLE_RATE = linux_audio.LINUX_VIRTUAL_DIRECT_CAPTURE_RATE
+    else:
+        assert device_index is not None
+        device_info = sd.query_devices(device_index)
+        preferred_rate: int = int(device_info["default_samplerate"])
+        if preferred_rate <= 0:
+            preferred_rate = 48000
+        CAPTURE_SAMPLE_RATE = select_input_sample_rate(device_index, preferred_rate)
+
     MAX_SAMPLES = int(CAPTURE_SAMPLE_RATE * BUFFER_SECONDS)
 
     with recent_subtitle_lines_lock:
@@ -660,7 +698,10 @@ def main() -> None:
         )
         openai_realtime_client.start()
 
-        print(f"Using device {device_index}: {device_info['name']}")
+        if use_pipewire_direct_capture:
+            print(f"Using Linux PipeWire direct input: {device_info['name']}")
+        else:
+            print(f"Using device {device_index}: {device_info['name']}")
         print(f"Realtime translation backend: OpenAI ({OPENAI_REALTIME_MODEL})")
         print(
             f"Capture sample rate: {CAPTURE_SAMPLE_RATE} Hz "
@@ -674,7 +715,10 @@ def main() -> None:
         openai_realtime_client = None
         model = WhisperModel(model_name, device=whisper_device, compute_type=compute_type)
 
-        print(f"Using device {device_index}: {device_info['name']}")
+        if use_pipewire_direct_capture:
+            print(f"Using Linux PipeWire direct input: {device_info['name']}")
+        else:
+            print(f"Using device {device_index}: {device_info['name']}")
         print(f"Model: {model_name} | task={WHISPER_TASK} | beam_size={WHISPER_BEAM_SIZE}")
         print(f"Compute: device={whisper_device} | compute_type={compute_type}")
         print(f"Capture sample rate: {CAPTURE_SAMPLE_RATE} Hz (resampling to {TARGET_SAMPLE_RATE} Hz)")
@@ -686,20 +730,41 @@ def main() -> None:
         processing_thread = threading.Thread(target=processing_loop, daemon=True)
         processing_thread.start()
 
-    stream = sd.InputStream(
-        device=device_index,
-        channels=1,
-        samplerate=CAPTURE_SAMPLE_RATE,
-        dtype="float32",
-        callback=audio_callback,
-        blocksize=int(CAPTURE_SAMPLE_RATE * 0.5),
-    )
+    stream: Optional[sd.InputStream] = None
+    pipewire_stop_event: Optional[threading.Event] = None
+    pipewire_capture_thread: Optional[threading.Thread] = None
+
+    if use_pipewire_direct_capture:
+        if not linux_audio.has_pipewire_direct_backend(CAPTURE_SAMPLE_RATE):
+            raise RuntimeError(
+                "PipeWire direct virtual input was selected, but 'ffmpeg' is not installed. "
+                "Install ffmpeg and try again."
+            )
+        pipewire_stop_event = threading.Event()
+        pipewire_capture_thread = threading.Thread(
+            target=linux_audio.pipewire_direct_capture_worker,
+            args=(pipewire_stop_event, CAPTURE_SAMPLE_RATE, _handle_audio_chunk, add_runtime_log),
+            daemon=True,
+        )
+    else:
+        assert device_index is not None
+        stream = sd.InputStream(
+            device=device_index,
+            channels=1,
+            samplerate=CAPTURE_SAMPLE_RATE,
+            dtype="float32",
+            callback=audio_callback,
+            blocksize=int(CAPTURE_SAMPLE_RATE * 0.5),
+        )
 
     def _on_dashboard_close() -> None:
         print("Stopping.")
 
     try:
-        stream.start()
+        if pipewire_capture_thread is not None:
+            pipewire_capture_thread.start()
+        elif stream is not None:
+            stream.start()
         print("Listening... Close the runtime window to stop.")
         run_runtime_dashboard(
             get_audio_activity=get_audio_activity_snapshot,
@@ -712,14 +777,22 @@ def main() -> None:
             openai_realtime_client.stop()
             openai_realtime_client = None
 
-        try:
-            stream.stop()
-        except Exception:
-            pass
-        try:
-            stream.close()
-        except Exception:
-            pass
+        if pipewire_stop_event is not None:
+            pipewire_stop_event.set()
+        if pipewire_capture_thread is not None:
+            pipewire_capture_thread.join(timeout=2.0)
+
+        if stream is not None:
+            try:
+                stream.stop()
+            except Exception:
+                pass
+            try:
+                stream.close()
+            except Exception:
+                pass
+
+        linux_audio.teardown_virtual_input_sink()
 
 
 if __name__ == "__main__":