diff options
| -rw-r--r-- | README.md | 20 | ||||
| -rw-r--r-- | gui/gui_settings.py | 29 | ||||
| -rw-r--r-- | linux_audio.py | 385 | ||||
| -rw-r--r-- | server.py | 157 |
4 files changed, 543 insertions, 48 deletions
@@ -13,7 +13,25 @@ uv sync uv run server.py ``` -`server.py` serves a backend for translating incoming audio data. It expects some other client to hit the `/events` endpoint to fetch the translated data. A GUI is available for configuration +`server.py` serves a backend for translating incoming audio data. It expects some other client to hit the `/events` endpoint to fetch the translated data. A GUI is available for configuration. + +## Linux virtual audio sink/input (PipeWire) +On Linux, startup now attempts to create: +- a virtual sink via `module-null-sink` + - sink name: `auto_live_tl_sink` + - sink description: `auto-live-tl-virtual-sink` +- a virtual input source via `module-remap-source` + - source name: `auto_live_tl_input` + - source description: `auto-live-tl-virtual-input` +- a headphone/speaker mirror loop via `module-loopback` + - source: `auto_live_tl_sink.monitor` + - sink: your current default output sink + +This gives you a direct mic-like input device in the Settings UI (shown with `[virtual input]`) while still hearing audio on your normal output device. + +If your PortAudio backend cannot see PipeWire sources, the app adds a fallback option named `auto-live-tl-virtual-input (PipeWire direct)` and captures from PipeWire using `ffmpeg`. + +If `pactl` is unavailable (or device creation fails), the app continues running and falls back to normal input devices. # Whisper + Ollama (Local Setup) > It's highly recommended that you run this with a GPU, running with CPU is possible but inference will be very slow outside of using tiny models (which compromise accuracy) diff --git a/gui/gui_settings.py b/gui/gui_settings.py index 2956b0e..6ac0763 100644 --- a/gui/gui_settings.py +++ b/gui/gui_settings.py @@ -67,10 +67,17 @@ class _SettingsDialog(QDialog): whisper_layout = QFormLayout() whisper_layout.setLabelAlignment(Qt.AlignmentFlag.AlignLeft) - device_options = [ - f"[{idx}] {dev['name']} ({dev.get('max_input_channels', 0)} ch)" - for idx, dev in input_devices - ] + def _format_device_option(idx: int, dev: Dict[str, Any]) -> str: + name = str(dev.get("name", "")) + lowered = name.lower() + virtual_hint = "" + if "auto_live_tl_input" in lowered or "auto-live-tl-virtual-input" in lowered: + virtual_hint = " [virtual input]" + elif "auto_live_tl_sink" in lowered or "auto-live-tl-virtual-sink" in lowered: + virtual_hint = " [virtual sink monitor]" + return f"[{idx}] {name}{virtual_hint} ({dev.get('max_input_channels', 0)} ch)" + + device_options = [_format_device_option(idx, dev) for idx, dev in input_devices] self.device_combo = QComboBox(whisper_tab) self.device_combo.addItems(device_options) self.device_combo.setEditable(False) @@ -78,7 +85,16 @@ class _SettingsDialog(QDialog): if default_device_name in self.device_names: self.device_combo.setCurrentIndex(self.device_names.index(default_device_name)) else: - self.device_combo.setCurrentIndex(0) + preferred_index = -1 + for i, name in enumerate(self.device_names): + lowered = str(name).lower() + if "auto-live-tl-virtual-input" in lowered or "auto_live_tl_input" in lowered: + preferred_index = i + break + if preferred_index >= 0: + self.device_combo.setCurrentIndex(preferred_index) + else: + self.device_combo.setCurrentIndex(0) whisper_layout.addRow(QLabel("Audio input device:"), self.device_combo) self.model_combo = QComboBox(whisper_tab) @@ -348,6 +364,9 @@ class _SettingsDialog(QDialog): return device_index = self.device_indices[selection] + if device_index < 0: + self._monitor_error = "Live monitor preview is unavailable for PipeWire direct input." + return try: device_info = sd.query_devices(device_index) except Exception as exc: diff --git a/linux_audio.py b/linux_audio.py new file mode 100644 index 0000000..ea162bb --- /dev/null +++ b/linux_audio.py @@ -0,0 +1,385 @@ +import os +import select +import shutil +import subprocess +import sys +import threading +import time +from typing import Any, Callable, Dict, List, Optional, Tuple + +import numpy as np +import sounddevice as sd + +LINUX_VIRTUAL_SINK_MODULE: str = "module-null-sink" +LINUX_VIRTUAL_SOURCE_MODULE: str = "module-remap-source" +LINUX_VIRTUAL_LOOPBACK_MODULE: str = "module-loopback" +LINUX_VIRTUAL_SINK_NAME: str = "auto_live_tl_sink" +LINUX_VIRTUAL_SOURCE_NAME: str = "auto_live_tl_input" +LINUX_VIRTUAL_SINK_DESCRIPTION: str = "auto-live-tl-virtual-sink" +LINUX_VIRTUAL_SOURCE_DESCRIPTION: str = "auto-live-tl-virtual-input" +LINUX_VIRTUAL_DEVICE_WAIT_SECONDS: float = 2.5 +LINUX_VIRTUAL_DIRECT_DEVICE_NAME: str = "auto-live-tl-virtual-input (PipeWire direct)" +LINUX_VIRTUAL_DIRECT_CAPTURE_RATE: int = 48000 +LINUX_DIRECT_CAPTURE_IDLE_TIMEOUT_SECONDS: float = 3.0 + +_virtual_sink_module_id: Optional[int] = None +_virtual_source_module_id: Optional[int] = None +_virtual_loopback_module_id: Optional[int] = None +_virtual_sink_created_by_app: bool = False +_virtual_source_created_by_app: bool = False +_virtual_loopback_created_by_app: bool = False + + +def is_linux() -> bool: + return sys.platform.startswith("linux") + + +def is_virtual_input_name(device_name: str) -> bool: + lowered = device_name.lower() + return ( + LINUX_VIRTUAL_SOURCE_NAME.lower() in lowered + or LINUX_VIRTUAL_SOURCE_DESCRIPTION.lower() in lowered + or LINUX_VIRTUAL_SINK_NAME.lower() in lowered + or LINUX_VIRTUAL_SINK_DESCRIPTION.lower() in lowered + ) + + +def _run_pactl(args: List[str]) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["pactl", *args], + check=False, + capture_output=True, + text=True, + ) + + +def _find_existing_module_id(module_name: str, arg_match: str) -> Optional[int]: + result = _run_pactl(["list", "short", "modules"]) + if result.returncode != 0: + return None + + for line in result.stdout.splitlines(): + parts = line.split("\t") + if len(parts) < 3: + continue + module_id_raw, listed_module_name, module_args = parts[0], parts[1], parts[2] + if listed_module_name != module_name: + continue + if arg_match not in module_args: + continue + try: + return int(module_id_raw) + except ValueError: + continue + return None + + +def _wait_for_virtual_input_device() -> bool: + deadline = time.monotonic() + LINUX_VIRTUAL_DEVICE_WAIT_SECONDS + while time.monotonic() < deadline: + try: + devices = sd.query_devices() + except Exception: + time.sleep(0.1) + continue + for dev in devices: + if dev.get("max_input_channels", 0) > 0 and is_virtual_input_name(str(dev.get("name", ""))): + return True + time.sleep(0.1) + return False + + +def _get_default_sink_name() -> str: + result = _run_pactl(["get-default-sink"]) + if result.returncode == 0: + sink_name = (result.stdout or "").strip() + if sink_name: + return sink_name + return "@DEFAULT_SINK@" + + +def ensure_virtual_input_sink() -> None: + global _virtual_sink_module_id, _virtual_source_module_id, _virtual_loopback_module_id + global _virtual_sink_created_by_app, _virtual_source_created_by_app, _virtual_loopback_created_by_app + + if not is_linux(): + return + + if shutil.which("pactl") is None: + print("[audio] 'pactl' not found, skipping PipeWire virtual device creation.") + return + + existing_sink_id = _find_existing_module_id( + LINUX_VIRTUAL_SINK_MODULE, + f"sink_name={LINUX_VIRTUAL_SINK_NAME}", + ) + if existing_sink_id is not None: + _virtual_sink_module_id = existing_sink_id + _virtual_sink_created_by_app = False + print(f"[audio] Reusing existing virtual sink module id={existing_sink_id}.") + else: + sink_result = _run_pactl( + [ + "load-module", + LINUX_VIRTUAL_SINK_MODULE, + f"sink_name={LINUX_VIRTUAL_SINK_NAME}", + f"sink_properties=device.description={LINUX_VIRTUAL_SINK_DESCRIPTION}", + ] + ) + if sink_result.returncode != 0: + stderr = (sink_result.stderr or "").strip() + print(f"[audio] Failed to create PipeWire virtual sink: {stderr or 'unknown error'}") + return + + sink_id_text = (sink_result.stdout or "").strip() + try: + _virtual_sink_module_id = int(sink_id_text) + _virtual_sink_created_by_app = True + print( + "[audio] Created PipeWire virtual sink " + f"'{LINUX_VIRTUAL_SINK_DESCRIPTION}' (module id={_virtual_sink_module_id})." + ) + except ValueError: + _virtual_sink_module_id = None + _virtual_sink_created_by_app = True + print("[audio] Created PipeWire virtual sink, but could not parse module id.") + + existing_source_id = _find_existing_module_id( + LINUX_VIRTUAL_SOURCE_MODULE, + f"source_name={LINUX_VIRTUAL_SOURCE_NAME}", + ) + if existing_source_id is not None: + _virtual_source_module_id = existing_source_id + _virtual_source_created_by_app = False + print(f"[audio] Reusing existing virtual input module id={existing_source_id}.") + else: + source_result = _run_pactl( + [ + "load-module", + LINUX_VIRTUAL_SOURCE_MODULE, + f"master={LINUX_VIRTUAL_SINK_NAME}.monitor", + f"source_name={LINUX_VIRTUAL_SOURCE_NAME}", + f"source_properties=device.description={LINUX_VIRTUAL_SOURCE_DESCRIPTION}", + ] + ) + if source_result.returncode != 0: + stderr = (source_result.stderr or "").strip() + print(f"[audio] Failed to create virtual input source: {stderr or 'unknown error'}") + return + + source_id_text = (source_result.stdout or "").strip() + try: + _virtual_source_module_id = int(source_id_text) + _virtual_source_created_by_app = True + print( + "[audio] Created virtual input source " + f"'{LINUX_VIRTUAL_SOURCE_DESCRIPTION}' (module id={_virtual_source_module_id})." + ) + except ValueError: + _virtual_source_module_id = None + _virtual_source_created_by_app = True + print("[audio] Created virtual input source, but could not parse module id.") + + existing_loopback_id = _find_existing_module_id( + LINUX_VIRTUAL_LOOPBACK_MODULE, + f"source={LINUX_VIRTUAL_SINK_NAME}.monitor", + ) + if existing_loopback_id is not None: + _virtual_loopback_module_id = existing_loopback_id + _virtual_loopback_created_by_app = False + print(f"[audio] Reusing existing headphone loopback module id={existing_loopback_id}.") + else: + default_sink = _get_default_sink_name() + loopback_result = _run_pactl( + [ + "load-module", + LINUX_VIRTUAL_LOOPBACK_MODULE, + f"source={LINUX_VIRTUAL_SINK_NAME}.monitor", + f"sink={default_sink}", + "latency_msec=60", + ] + ) + if loopback_result.returncode != 0: + stderr = (loopback_result.stderr or "").strip() + print(f"[audio] Failed to create headphone loopback: {stderr or 'unknown error'}") + else: + loopback_id_text = (loopback_result.stdout or "").strip() + try: + _virtual_loopback_module_id = int(loopback_id_text) + _virtual_loopback_created_by_app = True + print( + "[audio] Mirroring virtual sink to default output " + f"(module id={_virtual_loopback_module_id}, sink={default_sink})." + ) + except ValueError: + _virtual_loopback_module_id = None + _virtual_loopback_created_by_app = True + print("[audio] Headphone loopback created, but could not parse module id.") + + if not _wait_for_virtual_input_device(): + print("[audio] Virtual input device was not detected by PortAudio yet.") + + +def teardown_virtual_input_sink() -> None: + global _virtual_sink_module_id, _virtual_source_module_id, _virtual_loopback_module_id + global _virtual_sink_created_by_app, _virtual_source_created_by_app, _virtual_loopback_created_by_app + + if not is_linux() or shutil.which("pactl") is None: + return + + if _virtual_loopback_created_by_app and _virtual_loopback_module_id is not None: + result = _run_pactl(["unload-module", str(_virtual_loopback_module_id)]) + if result.returncode != 0: + stderr = (result.stderr or "").strip() + print(f"[audio] Failed to unload headphone loopback module {_virtual_loopback_module_id}: {stderr or 'unknown error'}") + else: + print(f"[audio] Unloaded headphone loopback module {_virtual_loopback_module_id}.") + + if _virtual_source_created_by_app and _virtual_source_module_id is not None: + result = _run_pactl(["unload-module", str(_virtual_source_module_id)]) + if result.returncode != 0: + stderr = (result.stderr or "").strip() + print(f"[audio] Failed to unload virtual input module {_virtual_source_module_id}: {stderr or 'unknown error'}") + else: + print(f"[audio] Unloaded virtual input module {_virtual_source_module_id}.") + + if _virtual_sink_created_by_app and _virtual_sink_module_id is not None: + result = _run_pactl(["unload-module", str(_virtual_sink_module_id)]) + if result.returncode != 0: + stderr = (result.stderr or "").strip() + print(f"[audio] Failed to unload virtual sink module {_virtual_sink_module_id}: {stderr or 'unknown error'}") + else: + print(f"[audio] Unloaded virtual sink module {_virtual_sink_module_id}.") + + _virtual_loopback_module_id = None + _virtual_source_module_id = None + _virtual_sink_module_id = None + _virtual_loopback_created_by_app = False + _virtual_source_created_by_app = False + _virtual_sink_created_by_app = False + + +def append_pipewire_direct_device_option( + input_devices: List[Tuple[int, Dict[str, Any]]], + has_virtual_input: bool, +) -> None: + if not has_virtual_input: + print( + "[audio] Virtual input device not visible in sounddevice device list. " + "Adding PipeWire direct capture option instead." + ) + + input_devices.append( + ( + -1, + { + "name": LINUX_VIRTUAL_DIRECT_DEVICE_NAME, + "max_input_channels": 1, + "default_samplerate": float(LINUX_VIRTUAL_DIRECT_CAPTURE_RATE), + }, + ) + ) + + +def _build_pipewire_direct_commands(sample_rate: int) -> List[List[str]]: + commands: List[List[str]] = [] + + if shutil.which("ffmpeg") is not None: + for source_name in [LINUX_VIRTUAL_SOURCE_NAME, f"{LINUX_VIRTUAL_SINK_NAME}.monitor"]: + commands.append( + [ + "ffmpeg", + "-nostdin", + "-hide_banner", + "-loglevel", + "error", + "-f", + "pulse", + "-i", + source_name, + "-ac", + "1", + "-ar", + str(sample_rate), + "-f", + "f32le", + "-", + ] + ) + + return commands + + +def has_pipewire_direct_backend(sample_rate: int) -> bool: + return bool(_build_pipewire_direct_commands(sample_rate)) + + +def pipewire_direct_capture_worker( + stop_event: threading.Event, + sample_rate: int, + on_chunk: Callable[[np.ndarray], None], + add_runtime_log: Callable[[str, str], None], +) -> None: + try: + _run_pipewire_direct_capture(stop_event, sample_rate, on_chunk) + except Exception as exc: + message = f"PipeWire direct capture failed: {exc}" + print(f"[audio] {message}") + add_runtime_log("AUDIO", message) + + +def _run_pipewire_direct_capture( + stop_event: threading.Event, + sample_rate: int, + on_chunk: Callable[[np.ndarray], None], +) -> None: + commands = _build_pipewire_direct_commands(sample_rate) + if not commands: + raise RuntimeError("PipeWire direct capture needs 'ffmpeg' installed.") + + bytes_per_chunk = int(sample_rate * 0.5) * 4 + + for command in commands: + process: Optional[subprocess.Popen[bytes]] = None + try: + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + time.sleep(0.2) + if process.poll() is not None: + continue + + assert process.stdout is not None + print(f"[audio] PipeWire direct capture backend: {' '.join(command[:2])}") + + silent_windows = 0 + while not stop_event.is_set(): + ready, _, _ = select.select([process.stdout], [], [], 1.0) + if not ready: + silent_windows += 1 + if silent_windows * 1.0 >= LINUX_DIRECT_CAPTURE_IDLE_TIMEOUT_SECONDS: + raise RuntimeError("backend produced no audio frames") + continue + + raw = os.read(process.stdout.fileno(), bytes_per_chunk) + if not raw: + raise RuntimeError("backend stream ended") + + silent_windows = 0 + chunk = np.frombuffer(raw, dtype=np.float32).copy() + if chunk.size == 0: + continue + on_chunk(chunk) + + return + except Exception as exc: + backend_name = " ".join(command[:2]) + print(f"[audio] Direct capture backend failed ({backend_name}): {exc}") + continue + finally: + if process is not None and process.poll() is None: + process.terminate() + try: + process.wait(timeout=1.5) + except subprocess.TimeoutExpired: + process.kill() + + raise RuntimeError("All PipeWire direct capture backends failed.") @@ -3,6 +3,7 @@ import threading import json import queue import os +import atexit from collections import Counter, deque import re from typing import Any, Deque, Dict, Optional, Set, List, Iterator, Callable @@ -20,6 +21,7 @@ from gui.gui import select_settings, prompt_input_sample_rate, run_runtime_dashb from openai_realtime import OpenAIRealtimeTranslator from routes import register_routes from config import _SYSTEM_PROMPT, _LLM_EMPTY_SENTINELS, _HALLUCINATION_PHRASES +import linux_audio TARGET_SAMPLE_RATE: int = 16000 CAPTURE_SAMPLE_RATE: int = 0 @@ -32,6 +34,7 @@ SSE_KEEPALIVE_SECONDS: int = 15 RUNTIME_SUBTITLE_LINES_MAX: int = 120 RUNTIME_LOG_LINES_MAX: int = 300 + USE_OLLAMA_CLEANUP: bool = True USE_OPENAI_REALTIME_TRANSLATE: bool = False OLLAMA_MODEL: str = "qwen2.5:7b-instruct" @@ -113,6 +116,8 @@ _raw_batch_lock: threading.Lock = threading.Lock() openai_realtime_client: Optional[OpenAIRealtimeTranslator] = None +_virtual_sink_cleanup_registered: bool = False + def resample_audio(audio_np: np.ndarray, src_rate: int, dst_rate: int) -> np.ndarray: """ Resamples audio to TARGET_SAMPLE_RATE (default is 16000hz), speeds up inference time, fetched as a nd array @@ -131,6 +136,7 @@ def resample_audio(audio_np: np.ndarray, src_rate: int, dst_rate: int) -> np.nda + def load_settings() -> Dict[str, Any]: if not os.path.exists(SETTINGS_PATH): return DEFAULT_SETTINGS.copy() @@ -174,7 +180,7 @@ def cleanup_subtitle_with_ollama(raw_text: str, context: List[str]) -> Optional[ ], options=OLLAMA_OPTIONS, ) - return response.message.content.strip() + return (response.message.content or "").strip() except Exception as exc: print(f"⚠️ OLLAMA cleanup error: {exc}") return None @@ -197,7 +203,7 @@ def ensure_ollama_ready(status_callback: Optional[Callable[[str], None]] = None) f"Cannot reach Ollama — is the server running? ({exc})" ) from exc - model_names: List[str] = [m.model for m in local.models] + model_names: List[str] = [str(m.model) for m in local.models if m.model] if not any(name.startswith(OLLAMA_MODEL) for name in model_names): report(f"Model '{OLLAMA_MODEL}' not found locally. Pulling now (this can take a while)...") try: @@ -478,14 +484,7 @@ def publish_audio_activity(chunk_rms: float) -> None: broadcast_event(SSE_EVENT_AUDIO_ACTIVITY, payload) -def audio_callback(indata: np.ndarray, frames: int, time_info: Any, status: Any) -> None: - """ - Callback definition for audio sink. Sends audio to local Whisper buffer or OpenAI realtime queue. - """ - if status: - print(f"Audio status: {status}") - # Take first channel - chunk: np.ndarray = indata[:, 0].copy() +def _handle_audio_chunk(chunk: np.ndarray) -> None: chunk_rms: float = float(np.sqrt(np.mean(np.square(chunk)))) if len(chunk) > 0 else 0.0 publish_audio_activity(chunk_rms) @@ -501,6 +500,18 @@ def audio_callback(indata: np.ndarray, frames: int, time_info: Any, status: Any) audio_buffer = audio_buffer[-MAX_SAMPLES:] +def audio_callback(indata: np.ndarray, frames: int, time_info: Any, status: Any) -> None: + """ + Callback definition for audio sink. Sends audio to local Whisper buffer or OpenAI realtime queue. + """ + if status: + print(f"Audio status: {status}") + if indata is None or len(indata) == 0: + return + chunk: np.ndarray = indata[:, 0].copy() + _handle_audio_chunk(chunk) + + def is_silent(audio_16k: Optional[np.ndarray]) -> bool: """ Basic rudimentary silence detection, do not run whisper if rms value isn't reached @@ -547,19 +558,31 @@ def select_input_sample_rate(device_index: int, preferred_rate: int) -> int: return prompt_input_sample_rate(device_index, common_rates) + def main() -> None: global CAPTURE_SAMPLE_RATE, MAX_SAMPLES, model, WHISPER_TASK, WHISPER_BEAM_SIZE, WHISPER_LANGUAGE global BUFFER_SECONDS, PROCESS_INTERVAL_SECONDS, USE_OLLAMA_CLEANUP, USE_OPENAI_REALTIME_TRANSLATE global OLLAMA_MODEL, OLLAMA_CONTEXT_WINDOW, RAW_BATCH_SIZE, subtitle_context global AUDIO_ACTIVITY_THRESHOLD, last_audio_activity_payload, _audio_active_until, _audio_last_emit global OPENAI_API_KEY, OPENAI_OUTPUT_LANGUAGE, OPENAI_REALTIME_MODEL, OPENAI_SAFETY_IDENTIFIER - global openai_realtime_client + global openai_realtime_client, _virtual_sink_cleanup_registered start_subtitle_server() + linux_audio.ensure_virtual_input_sink() + if linux_audio.is_linux() and not _virtual_sink_cleanup_registered: + atexit.register(linux_audio.teardown_virtual_input_sink) + _virtual_sink_cleanup_registered = True + settings: Dict[str, Any] = load_settings() devices = sd.query_devices() input_devices = [(idx, dev) for idx, dev in enumerate(devices) if dev["max_input_channels"] > 0] + if linux_audio.is_linux(): + has_virtual_input = any( + linux_audio.is_virtual_input_name(str(dev.get("name", ""))) + for _idx, dev in input_devices + ) + linux_audio.append_pipewire_direct_device_option(input_devices, has_virtual_input) settings = select_settings( settings, input_devices, @@ -597,14 +620,20 @@ def main() -> None: llm_thread.start() device_name: str = settings.get("audio_device_name", "") + use_pipewire_direct_capture = ( + linux_audio.is_linux() + and device_name == linux_audio.LINUX_VIRTUAL_DIRECT_DEVICE_NAME + ) + matched_index: Optional[int] = None - for idx, dev in enumerate(devices): - if dev.get("name") == device_name and dev.get("max_input_channels", 0) > 0: - matched_index = idx - break - if matched_index is None: - raise RuntimeError("Saved audio device not found. Please reselect in the settings window.") - device_index: int = matched_index + if not use_pipewire_direct_capture: + for idx, dev in enumerate(devices): + if dev.get("name") == device_name and dev.get("max_input_channels", 0) > 0: + matched_index = idx + break + if matched_index is None: + raise RuntimeError("Saved audio device not found. Please reselect in the settings window.") + device_index: Optional[int] = matched_index model_name: str = settings["model_name"] whisper_device: str = settings["device"] @@ -631,11 +660,20 @@ def main() -> None: _audio_last_emit = 0.0 broadcast_event(SSE_EVENT_AUDIO_ACTIVITY, last_audio_activity_payload) - device_info = sd.query_devices(device_index) - preferred_rate: int = int(device_info["default_samplerate"]) - if preferred_rate <= 0: - preferred_rate = 48000 - CAPTURE_SAMPLE_RATE = select_input_sample_rate(device_index, preferred_rate) + if use_pipewire_direct_capture: + device_info = { + "name": linux_audio.LINUX_VIRTUAL_DIRECT_DEVICE_NAME, + "default_samplerate": float(linux_audio.LINUX_VIRTUAL_DIRECT_CAPTURE_RATE), + } + CAPTURE_SAMPLE_RATE = linux_audio.LINUX_VIRTUAL_DIRECT_CAPTURE_RATE + else: + assert device_index is not None + device_info = sd.query_devices(device_index) + preferred_rate: int = int(device_info["default_samplerate"]) + if preferred_rate <= 0: + preferred_rate = 48000 + CAPTURE_SAMPLE_RATE = select_input_sample_rate(device_index, preferred_rate) + MAX_SAMPLES = int(CAPTURE_SAMPLE_RATE * BUFFER_SECONDS) with recent_subtitle_lines_lock: @@ -660,7 +698,10 @@ def main() -> None: ) openai_realtime_client.start() - print(f"Using device {device_index}: {device_info['name']}") + if use_pipewire_direct_capture: + print(f"Using Linux PipeWire direct input: {device_info['name']}") + else: + print(f"Using device {device_index}: {device_info['name']}") print(f"Realtime translation backend: OpenAI ({OPENAI_REALTIME_MODEL})") print( f"Capture sample rate: {CAPTURE_SAMPLE_RATE} Hz " @@ -674,7 +715,10 @@ def main() -> None: openai_realtime_client = None model = WhisperModel(model_name, device=whisper_device, compute_type=compute_type) - print(f"Using device {device_index}: {device_info['name']}") + if use_pipewire_direct_capture: + print(f"Using Linux PipeWire direct input: {device_info['name']}") + else: + print(f"Using device {device_index}: {device_info['name']}") print(f"Model: {model_name} | task={WHISPER_TASK} | beam_size={WHISPER_BEAM_SIZE}") print(f"Compute: device={whisper_device} | compute_type={compute_type}") print(f"Capture sample rate: {CAPTURE_SAMPLE_RATE} Hz (resampling to {TARGET_SAMPLE_RATE} Hz)") @@ -686,20 +730,41 @@ def main() -> None: processing_thread = threading.Thread(target=processing_loop, daemon=True) processing_thread.start() - stream = sd.InputStream( - device=device_index, - channels=1, - samplerate=CAPTURE_SAMPLE_RATE, - dtype="float32", - callback=audio_callback, - blocksize=int(CAPTURE_SAMPLE_RATE * 0.5), - ) + stream: Optional[sd.InputStream] = None + pipewire_stop_event: Optional[threading.Event] = None + pipewire_capture_thread: Optional[threading.Thread] = None + + if use_pipewire_direct_capture: + if not linux_audio.has_pipewire_direct_backend(CAPTURE_SAMPLE_RATE): + raise RuntimeError( + "PipeWire direct virtual input was selected, but 'ffmpeg' is not installed. " + "Install ffmpeg and try again." + ) + pipewire_stop_event = threading.Event() + pipewire_capture_thread = threading.Thread( + target=linux_audio.pipewire_direct_capture_worker, + args=(pipewire_stop_event, CAPTURE_SAMPLE_RATE, _handle_audio_chunk, add_runtime_log), + daemon=True, + ) + else: + assert device_index is not None + stream = sd.InputStream( + device=device_index, + channels=1, + samplerate=CAPTURE_SAMPLE_RATE, + dtype="float32", + callback=audio_callback, + blocksize=int(CAPTURE_SAMPLE_RATE * 0.5), + ) def _on_dashboard_close() -> None: print("Stopping.") try: - stream.start() + if pipewire_capture_thread is not None: + pipewire_capture_thread.start() + elif stream is not None: + stream.start() print("Listening... Close the runtime window to stop.") run_runtime_dashboard( get_audio_activity=get_audio_activity_snapshot, @@ -712,14 +777,22 @@ def main() -> None: openai_realtime_client.stop() openai_realtime_client = None - try: - stream.stop() - except Exception: - pass - try: - stream.close() - except Exception: - pass + if pipewire_stop_event is not None: + pipewire_stop_event.set() + if pipewire_capture_thread is not None: + pipewire_capture_thread.join(timeout=2.0) + + if stream is not None: + try: + stream.stop() + except Exception: + pass + try: + stream.close() + except Exception: + pass + + linux_audio.teardown_virtual_input_sink() if __name__ == "__main__": |