diff options
Diffstat (limited to 'server.py')
| -rw-r--r-- | server.py | 157 |
1 files changed, 115 insertions, 42 deletions
@@ -3,6 +3,7 @@ import threading import json import queue import os +import atexit from collections import Counter, deque import re from typing import Any, Deque, Dict, Optional, Set, List, Iterator, Callable @@ -20,6 +21,7 @@ from gui.gui import select_settings, prompt_input_sample_rate, run_runtime_dashb from openai_realtime import OpenAIRealtimeTranslator from routes import register_routes from config import _SYSTEM_PROMPT, _LLM_EMPTY_SENTINELS, _HALLUCINATION_PHRASES +import linux_audio TARGET_SAMPLE_RATE: int = 16000 CAPTURE_SAMPLE_RATE: int = 0 @@ -32,6 +34,7 @@ SSE_KEEPALIVE_SECONDS: int = 15 RUNTIME_SUBTITLE_LINES_MAX: int = 120 RUNTIME_LOG_LINES_MAX: int = 300 + USE_OLLAMA_CLEANUP: bool = True USE_OPENAI_REALTIME_TRANSLATE: bool = False OLLAMA_MODEL: str = "qwen2.5:7b-instruct" @@ -113,6 +116,8 @@ _raw_batch_lock: threading.Lock = threading.Lock() openai_realtime_client: Optional[OpenAIRealtimeTranslator] = None +_virtual_sink_cleanup_registered: bool = False + def resample_audio(audio_np: np.ndarray, src_rate: int, dst_rate: int) -> np.ndarray: """ Resamples audio to TARGET_SAMPLE_RATE (default is 16000hz), speeds up inference time, fetched as a nd array @@ -131,6 +136,7 @@ def resample_audio(audio_np: np.ndarray, src_rate: int, dst_rate: int) -> np.nda + def load_settings() -> Dict[str, Any]: if not os.path.exists(SETTINGS_PATH): return DEFAULT_SETTINGS.copy() @@ -174,7 +180,7 @@ def cleanup_subtitle_with_ollama(raw_text: str, context: List[str]) -> Optional[ ], options=OLLAMA_OPTIONS, ) - return response.message.content.strip() + return (response.message.content or "").strip() except Exception as exc: print(f"⚠️ OLLAMA cleanup error: {exc}") return None @@ -197,7 +203,7 @@ def ensure_ollama_ready(status_callback: Optional[Callable[[str], None]] = None) f"Cannot reach Ollama — is the server running? ({exc})" ) from exc - model_names: List[str] = [m.model for m in local.models] + model_names: List[str] = [str(m.model) for m in local.models if m.model] if not any(name.startswith(OLLAMA_MODEL) for name in model_names): report(f"Model '{OLLAMA_MODEL}' not found locally. Pulling now (this can take a while)...") try: @@ -478,14 +484,7 @@ def publish_audio_activity(chunk_rms: float) -> None: broadcast_event(SSE_EVENT_AUDIO_ACTIVITY, payload) -def audio_callback(indata: np.ndarray, frames: int, time_info: Any, status: Any) -> None: - """ - Callback definition for audio sink. Sends audio to local Whisper buffer or OpenAI realtime queue. - """ - if status: - print(f"Audio status: {status}") - # Take first channel - chunk: np.ndarray = indata[:, 0].copy() +def _handle_audio_chunk(chunk: np.ndarray) -> None: chunk_rms: float = float(np.sqrt(np.mean(np.square(chunk)))) if len(chunk) > 0 else 0.0 publish_audio_activity(chunk_rms) @@ -501,6 +500,18 @@ def audio_callback(indata: np.ndarray, frames: int, time_info: Any, status: Any) audio_buffer = audio_buffer[-MAX_SAMPLES:] +def audio_callback(indata: np.ndarray, frames: int, time_info: Any, status: Any) -> None: + """ + Callback definition for audio sink. Sends audio to local Whisper buffer or OpenAI realtime queue. + """ + if status: + print(f"Audio status: {status}") + if indata is None or len(indata) == 0: + return + chunk: np.ndarray = indata[:, 0].copy() + _handle_audio_chunk(chunk) + + def is_silent(audio_16k: Optional[np.ndarray]) -> bool: """ Basic rudimentary silence detection, do not run whisper if rms value isn't reached @@ -547,19 +558,31 @@ def select_input_sample_rate(device_index: int, preferred_rate: int) -> int: return prompt_input_sample_rate(device_index, common_rates) + def main() -> None: global CAPTURE_SAMPLE_RATE, MAX_SAMPLES, model, WHISPER_TASK, WHISPER_BEAM_SIZE, WHISPER_LANGUAGE global BUFFER_SECONDS, PROCESS_INTERVAL_SECONDS, USE_OLLAMA_CLEANUP, USE_OPENAI_REALTIME_TRANSLATE global OLLAMA_MODEL, OLLAMA_CONTEXT_WINDOW, RAW_BATCH_SIZE, subtitle_context global AUDIO_ACTIVITY_THRESHOLD, last_audio_activity_payload, _audio_active_until, _audio_last_emit global OPENAI_API_KEY, OPENAI_OUTPUT_LANGUAGE, OPENAI_REALTIME_MODEL, OPENAI_SAFETY_IDENTIFIER - global openai_realtime_client + global openai_realtime_client, _virtual_sink_cleanup_registered start_subtitle_server() + linux_audio.ensure_virtual_input_sink() + if linux_audio.is_linux() and not _virtual_sink_cleanup_registered: + atexit.register(linux_audio.teardown_virtual_input_sink) + _virtual_sink_cleanup_registered = True + settings: Dict[str, Any] = load_settings() devices = sd.query_devices() input_devices = [(idx, dev) for idx, dev in enumerate(devices) if dev["max_input_channels"] > 0] + if linux_audio.is_linux(): + has_virtual_input = any( + linux_audio.is_virtual_input_name(str(dev.get("name", ""))) + for _idx, dev in input_devices + ) + linux_audio.append_pipewire_direct_device_option(input_devices, has_virtual_input) settings = select_settings( settings, input_devices, @@ -597,14 +620,20 @@ def main() -> None: llm_thread.start() device_name: str = settings.get("audio_device_name", "") + use_pipewire_direct_capture = ( + linux_audio.is_linux() + and device_name == linux_audio.LINUX_VIRTUAL_DIRECT_DEVICE_NAME + ) + matched_index: Optional[int] = None - for idx, dev in enumerate(devices): - if dev.get("name") == device_name and dev.get("max_input_channels", 0) > 0: - matched_index = idx - break - if matched_index is None: - raise RuntimeError("Saved audio device not found. Please reselect in the settings window.") - device_index: int = matched_index + if not use_pipewire_direct_capture: + for idx, dev in enumerate(devices): + if dev.get("name") == device_name and dev.get("max_input_channels", 0) > 0: + matched_index = idx + break + if matched_index is None: + raise RuntimeError("Saved audio device not found. Please reselect in the settings window.") + device_index: Optional[int] = matched_index model_name: str = settings["model_name"] whisper_device: str = settings["device"] @@ -631,11 +660,20 @@ def main() -> None: _audio_last_emit = 0.0 broadcast_event(SSE_EVENT_AUDIO_ACTIVITY, last_audio_activity_payload) - device_info = sd.query_devices(device_index) - preferred_rate: int = int(device_info["default_samplerate"]) - if preferred_rate <= 0: - preferred_rate = 48000 - CAPTURE_SAMPLE_RATE = select_input_sample_rate(device_index, preferred_rate) + if use_pipewire_direct_capture: + device_info = { + "name": linux_audio.LINUX_VIRTUAL_DIRECT_DEVICE_NAME, + "default_samplerate": float(linux_audio.LINUX_VIRTUAL_DIRECT_CAPTURE_RATE), + } + CAPTURE_SAMPLE_RATE = linux_audio.LINUX_VIRTUAL_DIRECT_CAPTURE_RATE + else: + assert device_index is not None + device_info = sd.query_devices(device_index) + preferred_rate: int = int(device_info["default_samplerate"]) + if preferred_rate <= 0: + preferred_rate = 48000 + CAPTURE_SAMPLE_RATE = select_input_sample_rate(device_index, preferred_rate) + MAX_SAMPLES = int(CAPTURE_SAMPLE_RATE * BUFFER_SECONDS) with recent_subtitle_lines_lock: @@ -660,7 +698,10 @@ def main() -> None: ) openai_realtime_client.start() - print(f"Using device {device_index}: {device_info['name']}") + if use_pipewire_direct_capture: + print(f"Using Linux PipeWire direct input: {device_info['name']}") + else: + print(f"Using device {device_index}: {device_info['name']}") print(f"Realtime translation backend: OpenAI ({OPENAI_REALTIME_MODEL})") print( f"Capture sample rate: {CAPTURE_SAMPLE_RATE} Hz " @@ -674,7 +715,10 @@ def main() -> None: openai_realtime_client = None model = WhisperModel(model_name, device=whisper_device, compute_type=compute_type) - print(f"Using device {device_index}: {device_info['name']}") + if use_pipewire_direct_capture: + print(f"Using Linux PipeWire direct input: {device_info['name']}") + else: + print(f"Using device {device_index}: {device_info['name']}") print(f"Model: {model_name} | task={WHISPER_TASK} | beam_size={WHISPER_BEAM_SIZE}") print(f"Compute: device={whisper_device} | compute_type={compute_type}") print(f"Capture sample rate: {CAPTURE_SAMPLE_RATE} Hz (resampling to {TARGET_SAMPLE_RATE} Hz)") @@ -686,20 +730,41 @@ def main() -> None: processing_thread = threading.Thread(target=processing_loop, daemon=True) processing_thread.start() - stream = sd.InputStream( - device=device_index, - channels=1, - samplerate=CAPTURE_SAMPLE_RATE, - dtype="float32", - callback=audio_callback, - blocksize=int(CAPTURE_SAMPLE_RATE * 0.5), - ) + stream: Optional[sd.InputStream] = None + pipewire_stop_event: Optional[threading.Event] = None + pipewire_capture_thread: Optional[threading.Thread] = None + + if use_pipewire_direct_capture: + if not linux_audio.has_pipewire_direct_backend(CAPTURE_SAMPLE_RATE): + raise RuntimeError( + "PipeWire direct virtual input was selected, but 'ffmpeg' is not installed. " + "Install ffmpeg and try again." + ) + pipewire_stop_event = threading.Event() + pipewire_capture_thread = threading.Thread( + target=linux_audio.pipewire_direct_capture_worker, + args=(pipewire_stop_event, CAPTURE_SAMPLE_RATE, _handle_audio_chunk, add_runtime_log), + daemon=True, + ) + else: + assert device_index is not None + stream = sd.InputStream( + device=device_index, + channels=1, + samplerate=CAPTURE_SAMPLE_RATE, + dtype="float32", + callback=audio_callback, + blocksize=int(CAPTURE_SAMPLE_RATE * 0.5), + ) def _on_dashboard_close() -> None: print("Stopping.") try: - stream.start() + if pipewire_capture_thread is not None: + pipewire_capture_thread.start() + elif stream is not None: + stream.start() print("Listening... Close the runtime window to stop.") run_runtime_dashboard( get_audio_activity=get_audio_activity_snapshot, @@ -712,14 +777,22 @@ def main() -> None: openai_realtime_client.stop() openai_realtime_client = None - try: - stream.stop() - except Exception: - pass - try: - stream.close() - except Exception: - pass + if pipewire_stop_event is not None: + pipewire_stop_event.set() + if pipewire_capture_thread is not None: + pipewire_capture_thread.join(timeout=2.0) + + if stream is not None: + try: + stream.stop() + except Exception: + pass + try: + stream.close() + except Exception: + pass + + linux_audio.teardown_virtual_input_sink() if __name__ == "__main__": |