From eabe8db2e89705df1e9be8cc501e9ab4298e4151 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 18 Feb 2026 13:37:23 +0000 Subject: [PATCH] Remove CPU fallbacks for voice note transcribe; auto/cuda/cpu fail fast - Remove _cuda_failed_models and inference-time CPU fallback - auto: try CUDA only, fail fast on RuntimeError (no CPU fallback) - cpu/cuda: use device directly, fail fast on errors - Update docs in config, .env.example, README Co-authored-by: Ali Khokhar --- .env.example | 2 +- README.md | 4 ++-- config/settings.py | 2 +- messaging/transcription.py | 34 ++++++---------------------------- 4 files changed, 10 insertions(+), 32 deletions(-) diff --git a/.env.example b/.env.example index 9d73362b..4f763fa3 100644 --- a/.env.example +++ b/.env.example @@ -38,7 +38,7 @@ WHISPER_MODEL=base HF_TOKEN="" -# WHISPER_DEVICE: "cpu" | "cuda" | "auto" (auto = try GPU, fall back to CPU) +# WHISPER_DEVICE: "cpu" | "cuda" | "auto" (auto = try CUDA, fail fast; no fallback) WHISPER_DEVICE=cpu diff --git a/README.md b/README.md index 27bc732a..5680c373 100644 --- a/README.md +++ b/README.md @@ -249,7 +249,7 @@ uv sync --extra voice |----------|-------------|---------| | `VOICE_NOTE_ENABLED` | Enable voice note handling | `true` | | `WHISPER_MODEL` | Model size: `tiny`, `base`, `small`, `medium`, `large-v2` | `base` | -| `WHISPER_DEVICE` | `cpu` \| `cuda` \| `auto` (auto = try GPU, fall back to CPU) | `cpu` | +| `WHISPER_DEVICE` | `cpu` \| `cuda` \| `auto` (auto = try CUDA, fail fast; no fallback) | `cpu` | | `HF_TOKEN` | Hugging Face token for faster model downloads (optional; [create one](https://huggingface.co/settings/tokens)) | — | --- @@ -335,7 +335,7 @@ Browse: [model.lmstudio.ai](https://model.lmstudio.ai) | `ALLOWED_TELEGRAM_USER_ID` | Allowed Telegram User ID | `""` | | `VOICE_NOTE_ENABLED` | Enable voice note handling | `true` | | `WHISPER_MODEL` | Local Whisper model size | `base` | -| `WHISPER_DEVICE` | `cpu` \| `cuda` \| `auto` | `cpu` | +| `WHISPER_DEVICE` | `cpu` \| `cuda` \| `auto` (fail fast; no fallback) | `cpu` | | `MESSAGING_RATE_LIMIT` | Messaging messages per window | `1` | | `MESSAGING_RATE_WINDOW` | Messaging window (seconds) | `1` | | `CLAUDE_WORKSPACE` | Directory for agent workspace | `./agent_workspace` | diff --git a/config/settings.py b/config/settings.py index 93e130b2..2dd40995 100644 --- a/config/settings.py +++ b/config/settings.py @@ -80,7 +80,7 @@ class Settings(BaseSettings): hf_token: str = Field(default="", validation_alias="HF_TOKEN") # Model size: "tiny" | "base" | "small" | "medium" | "large-v2" whisper_model: str = Field(default="base", validation_alias="WHISPER_MODEL") - # Device: "cpu" | "cuda" | "auto" (auto = try cuda, fall back to cpu) + # Device: "cpu" | "cuda" | "auto" (auto = try cuda, fail fast; no fallback) whisper_device: str = Field(default="cpu", validation_alias="WHISPER_DEVICE") # ==================== Bot Wrapper Config ==================== diff --git a/messaging/transcription.py b/messaging/transcription.py index 5ca5c556..a32d1298 100644 --- a/messaging/transcription.py +++ b/messaging/transcription.py @@ -15,8 +15,6 @@ MAX_AUDIO_SIZE_BYTES = 25 * 1024 * 1024 # Lazy-loaded models: (model_name, device) -> model _model_cache: dict[tuple[str, str], Any] = {} -# Models for which CUDA failed at inference; skip cuda on subsequent requests -_cuda_failed_models: set[str] = set() class _WhisperModelLike(Protocol): @@ -27,10 +25,8 @@ class _WhisperModelLike(Protocol): def _get_local_model(whisper_model: str, device: str) -> _WhisperModelLike: """Lazy-load faster-whisper model. Raises ImportError if not installed.""" - global _model_cache, _cuda_failed_models + global _model_cache resolved = device if device in ("cpu", "cuda") else "auto" - if resolved in ("cuda", "auto") and whisper_model in _cuda_failed_models: - resolved = "cpu" cache_key = (whisper_model, resolved) if cache_key not in _model_cache: try: @@ -44,12 +40,8 @@ def _get_local_model(whisper_model: str, device: str) -> _WhisperModelLike: faster_whisper = importlib.import_module("faster_whisper") WhisperModel = faster_whisper.WhisperModel if resolved == "auto": - try: - _model_cache[cache_key] = WhisperModel(whisper_model, device="cuda") - except RuntimeError: - _model_cache[cache_key] = WhisperModel( - whisper_model, device="cpu", compute_type="float32" - ) + # Try CUDA; fail fast if CUDA unavailable (no CPU fallback) + _model_cache[cache_key] = WhisperModel(whisper_model, device="cuda") elif resolved == "cpu": _model_cache[cache_key] = WhisperModel( whisper_model, device="cpu", compute_type="float32" @@ -77,7 +69,7 @@ def transcribe_audio( file_path: Path to audio file (OGG, MP3, MP4, WAV, M4A supported) mime_type: MIME type of the audio (e.g. "audio/ogg") whisper_model: Model size: "tiny", "base", "small", "medium", "large-v2" - whisper_device: "cpu" | "cuda" | "auto" (auto = try GPU, fall back to CPU) + whisper_device: "cpu" | "cuda" | "auto" (auto = try CUDA, fail fast; no fallback) Returns: Transcribed text @@ -100,23 +92,9 @@ def transcribe_audio( def _transcribe_local(file_path: Path, whisper_model: str, whisper_device: str) -> str: - """Transcribe using local faster-whisper.""" + """Transcribe using local faster-whisper. Fails fast on device errors (no fallback).""" model: _WhisperModelLike = _get_local_model(whisper_model, whisper_device) - try: - segments, _info = model.transcribe(str(file_path), beam_size=5) - except RuntimeError as e: - err_lower = str(e).lower() - if "cublas" in err_lower or "cuda" in err_lower: - # CUDA deferred load failed at inference; remember and fall back to CPU - global _model_cache, _cuda_failed_models - _cuda_failed_models.add(whisper_model) - for key in list(_model_cache): - if key[0] == whisper_model: - del _model_cache[key] - model = _get_local_model(whisper_model, "cpu") - segments, _info = model.transcribe(str(file_path), beam_size=5) - else: - raise + segments, _info = model.transcribe(str(file_path), beam_size=5) parts = [s.text for s in segments if s.text] result = " ".join(parts).strip() logger.debug(f"Local transcription: {len(result)} chars")