Remove CPU fallbacks for voice note transcribe; auto/cuda/cpu fail fast

- Remove _cuda_failed_models and inference-time CPU fallback
- auto: try CUDA only, fail fast on RuntimeError (no CPU fallback)
- cpu/cuda: use device directly, fail fast on errors
- Update docs in config, .env.example, README

Co-authored-by: Ali Khokhar <alishahryar2@gmail.com>

.env.example

@@ -38,7 +38,7 @@ WHISPER_MODEL=base
 HF_TOKEN=""
 
 
-# WHISPER_DEVICE: "cpu" | "cuda" | "auto" (auto = try GPU, fall back to CPU)
+# WHISPER_DEVICE: "cpu" | "cuda" | "auto" (auto = try CUDA, fail fast; no fallback)
 WHISPER_DEVICE=cpu
 
 

README.md

@@ -249,7 +249,7 @@ uv sync --extra voice
 |----------|-------------|---------|
 | `VOICE_NOTE_ENABLED` | Enable voice note handling | `true` |
 | `WHISPER_MODEL` | Model size: `tiny`, `base`, `small`, `medium`, `large-v2` | `base` |
-| `WHISPER_DEVICE` | `cpu` \| `cuda` \| `auto` (auto = try GPU, fall back to CPU) | `cpu` |
+| `WHISPER_DEVICE` | `cpu` \| `cuda` \| `auto` (auto = try CUDA, fail fast; no fallback) | `cpu` |
 | `HF_TOKEN` | Hugging Face token for faster model downloads (optional; [create one](https://huggingface.co/settings/tokens)) | — |
 
 ---
@@ -335,7 +335,7 @@ Browse: [model.lmstudio.ai](https://model.lmstudio.ai)
 | `ALLOWED_TELEGRAM_USER_ID` | Allowed Telegram User ID | `""` |
 | `VOICE_NOTE_ENABLED` | Enable voice note handling | `true` |
 | `WHISPER_MODEL` | Local Whisper model size | `base` |
-| `WHISPER_DEVICE` | `cpu` \| `cuda` \| `auto` | `cpu` |
+| `WHISPER_DEVICE` | `cpu` \| `cuda` \| `auto` (fail fast; no fallback) | `cpu` |
 | `MESSAGING_RATE_LIMIT` | Messaging messages per window | `1` |
 | `MESSAGING_RATE_WINDOW` | Messaging window (seconds) | `1` |
 | `CLAUDE_WORKSPACE` | Directory for agent workspace | `./agent_workspace` |

config/settings.py

@@ -80,7 +80,7 @@ class Settings(BaseSettings):
     hf_token: str = Field(default="", validation_alias="HF_TOKEN")
     # Model size: "tiny" | "base" | "small" | "medium" | "large-v2"
     whisper_model: str = Field(default="base", validation_alias="WHISPER_MODEL")
-    # Device: "cpu" | "cuda" | "auto" (auto = try cuda, fall back to cpu)
+    # Device: "cpu" | "cuda" | "auto" (auto = try cuda, fail fast; no fallback)
     whisper_device: str = Field(default="cpu", validation_alias="WHISPER_DEVICE")
 
     # ==================== Bot Wrapper Config ====================

messaging/transcription.py

@@ -15,8 +15,6 @@ MAX_AUDIO_SIZE_BYTES = 25 * 1024 * 1024
 
 # Lazy-loaded models: (model_name, device) -> model
 _model_cache: dict[tuple[str, str], Any] = {}
-# Models for which CUDA failed at inference; skip cuda on subsequent requests
-_cuda_failed_models: set[str] = set()
 
 
 class _WhisperModelLike(Protocol):
@@ -27,10 +25,8 @@ class _WhisperModelLike(Protocol):
 
 def _get_local_model(whisper_model: str, device: str) -> _WhisperModelLike:
     """Lazy-load faster-whisper model. Raises ImportError if not installed."""
-    global _model_cache, _cuda_failed_models
+    global _model_cache
     resolved = device if device in ("cpu", "cuda") else "auto"
-    if resolved in ("cuda", "auto") and whisper_model in _cuda_failed_models:
-        resolved = "cpu"
     cache_key = (whisper_model, resolved)
     if cache_key not in _model_cache:
         try:
@@ -44,12 +40,8 @@ def _get_local_model(whisper_model: str, device: str) -> _WhisperModelLike:
             faster_whisper = importlib.import_module("faster_whisper")
             WhisperModel = faster_whisper.WhisperModel
             if resolved == "auto":
-                try:
-                    _model_cache[cache_key] = WhisperModel(whisper_model, device="cuda")
-                except RuntimeError:
-                    _model_cache[cache_key] = WhisperModel(
-                        whisper_model, device="cpu", compute_type="float32"
-                    )
+                # Try CUDA; fail fast if CUDA unavailable (no CPU fallback)
+                _model_cache[cache_key] = WhisperModel(whisper_model, device="cuda")
             elif resolved == "cpu":
                 _model_cache[cache_key] = WhisperModel(
                     whisper_model, device="cpu", compute_type="float32"
@@ -77,7 +69,7 @@ def transcribe_audio(
         file_path: Path to audio file (OGG, MP3, MP4, WAV, M4A supported)
         mime_type: MIME type of the audio (e.g. "audio/ogg")
         whisper_model: Model size: "tiny", "base", "small", "medium", "large-v2"
-        whisper_device: "cpu" | "cuda" | "auto" (auto = try GPU, fall back to CPU)
+        whisper_device: "cpu" | "cuda" | "auto" (auto = try CUDA, fail fast; no fallback)
 
     Returns:
         Transcribed text
@@ -100,23 +92,9 @@ def transcribe_audio(
 
 
 def _transcribe_local(file_path: Path, whisper_model: str, whisper_device: str) -> str:
-    """Transcribe using local faster-whisper."""
+    """Transcribe using local faster-whisper. Fails fast on device errors (no fallback)."""
     model: _WhisperModelLike = _get_local_model(whisper_model, whisper_device)
-    try:
-        segments, _info = model.transcribe(str(file_path), beam_size=5)
-    except RuntimeError as e:
-        err_lower = str(e).lower()
-        if "cublas" in err_lower or "cuda" in err_lower:
-            # CUDA deferred load failed at inference; remember and fall back to CPU
-            global _model_cache, _cuda_failed_models
-            _cuda_failed_models.add(whisper_model)
-            for key in list(_model_cache):
-                if key[0] == whisper_model:
-                    del _model_cache[key]
-            model = _get_local_model(whisper_model, "cpu")
-            segments, _info = model.transcribe(str(file_path), beam_size=5)
-        else:
-            raise
+    segments, _info = model.transcribe(str(file_path), beam_size=5)
     parts = [s.text for s in segments if s.text]
     result = " ".join(parts).strip()
     logger.debug(f"Local transcription: {len(result)} chars")