From 45b7e4cafdb4178749afc66931fe4dc10d7e64ef Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 19 Feb 2026 14:39:42 +0000 Subject: [PATCH] Make PROVIDER_MAX_CONCURRENCY required with default of 5 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `max_concurrency` is now always an `int` (default 5) — `None`/unlimited is no longer a valid state; omitting the env var uses the default - `GlobalRateLimiter`: semaphore is always created; `concurrency_slot()` no longer has None guards; log message always includes concurrency - `ProviderConfig.max_concurrency`: `int = 5` (was `int | None = None`) - `Settings.provider_max_concurrency`: `int = Field(default=5, ...)` — setting env var to an invalid value (e.g. empty string) raises - `.env.example`: uncommented `PROVIDER_MAX_CONCURRENCY=5` - README: updated config table default from `—` to `5` - Tests: removed `test_concurrency_slot_noop_when_not_configured`; updated mock settings to use `5` instead of `None` https://claude.ai/code/session_014mrF1WMNgmNjtPBuoQHsbg --- .env.example | 4 ++-- README.md | 2 +- config/settings.py | 4 ++-- providers/base.py | 2 +- providers/rate_limit.py | 26 +++++++-------------- tests/api/test_dependencies.py | 2 +- tests/providers/test_provider_rate_limit.py | 13 ----------- 7 files changed, 15 insertions(+), 38 deletions(-) diff --git a/.env.example b/.env.example index 6bc114c3..bf2569b8 100644 --- a/.env.example +++ b/.env.example @@ -2,8 +2,8 @@ PROVIDER_TYPE="nvidia_nim" PROVIDER_RATE_LIMIT=40 PROVIDER_RATE_WINDOW=60 -# Maximum simultaneous open provider streams (unset = unlimited) -# PROVIDER_MAX_CONCURRENCY=3 +# Maximum simultaneous open provider streams (default: 5) +PROVIDER_MAX_CONCURRENCY=5 # HTTP client timeouts (seconds) for provider API requests diff --git a/README.md b/README.md index 4adf3ab8..3035a41d 100644 --- a/README.md +++ b/README.md @@ -320,7 +320,7 @@ Browse: [model.lmstudio.ai](https://model.lmstudio.ai) | `LM_STUDIO_BASE_URL` | LM Studio server URL | `http://localhost:1234/v1` | | `PROVIDER_RATE_LIMIT` | LLM API requests per window | `40` | | `PROVIDER_RATE_WINDOW` | Rate limit window (seconds) | `60` | -| `PROVIDER_MAX_CONCURRENCY` | Max simultaneous open provider streams (unset = unlimited) | — | +| `PROVIDER_MAX_CONCURRENCY` | Max simultaneous open provider streams | `5` | | `HTTP_READ_TIMEOUT` | Read timeout for provider API requests (seconds) | `300` | | `HTTP_WRITE_TIMEOUT` | Write timeout for provider API requests (seconds) | `10` | | `HTTP_CONNECT_TIMEOUT` | Connect timeout for provider API requests (seconds) | `2` | diff --git a/config/settings.py b/config/settings.py index cde9b9dd..4f367bd7 100644 --- a/config/settings.py +++ b/config/settings.py @@ -48,8 +48,8 @@ class Settings(BaseSettings): provider_rate_window: int = Field( default=60, validation_alias="PROVIDER_RATE_WINDOW" ) - provider_max_concurrency: int | None = Field( - default=None, validation_alias="PROVIDER_MAX_CONCURRENCY" + provider_max_concurrency: int = Field( + default=5, validation_alias="PROVIDER_MAX_CONCURRENCY" ) # ==================== HTTP Client Timeouts ==================== diff --git a/providers/base.py b/providers/base.py index ad4a7ebc..83c655af 100644 --- a/providers/base.py +++ b/providers/base.py @@ -18,7 +18,7 @@ class ProviderConfig(BaseModel): base_url: str | None = None rate_limit: int | None = None rate_window: int = 60 - max_concurrency: int | None = None + max_concurrency: int = 5 http_read_timeout: float = 300.0 http_write_timeout: float = 10.0 http_connect_timeout: float = 2.0 diff --git a/providers/rate_limit.py b/providers/rate_limit.py index 02123e25..110307a1 100644 --- a/providers/rate_limit.py +++ b/providers/rate_limit.py @@ -34,7 +34,7 @@ class GlobalRateLimiter: self, rate_limit: int = 40, rate_window: float = 60.0, - max_concurrency: int | None = None, + max_concurrency: int = 5, ): # Prevent double initialization in singleton if hasattr(self, "_initialized"): @@ -44,7 +44,7 @@ class GlobalRateLimiter: raise ValueError("rate_limit must be > 0") if rate_window <= 0: raise ValueError("rate_window must be > 0") - if max_concurrency is not None and max_concurrency <= 0: + if max_concurrency <= 0: raise ValueError("max_concurrency must be > 0") self._rate_limit = rate_limit @@ -53,18 +53,11 @@ class GlobalRateLimiter: self._request_times: deque[float] = deque() self._blocked_until: float = 0 self._lock = asyncio.Lock() - self._concurrency_sem: asyncio.Semaphore | None = ( - asyncio.Semaphore(max_concurrency) if max_concurrency is not None else None - ) + self._concurrency_sem = asyncio.Semaphore(max_concurrency) self._initialized = True - concurrency_info = ( - f", max_concurrency={max_concurrency}" - if max_concurrency is not None - else "" - ) logger.info( - f"GlobalRateLimiter (Provider) initialized ({rate_limit} req / {rate_window}s{concurrency_info})" + f"GlobalRateLimiter (Provider) initialized ({rate_limit} req / {rate_window}s, max_concurrency={max_concurrency})" ) @classmethod @@ -72,7 +65,7 @@ class GlobalRateLimiter: cls, rate_limit: int | None = None, rate_window: float | None = None, - max_concurrency: int | None = None, + max_concurrency: int = 5, ) -> GlobalRateLimiter: """Get or create the singleton instance. @@ -167,16 +160,13 @@ class GlobalRateLimiter: async def concurrency_slot(self) -> AsyncIterator[None]: """Async context manager that holds one concurrency slot for a stream. - Blocks until a slot is available when max_concurrency is set. - Is a no-op when max_concurrency was not configured. + Blocks until a slot is available (controlled by max_concurrency). """ - if self._concurrency_sem is not None: - await self._concurrency_sem.acquire() + await self._concurrency_sem.acquire() try: yield finally: - if self._concurrency_sem is not None: - self._concurrency_sem.release() + self._concurrency_sem.release() async def execute_with_retry( self, diff --git a/tests/api/test_dependencies.py b/tests/api/test_dependencies.py index 2b36cd58..e50dffad 100644 --- a/tests/api/test_dependencies.py +++ b/tests/api/test_dependencies.py @@ -17,7 +17,7 @@ def _make_mock_settings(**overrides): mock.nvidia_nim_api_key = "test_key" mock.provider_rate_limit = 40 mock.provider_rate_window = 60 - mock.provider_max_concurrency = None + mock.provider_max_concurrency = 5 mock.open_router_api_key = "test_openrouter_key" mock.lm_studio_base_url = "http://localhost:1234/v1" mock.nim = NimSettings() diff --git a/tests/providers/test_provider_rate_limit.py b/tests/providers/test_provider_rate_limit.py index cd0e0706..c8d2061f 100644 --- a/tests/providers/test_provider_rate_limit.py +++ b/tests/providers/test_provider_rate_limit.py @@ -259,19 +259,6 @@ class TestProviderRateLimiter: with pytest.raises(ValueError, match="max_concurrency must be > 0"): GlobalRateLimiter(rate_limit=10, rate_window=60, max_concurrency=0) - @pytest.mark.asyncio - async def test_concurrency_slot_noop_when_not_configured(self): - """concurrency_slot() is a no-op when max_concurrency is None.""" - GlobalRateLimiter.reset_instance() - limiter = GlobalRateLimiter.get_instance(rate_limit=100, rate_window=60) - assert limiter._concurrency_sem is None - - # Should not block and complete immediately - entered = False - async with limiter.concurrency_slot(): - entered = True - assert entered - @pytest.mark.asyncio async def test_concurrency_slot_limits_simultaneous_streams(self): """At most max_concurrency streams can hold a slot simultaneously."""