diff --git a/devplacepy/database/CLAUDE.md b/devplacepy/database/CLAUDE.md index 484573f1..73ea73f0 100644 --- a/devplacepy/database/CLAUDE.md +++ b/devplacepy/database/CLAUDE.md @@ -12,7 +12,9 @@ PRAGMA cache_size=-8000; -- 8MB page cache PRAGMA temp_store=MEMORY; -- temp tables in memory ``` -Configured via `dataset.connect(engine_kwargs={"connect_args": {"timeout": 30, "check_same_thread": False}}, on_connect_statements=[...])`. In addition to the pragmas above, the connection is tuned with a 30s busy timeout, an 8MB page cache, and a 256MB mmap. +Configured via `dataset.connect(engine_kwargs={"connect_args": {"timeout": 30, "check_same_thread": False}, "poolclass": NullPool}, on_connect_statements=[...])`. In addition to the pragmas above, the connection is tuned with a 30s busy timeout, an 8MB page cache, and a 256MB mmap. + +**`poolclass=NullPool` is load-bearing - never revert it to SQLAlchemy's default `QueuePool` (caused a production outage).** `dataset.Database.executable` caches ONE DBAPI connection per OS thread ID **forever** and never returns it to the pool except via `db.close()`, which nothing in this codebase calls (`dataset/database.py`: `self.connections[tid] = self.engine.connect()`). That is fine as long as the same handful of threads ever touch the DB - but FastAPI runs every sync route dependency (`get_setting` and friends, hit on nearly every request) through `anyio.to_thread.run_sync`, whose worker pool scales up and recycles threads elastically under load, and container sync (`asyncio.to_thread`) adds more. Each new thread's first query permanently claims one pool slot. With the default bounded `QueuePool` (`pool_size=5, max_overflow=10` = 15 total), a burst of concurrent load creates enough new threads that the pool fills for good within minutes, and every request thereafter - including the Docker healthcheck's own probe - blocks the full 30s pool timeout and then raises `sqlalchemy.exc.TimeoutError: QueuePool limit of size 5 overflow 10 reached`, wedging the whole app (nginx waits on an app that is waiting on itself; every external caller sees a bare connection timeout, not an HTTP error). `NullPool` removes the artificial ceiling: each `engine.connect()` opens a real, unpooled SQLite connection, so the existing "one connection cached per thread forever" behavior just works, exactly as WAL mode is designed to support. Never pass `pool_size`/`max_overflow` alongside `NullPool` (SQLAlchemy rejects them). Do not "fix" the underlying thread churn instead - that means touching the sync-dependency/threadpool model, which the hard rule below forbids. `init_db()` is idempotent - every index is created via `CREATE INDEX IF NOT EXISTS` (through the `_index()` helper, wrapped in try/except so it is safe to run on every startup regardless of table state), and seed defaults for `site_settings` only insert when the key doesn't already exist. diff --git a/devplacepy/database/core.py b/devplacepy/database/core.py index 45b99bc8..febd70ba 100644 --- a/devplacepy/database/core.py +++ b/devplacepy/database/core.py @@ -4,6 +4,7 @@ import dataset import logging from pathlib import Path from sqlalchemy import or_ +from sqlalchemy.pool import NullPool from collections import defaultdict from datetime import datetime, timedelta, timezone from devplacepy.cache import TTLCache @@ -30,6 +31,7 @@ db = dataset.connect( "timeout": 30, "check_same_thread": False, }, + "poolclass": NullPool, }, on_connect_statements=[ "PRAGMA journal_mode=WAL", diff --git a/devplacepy/services/openai_gateway/gateway.py b/devplacepy/services/openai_gateway/gateway.py index 7527dd6a..08145886 100644 --- a/devplacepy/services/openai_gateway/gateway.py +++ b/devplacepy/services/openai_gateway/gateway.py @@ -198,34 +198,31 @@ class GatewayRuntime: self.in_flight += 1 if self.in_flight > self.peak_in_flight: self.peak_in_flight = self.in_flight - wait_start = time.monotonic() connect_holder = {"ms": 0.0} attempts = 1 resp = None exc = None try: - async with sem: - timing["queue_wait_ms"] = round( - (time.monotonic() - wait_start) * 1000, 3 - ) - async def do_call(): - request = client.build_request( - method, url, headers=headers, json=json_body, content=content - ) - request.extensions["trace"] = _connect_tracer(connect_holder) - return await client.send(request) + async def do_call(): + request = client.build_request( + method, url, headers=headers, json=json_body, content=content + ) + request.extensions["trace"] = _connect_tracer(connect_holder) + return await client.send(request) - send_start = time.monotonic() - resp, exc, attempts = await retry_send( - do_call, - cfg["gateway_max_retries"], - cfg["gateway_retry_backoff_ms"], - log, - ) - timing["upstream_latency_ms"] = round( - (time.monotonic() - send_start) * 1000, 3 - ) + send_start = time.monotonic() + resp, exc, attempts, queue_wait_ms = await retry_send( + do_call, + sem, + cfg["gateway_max_retries"], + cfg["gateway_retry_backoff_ms"], + log, + ) + timing["upstream_latency_ms"] = round( + (time.monotonic() - send_start) * 1000, 3 + ) + timing["queue_wait_ms"] = round(queue_wait_ms, 3) finally: self.in_flight -= 1 timing["connect_ms"] = round(connect_holder["ms"], 3) diff --git a/devplacepy/services/openai_gateway/reliability.py b/devplacepy/services/openai_gateway/reliability.py index 22df079a..2f1c22bd 100644 --- a/devplacepy/services/openai_gateway/reliability.py +++ b/devplacepy/services/openai_gateway/reliability.py @@ -66,21 +66,30 @@ async def _backoff(backoff_ms: int, attempt: int) -> None: async def retry_send( do_call: Callable[[], Awaitable[httpx.Response]], + sem: asyncio.Semaphore, max_retries: int, backoff_ms: int, log: Optional[Callable[[str], None]] = None, -) -> tuple[Optional[httpx.Response], Optional[Exception], int]: +) -> tuple[Optional[httpx.Response], Optional[Exception], int, float]: log = log or (lambda message: None) attempts = 0 last_exc: Optional[Exception] = None + queue_wait_ms = 0.0 while attempts <= max_retries: attempts += 1 - try: - resp = await do_call() - except httpx.RequestError as exc: + wait_start = time.monotonic() + async with sem: + queue_wait_ms += (time.monotonic() - wait_start) * 1000 + try: + resp = await do_call() + exc = None + except httpx.RequestError as e: + resp = None + exc = e + if exc is not None: last_exc = exc if attempts > max_retries: - return None, exc, attempts + return None, exc, attempts, queue_wait_ms log( f"upstream connection failed, retrying ({attempts}/{max_retries}): {exc}" ) @@ -90,5 +99,5 @@ async def retry_send( log(f"upstream {resp.status_code}, retrying ({attempts}/{max_retries})") await _backoff(backoff_ms, attempts) continue - return resp, None, attempts - return None, last_exc, attempts + return resp, None, attempts, queue_wait_ms + return None, last_exc, attempts, queue_wait_ms diff --git a/nginx/nginx.conf.template b/nginx/nginx.conf.template index 8a84a2cd..c5248d5e 100644 --- a/nginx/nginx.conf.template +++ b/nginx/nginx.conf.template @@ -208,8 +208,8 @@ server { proxy_set_header X-Forwarded-Proto $scheme; proxy_http_version 1.1; proxy_set_header Connection ""; - proxy_read_timeout 120s; - proxy_send_timeout 120s; + proxy_read_timeout 900s; + proxy_send_timeout 900s; } location / { @@ -226,7 +226,7 @@ server { proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; - proxy_connect_timeout 30s; + proxy_connect_timeout 900s; proxy_read_timeout 3600s; proxy_send_timeout 3600s;