Waw

devplacepy/database/CLAUDE.md

@@ -12,7 +12,9 @@ PRAGMA cache_size=-8000;       -- 8MB page cache
 PRAGMA temp_store=MEMORY;      -- temp tables in memory
 ```
 
-Configured via `dataset.connect(engine_kwargs={"connect_args": {"timeout": 30, "check_same_thread": False}}, on_connect_statements=[...])`. In addition to the pragmas above, the connection is tuned with a 30s busy timeout, an 8MB page cache, and a 256MB mmap.
+Configured via `dataset.connect(engine_kwargs={"connect_args": {"timeout": 30, "check_same_thread": False}, "poolclass": NullPool}, on_connect_statements=[...])`. In addition to the pragmas above, the connection is tuned with a 30s busy timeout, an 8MB page cache, and a 256MB mmap.
+
+**`poolclass=NullPool` is load-bearing - never revert it to SQLAlchemy's default `QueuePool` (caused a production outage).** `dataset.Database.executable` caches ONE DBAPI connection per OS thread ID **forever** and never returns it to the pool except via `db.close()`, which nothing in this codebase calls (`dataset/database.py`: `self.connections[tid] = self.engine.connect()`). That is fine as long as the same handful of threads ever touch the DB - but FastAPI runs every sync route dependency (`get_setting` and friends, hit on nearly every request) through `anyio.to_thread.run_sync`, whose worker pool scales up and recycles threads elastically under load, and container sync (`asyncio.to_thread`) adds more. Each new thread's first query permanently claims one pool slot. With the default bounded `QueuePool` (`pool_size=5, max_overflow=10` = 15 total), a burst of concurrent load creates enough new threads that the pool fills for good within minutes, and every request thereafter - including the Docker healthcheck's own probe - blocks the full 30s pool timeout and then raises `sqlalchemy.exc.TimeoutError: QueuePool limit of size 5 overflow 10 reached`, wedging the whole app (nginx waits on an app that is waiting on itself; every external caller sees a bare connection timeout, not an HTTP error). `NullPool` removes the artificial ceiling: each `engine.connect()` opens a real, unpooled SQLite connection, so the existing "one connection cached per thread forever" behavior just works, exactly as WAL mode is designed to support. Never pass `pool_size`/`max_overflow` alongside `NullPool` (SQLAlchemy rejects them). Do not "fix" the underlying thread churn instead - that means touching the sync-dependency/threadpool model, which the hard rule below forbids.
 
 `init_db()` is idempotent - every index is created via `CREATE INDEX IF NOT EXISTS` (through the `_index()` helper, wrapped in try/except so it is safe to run on every startup regardless of table state), and seed defaults for `site_settings` only insert when the key doesn't already exist.
 

devplacepy/database/core.py

@@ -4,6 +4,7 @@ import dataset
 import logging
 from pathlib import Path
 from sqlalchemy import or_
+from sqlalchemy.pool import NullPool
 from collections import defaultdict
 from datetime import datetime, timedelta, timezone
 from devplacepy.cache import TTLCache
@@ -30,6 +31,7 @@ db = dataset.connect(
             "timeout": 30,
             "check_same_thread": False,
         },
+        "poolclass": NullPool,
     },
     on_connect_statements=[
         "PRAGMA journal_mode=WAL",

devplacepy/services/openai_gateway/gateway.py

@@ -198,34 +198,31 @@ class GatewayRuntime:
         self.in_flight += 1
         if self.in_flight > self.peak_in_flight:
             self.peak_in_flight = self.in_flight
-        wait_start = time.monotonic()
         connect_holder = {"ms": 0.0}
         attempts = 1
         resp = None
         exc = None
         try:
-            async with sem:
-                timing["queue_wait_ms"] = round(
-                    (time.monotonic() - wait_start) * 1000, 3
-                )
 
-                async def do_call():
+            async def do_call():
-                    request = client.build_request(
+                request = client.build_request(
-                        method, url, headers=headers, json=json_body, content=content
+                    method, url, headers=headers, json=json_body, content=content
-                    )
+                )
-                    request.extensions["trace"] = _connect_tracer(connect_holder)
+                request.extensions["trace"] = _connect_tracer(connect_holder)
-                    return await client.send(request)
+                return await client.send(request)
 
-                send_start = time.monotonic()
+            send_start = time.monotonic()
-                resp, exc, attempts = await retry_send(
+            resp, exc, attempts, queue_wait_ms = await retry_send(
-                    do_call,
+                do_call,
-                    cfg["gateway_max_retries"],
+                sem,
-                    cfg["gateway_retry_backoff_ms"],
+                cfg["gateway_max_retries"],
-                    log,
+                cfg["gateway_retry_backoff_ms"],
-                )
+                log,
-                timing["upstream_latency_ms"] = round(
+            )
-                    (time.monotonic() - send_start) * 1000, 3
+            timing["upstream_latency_ms"] = round(
-                )
+                (time.monotonic() - send_start) * 1000, 3
+            )
+            timing["queue_wait_ms"] = round(queue_wait_ms, 3)
         finally:
             self.in_flight -= 1
             timing["connect_ms"] = round(connect_holder["ms"], 3)

devplacepy/services/openai_gateway/reliability.py

@@ -66,21 +66,30 @@ async def _backoff(backoff_ms: int, attempt: int) -> None:
 
 async def retry_send(
     do_call: Callable[[], Awaitable[httpx.Response]],
+    sem: asyncio.Semaphore,
     max_retries: int,
     backoff_ms: int,
     log: Optional[Callable[[str], None]] = None,
-) -> tuple[Optional[httpx.Response], Optional[Exception], int]:
+) -> tuple[Optional[httpx.Response], Optional[Exception], int, float]:
     log = log or (lambda message: None)
     attempts = 0
     last_exc: Optional[Exception] = None
+    queue_wait_ms = 0.0
     while attempts <= max_retries:
         attempts += 1
-        try:
+        wait_start = time.monotonic()
-            resp = await do_call()
+        async with sem:
-        except httpx.RequestError as exc:
+            queue_wait_ms += (time.monotonic() - wait_start) * 1000
+            try:
+                resp = await do_call()
+                exc = None
+            except httpx.RequestError as e:
+                resp = None
+                exc = e
+        if exc is not None:
             last_exc = exc
             if attempts > max_retries:
-                return None, exc, attempts
+                return None, exc, attempts, queue_wait_ms
             log(
                 f"upstream connection failed, retrying ({attempts}/{max_retries}): {exc}"
             )
@@ -90,5 +99,5 @@ async def retry_send(
             log(f"upstream {resp.status_code}, retrying ({attempts}/{max_retries})")
             await _backoff(backoff_ms, attempts)
             continue
-        return resp, None, attempts
+        return resp, None, attempts, queue_wait_ms
-    return None, last_exc, attempts
+    return None, last_exc, attempts, queue_wait_ms

nginx/nginx.conf.template

@@ -208,8 +208,8 @@ server {
         proxy_set_header X-Forwarded-Proto $scheme;
         proxy_http_version 1.1;
         proxy_set_header Connection "";
-        proxy_read_timeout 120s;
+        proxy_read_timeout 900s;
-        proxy_send_timeout 120s;
+        proxy_send_timeout 900s;
     }
 
     location / {
@@ -226,7 +226,7 @@ server {
         proxy_set_header Upgrade $http_upgrade;
         proxy_set_header Connection $connection_upgrade;
 
-        proxy_connect_timeout 30s;
+        proxy_connect_timeout 900s;
         proxy_read_timeout 3600s;
         proxy_send_timeout 3600s;