Update

devplacepy/database/CLAUDE.md

@@ -99,7 +99,7 @@ if "comments" not in db.tables:
 
 - **NEVER index the bare `deleted_at` column - use a PARTIAL trash index `WHERE deleted_at IS NOT NULL`.** `ensure_soft_delete_columns` creates `idx_<table>_trash ON (deleted_at) WHERE deleted_at IS NOT NULL` (and drops any legacy full `idx_<table>_deleted`). A full `deleted_at` index is a planner hazard: the column is one giant `NULL` bucket plus many unique delete-timestamps, so `sqlite_stat1` mis-estimates `deleted_at IS NULL` as returning ~2 rows and the planner picks that index for live reads, then `USE TEMP B-TREE FOR ORDER BY` to sort the whole live set (the global feed was doing exactly this, with 82% of posts soft-deleted). The partial index serves the admin Trash view (`deleted_at IS NOT NULL`) cheaply and stops poisoning live `IS NULL` queries.
 
-- **For "live, newest-first" listings add a composite or live-partial index that includes the sort column.** A `WHERE deleted_at IS NULL ORDER BY created_at` query needs the ordering in the index or it filesorts. Posts use a partial `idx_posts_live_created ON (created_at) WHERE deleted_at IS NULL` (feed) plus `idx_posts_user_created (user_uid, created_at)` (profile); comments use `idx_comments_target_created (target_type, target_uid, created_at)`; votes use `idx_votes_user_target (user_uid, target_uid)` (the per-user "my_vote" check on every card); notifications/gists/projects use `(user_uid, created_at)`. All were verified to drop the `USE TEMP B-TREE FOR ORDER BY` step.
+- **For "live, newest-first" listings add a composite or live-partial index that includes the sort column.** A `WHERE deleted_at IS NULL ORDER BY created_at` query needs the ordering in the index or it filesorts. Posts use a partial `idx_posts_live_created ON (created_at) WHERE deleted_at IS NULL` (feed) plus `idx_posts_user_created (user_uid, created_at)` (profile); comments use `idx_comments_target_created (target_type, target_uid, created_at)`; votes use `idx_votes_user_target (user_uid, target_uid)` (the per-user "my_vote" check on every card); notifications/gists/projects use `(user_uid, created_at)`; follows use `idx_follows_follower_created (follower_uid, created_at)` + `idx_follows_following_created (following_uid, created_at)` (the followers/following tabs sort newest-first; the legacy single-column follower/following indexes were dropped as redundant prefixes). All were verified to drop the `USE TEMP B-TREE FOR ORDER BY` step.
 
 - **Index the non-`uid` lookup keys too, not just the sort/owner columns.** A demand-vs-supply audit added the last missing single-key lookups: the `resolve_by_slug` hot path filters `slug` on content detail pages, so posts/gists/news/projects each get `idx_<table>_slug (slug)`; `get_setting`/`set_setting` filter `key`, so `idx_site_settings_key (key)`; the container store's `find_one(slug=)`/`find_one(name=)` fallbacks get `idx_instances_slug`/`idx_instances_name`. The DM thread load `find(sender_uid=, receiver_uid=)` gets the covering composites `idx_messages_conversation (sender_uid, receiver_uid)` + `idx_messages_conversation_rev (receiver_uid, sender_uid)` (the read-flag `UPDATE` uses the reverse); the badge-has check gets `idx_badges_user_name (user_uid, badge_name)`; the admin user list `ORDER BY -created_at` gets `idx_users_created_at (created_at)` (the existing `(role, created_at)` cannot serve a full-table created_at sort). All are non-unique so `_index` always creates them even if legacy duplicate data exists. Column sets already resolved to ~1 row by an existing prefix index (votes `+target_type`, game_quests `+kind`, poll_options `position`) are intentionally left uncovered - a trailing column there only adds write cost.
 

devplacepy/database/__init__.py

@@ -5,7 +5,7 @@ from .core import refresh_snapshot, _local_cache_versions, _cache_version_cache,
 from .settings import _settings_cache, get_setting, get_int_setting, set_setting, clear_settings_cache, internal_gateway_key
 from .users import get_users_by_uids, _admins_cache, invalidate_admins_cache, get_admin_uids, set_user_timezone, set_last_seen, get_online_users, get_primary_admin_uid, search_users_by_username
 from .relations import _relations_cache, get_user_relations, get_blocked_uids, get_muted_uids, get_silenced_uids, invalidate_user_relations
-from .pagination import PAGE_SIZE, paginate, interleave_by_author, paginate_diverse, get_user_post_count, build_pagination
+from .pagination import PAGE_SIZE, paginate, interleave_by_author, paginate_diverse, get_user_post_count, clear_user_post_count, build_pagination
 from .soft_delete import SOFT_DELETE_TABLES, ensure_soft_delete_columns, soft_delete, soft_delete_in, restore, purge, list_deleted, count_deleted, restore_event, purge_event
 from .engagement import _comment_count_cache, get_comment_counts_by_post_uids, get_post_counts_by_user_uids, get_vote_counts, get_user_votes, get_reactions_by_targets, get_user_bookmarks, get_polls_by_post_uids, get_poll_for_post
 from .usage import _add_usage, _get_usage, add_correction_usage, get_correction_usage, add_modifier_usage, get_modifier_usage, NEWS_USAGE_KEY, add_news_usage, get_news_usage, ISSUE_USAGE_KEY, add_issue_usage, get_issue_usage, SEO_USAGE_KEY, add_seo_usage, get_seo_usage, AWARD_USAGE_KEY, add_award_usage, get_award_usage
@@ -36,7 +36,7 @@ from .follows import get_follow_counts, get_follow_list, get_following_among
 from .deepsearch import _ds_now, create_deepsearch_session, update_deepsearch_session, get_deepsearch_session, add_deepsearch_message, get_deepsearch_messages, get_cached_deepsearch_url, upsert_deepsearch_url_cache
 from .ranking import VOTABLE_TARGETS, STAR_TARGETS, _authors_cache, _ranked_authors, _rank_map, get_top_authors, get_leaderboard, get_user_rank, get_user_stars, clear_user_stars, update_target_stars, soft_delete_engagement, delete_engagement, get_target_owner_uid
 from .comments import _drop_blocked, _build_comment_items, load_comments, get_recent_comments_by_target_uids, get_recent_comments_by_post_uids, load_comments_by_target_uids
-from .content import resolve_by_slug, resolve_object_url, get_uids_by_username_match, text_search_clause, get_daily_topic, get_featured_news
+from .content import resolve_by_slug, resolve_object_url, get_uids_by_username_match, text_search_clause, get_daily_topic, get_featured_news, get_trending_topics
 from .attachments_data import get_attachments, get_attachments_by_type, get_news_images_by_uids, delete_attachment_record, delete_attachments, _delete_attachment_file, get_user_media, get_deleted_media
 from .stats import _stats_cache, get_site_stats, _analytics_cache, get_platform_analytics, _gist_languages_cache, get_gist_languages
 from .schema import BUG_TABLE_RENAMES, migrate_bug_tables_to_issue_tables, init_db, _refresh_query_planner_stats, OLD_GATEWAY_URL, migrate_ai_gateway_settings, backfill_api_keys, _backfill_gamification
@@ -98,6 +98,7 @@ __all__ = [
     "interleave_by_author",
     "paginate_diverse",
     "get_user_post_count",
+    "clear_user_post_count",
     "build_pagination",
     "SOFT_DELETE_TABLES",
     "ensure_soft_delete_columns",
@@ -225,6 +226,7 @@ __all__ = [
     "text_search_clause",
     "get_daily_topic",
     "get_featured_news",
+    "get_trending_topics",
     "get_attachments",
     "get_attachments_by_type",
     "get_news_images_by_uids",

devplacepy/database/awards.py

@@ -2,6 +2,7 @@
 
 from datetime import datetime, timedelta, timezone
 
+from devplacepy.cache import TTLCache
 from devplacepy.config import (
     AWARD_DISPLAY_HOURS_DEFAULT,
     AWARD_GIVE_COOLDOWN_HOURS_DEFAULT,
@@ -111,10 +112,22 @@ def recompute_user_award_stats(receiver_uid: str) -> None:
     users.update(payload, ["uid"])
 
 
+_prominence_cache = TTLCache(ttl=15, max_size=500)
+
+
 def award_is_prominent(user: dict | None) -> bool:
     if not user or not user.get("last_award_at") or not user.get("last_award_uid"):
         return False
-    award = resolve_by_slug(_awards_table(), user["last_award_uid"])
+    cached = _prominence_cache.get(user["last_award_uid"])
+    if cached is not None:
+        return cached
+    prominent = _compute_prominence(user["last_award_uid"])
+    _prominence_cache.set(user["last_award_uid"], prominent)
+    return prominent
+
+
+def _compute_prominence(award_uid: str) -> bool:
+    award = resolve_by_slug(_awards_table(), award_uid)
     if not award or not award.get("generated_at"):
         return False
     try:

devplacepy/database/comments.py

@@ -126,7 +126,7 @@ def load_comments_by_target_uids(target_type, target_uids, user=None):
     params["tt"] = target_type
     raw = list(
         db.query(
-            f"SELECT * FROM comments WHERE target_type=:tt AND target_uid IN ({placeholders}) ORDER BY created_at",
+            f"SELECT * FROM comments WHERE target_type=:tt AND target_uid IN ({placeholders}) AND deleted_at IS NULL ORDER BY created_at",
             **params,
         )
     )

devplacepy/database/content.py

@@ -1,7 +1,14 @@
 # retoor <retoor@molodetz.nl>
 
+from collections import Counter
+
+from devplacepy.cache import TTLCache
+
 from .core import db, get_table, or_
 
+_daily_topic_cache = TTLCache(ttl=60, max_size=1)
+_trending_cache = TTLCache(ttl=15, max_size=1)
+
 
 def resolve_by_slug(table, slug, include_deleted=False):
     has_soft_delete = table.has_column("deleted_at")
@@ -77,6 +84,15 @@ def text_search_clause(
 
 
 def get_daily_topic():
+    cached = _daily_topic_cache.get("topic")
+    if cached is not None:
+        return cached
+    topic = _load_daily_topic()
+    _daily_topic_cache.set("topic", topic)
+    return topic
+
+
+def _load_daily_topic():
     if "news" in db.tables:
         article = db["news"].find_one(
             status="published", deleted_at=None, order_by=["-synced_at"]
@@ -128,3 +144,24 @@ def get_featured_news(limit=5):
             }
         )
     return articles
+
+
+def get_trending_topics(limit: int = 6) -> list[dict]:
+    cached = _trending_cache.get("topics")
+    if cached is not None:
+        return cached[:limit]
+    if "posts" not in db.tables or "topic" not in db["posts"].columns:
+        return []
+    rows = db.query(
+        "SELECT topic FROM posts WHERE deleted_at IS NULL "
+        "AND topic IS NOT NULL AND topic != '' "
+        "ORDER BY created_at DESC LIMIT 200"
+    )
+    counter: Counter[str] = Counter()
+    for row in rows:
+        topic = (row["topic"] or "").strip()
+        if topic:
+            counter[topic] += 1
+    topics = [{"topic": t, "count": c} for t, c in counter.most_common(limit)]
+    _trending_cache.set("topics", topics)
+    return topics

devplacepy/database/core.py

@@ -76,18 +76,14 @@ def get_cache_version(name: str) -> int:
     try:
         _ensure_cache_state()
         with db:
-            row = next(
-                iter(
-                    db.query(
-                        "SELECT version FROM cache_state WHERE name = :name", name=name
-                    )
-                ),
-                None,
-            )
-        version = int(row["version"]) if row else 0
+            rows = list(db.query("SELECT name, version FROM cache_state"))
+        versions = {row["name"]: int(row["version"]) for row in rows}
     except Exception as e:
         logger.warning(f"Could not read cache version {name}: {e}")
         return 0
+    for key, version in versions.items():
+        _cache_version_cache.set(key, version)
+    version = versions.get(name, 0)
     _cache_version_cache.set(name, version)
     return version
 

devplacepy/database/pagination.py

@@ -1,5 +1,6 @@
 # retoor <retoor@molodetz.nl>
 
+from devplacepy.cache import TTLCache
 from .core import db, get_table
 from .relations import get_blocked_uids
 
@@ -7,6 +8,9 @@ from .relations import get_blocked_uids
 PAGE_SIZE = 25
 
 
+_user_post_count_cache = TTLCache(ttl=15, max_size=2000)
+
+
 def paginate(
     table,
     *clauses,
@@ -74,10 +78,19 @@ def paginate_diverse(
     return interleave_by_author(rows, uid_key=uid_key), next_cursor
 
 
+def clear_user_post_count(user_uid: str) -> None:
+    _user_post_count_cache.pop(user_uid)
+
+
 def get_user_post_count(user_uid: str) -> int:
+    cached = _user_post_count_cache.get(user_uid)
+    if cached is not None:
+        return cached
     if "posts" not in db.tables:
         return 0
-    return get_table("posts").count(user_uid=user_uid, deleted_at=None)
+    count = get_table("posts").count(user_uid=user_uid, deleted_at=None)
+    _user_post_count_cache.set(user_uid, count)
+    return count
 
 
 def build_pagination(page, total, per_page=25):

devplacepy/database/ranking.py

@@ -16,7 +16,7 @@ VOTABLE_TARGETS: dict[str, str] = {
 STAR_TARGETS: set[str] = {"post", "project", "gist"}
 
 
-_authors_cache = TTLCache(ttl=15, max_size=200)
+_authors_cache = TTLCache(ttl=60, max_size=200)
 
 
 _stars_cache = TTLCache(ttl=15, max_size=2000)
@@ -151,17 +151,19 @@ def delete_engagement(target_type: str, target_uids: list) -> None:
     if "reactions" in db.tables:
         placeholders, params = _in_clause(uids)
         params["tt"] = target_type
-        db.query(
-            f"DELETE FROM reactions WHERE target_type=:tt AND target_uid IN ({placeholders})",
-            **params,
-        )
+        with db:
+            db.query(
+                f"DELETE FROM reactions WHERE target_type=:tt AND target_uid IN ({placeholders})",
+                **params,
+            )
     if "bookmarks" in db.tables:
         placeholders, params = _in_clause(uids)
         params["tt"] = target_type
-        db.query(
-            f"DELETE FROM bookmarks WHERE target_type=:tt AND target_uid IN ({placeholders})",
-            **params,
-        )
+        with db:
+            db.query(
+                f"DELETE FROM bookmarks WHERE target_type=:tt AND target_uid IN ({placeholders})",
+                **params,
+            )
     if target_type == "post" and "polls" in db.tables:
         for uid in uids:
             for poll in db["polls"].find(post_uid=uid):

devplacepy/database/remote.py

@@ -0,0 +1,348 @@
+# retoor <retoor@molodetz.nl>
+
+import inspect
+import os
+
+import httpx
+
+from devplacepy.cache import TTLCache
+from devplacepy_services.base.db_codec import (
+    decode_value,
+    encode_args,
+    is_write,
+    is_write_sql,
+)
+
+_SERVICE_URL = os.environ.get("DEVPLACE_DB_SERVICE_URL", "http://127.0.0.1:10601").rstrip("/")
+_INTERNAL_KEY = os.environ.get("DEVPLACE_GATEWAY_INTERNAL_KEY", "").strip()
+_CLIENT: httpx.Client | None = None
+
+# Section 7.3: settings reads tolerate up to 5s staleness. patch_module()
+# generically RPCs every devplacepy.database call, bypassing the local
+# TTL cache get_setting/get_int_setting had in-process - without this,
+# every settings read (rate limiting, maintenance mode, admin dashboards)
+# pays a full HTTP round trip to the database broker.
+_SETTINGS_CACHE_TTL_SECONDS = 5
+_SETTINGS_CACHE = TTLCache(ttl=_SETTINGS_CACHE_TTL_SECONDS, max_size=512)
+_CACHED_SETTINGS_FNS = frozenset({"get_setting", "get_int_setting"})
+
+
+def _service_url() -> str:
+    return os.environ.get("DEVPLACE_DB_SERVICE_URL", _SERVICE_URL).rstrip("/")
+
+
+def _headers() -> dict[str, str]:
+    headers: dict[str, str] = {}
+    key = os.environ.get("DEVPLACE_GATEWAY_INTERNAL_KEY", _INTERNAL_KEY).strip()
+    if key:
+        headers["X-Internal-Key"] = key
+    return headers
+
+
+def _client() -> httpx.Client:
+    global _CLIENT
+    if _CLIENT is None:
+        _CLIENT = httpx.Client(timeout=30.0)
+    return _CLIENT
+
+
+def _post(path: str, body: dict) -> object:
+    response = _client().post(
+        f"{_service_url()}/{path.lstrip('/')}",
+        json=body,
+        headers=_headers(),
+    )
+    if response.status_code >= 400:
+        payload = response.json() if response.content else {}
+        message = payload.get("error", "Database service request failed")
+        raise RuntimeError(message)
+    if not response.content:
+        return None
+    return decode_value(response.json())
+
+
+def _invoke_cached(fn_name: str, args, kwargs):
+    cache_key = f"{fn_name}:{args!r}:{sorted(kwargs.items())!r}"
+    cached = _SETTINGS_CACHE.get(cache_key)
+    if cached is not None:
+        return cached
+    value = _invoke(fn_name, args, kwargs, write=False)
+    _SETTINGS_CACHE.set(cache_key, value)
+    return value
+
+
+def _invoke(fn_name: str, args, kwargs, *, write: bool = False):
+    encoded_args, encoded_kwargs = encode_args(args, kwargs)
+    payload = {
+        "fn": fn_name,
+        "args": encoded_args,
+        "kwargs": encoded_kwargs,
+        "write": write,
+    }
+    result = _post("internal/invoke", payload)
+    if isinstance(result, dict) and "result" in result:
+        return result["result"]
+    return result
+
+
+class RemoteSearchClause:
+    def __init__(self, term, fields, author_field=None):
+        self.term = term.strip()
+        self.fields = tuple(fields)
+        self.author_field = author_field
+
+
+class RemoteUidInClause:
+    def __init__(self, field, uids):
+        self.field = field
+        self.uids = frozenset(uids)
+
+
+class RemoteTable:
+    def __init__(self, db: "RemoteDb", name: str) -> None:
+        self._db = db
+        self._name = name
+        self._column_cache = None
+
+    def __getattr__(self, name: str):
+        def caller(*args, **kwargs):
+            return self._db._table_op(self._name, name, args, kwargs)
+
+        return caller
+
+    def has_column(self, name: str) -> bool:
+        cache = self._column_cache
+        if cache is None:
+            sample = self.find(_limit=1)
+            row = next(iter(sample), None)
+            cache = set(row.keys()) if row else set()
+            self._column_cache = cache
+        return name in cache
+
+    def count(self, **kwargs):
+        return self._db._table_op(self._name, "count", [], kwargs)
+
+    @property
+    def table(self):
+        return self
+
+    @property
+    def exists(self) -> bool:
+        return self._name in self._db.tables
+
+class RemoteDb:
+    def __init__(self) -> None:
+        self._tables_cache: list[str] | None = None
+
+    @property
+    def tables(self) -> list[str]:
+        if self._tables_cache is None:
+            result = _post("internal/db-op", {"op": "tables"})
+            self._tables_cache = list(result or [])
+        return self._tables_cache
+
+    def __getitem__(self, name: str) -> RemoteTable:
+        return RemoteTable(self, name)
+
+    def query(self, sql: str, **params):
+        encoded_args, encoded_kwargs = encode_args((sql,), params)
+        result = _post(
+            "internal/db-op",
+            {
+                "op": "query",
+                "args": encoded_args,
+                "kwargs": encoded_kwargs,
+                "write": is_write_sql(sql),
+            },
+        )
+        return result or []
+
+    def _table_op(self, table: str, method: str, args, kwargs, *, write: bool = False):
+        encoded_args, encoded_kwargs = encode_args(args, kwargs)
+        result = _post(
+            "internal/db-op",
+            {
+                "op": "table_op",
+                "table": table,
+                "method": method,
+                "args": encoded_args,
+                "kwargs": encoded_kwargs,
+                "write": write,
+            },
+        )
+        if method in {"insert", "update", "delete"}:
+            self._tables_cache = None
+        return result
+
+    @property
+    def executable(self):
+        return self
+
+    @property
+    def in_transaction(self) -> bool:
+        return False
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc, tb):
+        return False
+
+
+_LOCAL_REMOTE = frozenset(
+    {
+        "get_table",
+        "refresh_snapshot",
+        "_in_clause",
+        "_now_iso",
+        "text_search_clause",
+    }
+)
+
+
+def _remote_text_search_clause(
+    table, search, fields=("title", "description"), author_field=None
+):
+    term = (search or "").strip()
+    if not term:
+        return None
+    if type(table).__name__ == "RemoteTable":
+        return RemoteSearchClause(term, fields, author_field)
+    from devplacepy.database.content import text_search_clause as local_clause
+
+    return local_clause(table, search, fields, author_field=author_field)
+
+
+def _remote_get_table(name: str):
+    import devplacepy.database.core as core
+
+    return core.db[name]
+
+
+def _remote_refresh_snapshot() -> None:
+    return None
+
+
+def patch_module(module) -> None:
+    import devplacepy.database as db_module
+
+    for name in db_module.__all__:
+        if name in _LOCAL_REMOTE:
+            continue
+        target = getattr(module, name, None)
+        if target is None or not callable(target):
+            continue
+        if inspect.isclass(target):
+            continue
+
+        def make_wrapper(fn_name: str, fn_write: bool):
+            if fn_name in _CACHED_SETTINGS_FNS:
+
+                def wrapper(*args, **kwargs):
+                    return _invoke_cached(fn_name, args, kwargs)
+
+                wrapper.__name__ = fn_name
+                return wrapper
+
+            def wrapper(*args, **kwargs):
+                return _invoke(fn_name, args, kwargs, write=fn_write)
+
+            wrapper.__name__ = fn_name
+            return wrapper
+
+        setattr(module, name, make_wrapper(name, is_write(name)))
+
+
+def activate() -> None:
+    import devplacepy.database.core as core
+
+    core.db = RemoteDb()
+    import devplacepy.database as db_module
+
+    patch_module(db_module)
+    for submodule_name in (
+        "settings",
+        "users",
+        "relations",
+        "pagination",
+        "soft_delete",
+        "engagement",
+        "usage",
+        "awards",
+        "seo_meta",
+        "activity",
+        "customization",
+        "email",
+        "notifications",
+        "forks",
+        "follows",
+        "deepsearch",
+        "ranking",
+        "comments",
+        "content",
+        "attachments_data",
+        "stats",
+        "schema",
+    ):
+        try:
+            submodule = __import__(
+                f"devplacepy.database.{submodule_name}",
+                fromlist=[submodule_name],
+            )
+        except ImportError:
+            continue
+        patch_module(submodule)
+    for external_name in (
+        "devplacepy.services.statistics.tracking",
+        "devplacepy.services.base",
+        "devplacepy.attachments",
+        "devplacepy.project_files",
+    ):
+        try:
+            external = __import__(external_name, fromlist=[external_name.split(".")[-1]])
+        except ImportError:
+            continue
+        if hasattr(external, "db"):
+            external.db = RemoteDb()
+    db_module.db = core.db
+    db_module.get_table = _remote_get_table
+    core.get_table = _remote_get_table
+    db_module.refresh_snapshot = _remote_refresh_snapshot
+    core.refresh_snapshot = _remote_refresh_snapshot
+    db_module.text_search_clause = _remote_text_search_clause
+    import devplacepy.database.content as content_module
+
+    content_module.text_search_clause = _remote_text_search_clause
+    for submodule_name in (
+        "settings",
+        "users",
+        "relations",
+        "pagination",
+        "soft_delete",
+        "engagement",
+        "usage",
+        "awards",
+        "seo_meta",
+        "activity",
+        "customization",
+        "email",
+        "notifications",
+        "forks",
+        "follows",
+        "deepsearch",
+        "ranking",
+        "comments",
+        "content",
+        "attachments_data",
+        "stats",
+        "schema",
+    ):
+        try:
+            submodule = __import__(
+                f"devplacepy.database.{submodule_name}",
+                fromlist=[submodule_name],
+            )
+        except ImportError:
+            continue
+        if hasattr(submodule, "db"):
+            submodule.db = core.db

devplacepy/database/schema.py

@@ -1,6 +1,6 @@
 # retoor <retoor@molodetz.nl>
 
-from .core import DEFAULT_CORRECTION_PROMPT, DEFAULT_MODIFIER_PROMPT, INTERNAL_GATEWAY_URL, _index, _uid_index, db, defaultdict, get_table, logger
+from .core import DEFAULT_CORRECTION_PROMPT, DEFAULT_MODIFIER_PROMPT, INTERNAL_GATEWAY_URL, _drop_index, _index, _uid_index, db, defaultdict, get_table, logger
 from .settings import get_setting, set_setting
 from .soft_delete import SOFT_DELETE_TABLES, ensure_soft_delete_columns
 from .ranking import _authors_cache
@@ -183,8 +183,10 @@ def init_db():
     )
     _index(db, "badges", "idx_badges_user", ["user_uid"])
     _index(db, "badges", "idx_badges_user_name", ["user_uid", "badge_name"])
-    _index(db, "follows", "idx_follows_follower", ["follower_uid"])
-    _index(db, "follows", "idx_follows_following", ["following_uid"])
+    _drop_index(db, "idx_follows_follower")
+    _drop_index(db, "idx_follows_following")
+    _index(db, "follows", "idx_follows_follower_created", ["follower_uid", "created_at"])
+    _index(db, "follows", "idx_follows_following_created", ["following_uid", "created_at"])
     user_relations = get_table("user_relations")
     for column, example in (
         ("uid", ""),
@@ -363,9 +365,10 @@ def init_db():
         if not conversations.has_column("channel"):
             conversations.create_column_by_example("channel", "main")
         try:
-            db.query(
-                "UPDATE devii_conversations SET channel='main' WHERE channel IS NULL"
-            )
+            with db:
+                db.query(
+                    "UPDATE devii_conversations SET channel='main' WHERE channel IS NULL"
+                )
         except Exception as e:  # noqa: BLE001
             logger.warning(f"Could not backfill devii_conversations.channel: {e}")
     _index(
@@ -391,6 +394,7 @@ def init_db():
         db, "devii_tasks", "idx_devii_tasks_due", ["enabled", "status", "next_run_at"]
     )
     _index(db, "devii_lessons", "idx_devii_lessons_owner", ["owner_kind", "owner_id"])
+    _index(db, "devii_lessons", "idx_devii_lessons_owner_created", ["owner_kind", "owner_id", "created_at"])
     _index(
         db, "devii_virtual_tools", "idx_devii_vtools_owner", ["owner_kind", "owner_id"]
     )
@@ -430,6 +434,12 @@ def init_db():
         "idx_gw_usage_endpoint_time",
         ["endpoint", "created_at"],
     )
+    _index(
+        db,
+        "gateway_usage_ledger",
+        "idx_gw_usage_appref_time",
+        ["app_reference", "created_at"],
+    )
     _index(db, "gateway_concurrency_samples", "idx_gw_conc_time", ["created_at"])
     jobs_table = get_table("jobs")
     for column, example in (
@@ -559,10 +569,11 @@ def init_db():
             correction_usage.create_column_by_example(column, example)
     try:
         if "correction_usage" in db.tables:
-            db.query(
-                "CREATE UNIQUE INDEX IF NOT EXISTS idx_correction_usage_user "
-                "ON correction_usage (user_uid)"
-            )
+            with db:
+                db.query(
+                    "CREATE UNIQUE INDEX IF NOT EXISTS idx_correction_usage_user "
+                    "ON correction_usage (user_uid)"
+                )
     except Exception as e:
         logger.warning(f"Could not create unique index on correction_usage: {e}")
 
@@ -582,10 +593,11 @@ def init_db():
             modifier_usage.create_column_by_example(column, example)
     try:
         if "modifier_usage" in db.tables:
-            db.query(
-                "CREATE UNIQUE INDEX IF NOT EXISTS idx_modifier_usage_user "
-                "ON modifier_usage (user_uid)"
-            )
+            with db:
+                db.query(
+                    "CREATE UNIQUE INDEX IF NOT EXISTS idx_modifier_usage_user "
+                    "ON modifier_usage (user_uid)"
+                )
     except Exception as e:
         logger.warning(f"Could not create unique index on modifier_usage: {e}")
 
@@ -605,10 +617,11 @@ def init_db():
             news_usage.create_column_by_example(column, example)
     try:
         if "news_usage" in db.tables:
-            db.query(
-                "CREATE UNIQUE INDEX IF NOT EXISTS idx_news_usage_user "
-                "ON news_usage (user_uid)"
-            )
+            with db:
+                db.query(
+                    "CREATE UNIQUE INDEX IF NOT EXISTS idx_news_usage_user "
+                    "ON news_usage (user_uid)"
+                )
     except Exception as e:
         logger.warning(f"Could not create unique index on news_usage: {e}")
 
@@ -628,10 +641,11 @@ def init_db():
             issue_usage.create_column_by_example(column, example)
     try:
         if "issue_usage" in db.tables:
-            db.query(
-                "CREATE UNIQUE INDEX IF NOT EXISTS idx_issue_usage_user "
-                "ON issue_usage (user_uid)"
-            )
+            with db:
+                db.query(
+                    "CREATE UNIQUE INDEX IF NOT EXISTS idx_issue_usage_user "
+                    "ON issue_usage (user_uid)"
+                )
     except Exception as e:
         logger.warning(f"Could not create unique index on issue_usage: {e}")
 
@@ -651,10 +665,11 @@ def init_db():
             seo_usage.create_column_by_example(column, example)
     try:
         if "seo_usage" in db.tables:
-            db.query(
-                "CREATE UNIQUE INDEX IF NOT EXISTS idx_seo_usage_user "
-                "ON seo_usage (user_uid)"
-            )
+            with db:
+                db.query(
+                    "CREATE UNIQUE INDEX IF NOT EXISTS idx_seo_usage_user "
+                    "ON seo_usage (user_uid)"
+                )
     except Exception as e:
         logger.warning(f"Could not create unique index on seo_usage: {e}")
 
@@ -674,10 +689,11 @@ def init_db():
             award_usage.create_column_by_example(column, example)
     try:
         if "award_usage" in db.tables:
-            db.query(
-                "CREATE UNIQUE INDEX IF NOT EXISTS idx_award_usage_user "
-                "ON award_usage (user_uid)"
-            )
+            with db:
+                db.query(
+                    "CREATE UNIQUE INDEX IF NOT EXISTS idx_award_usage_user "
+                    "ON award_usage (user_uid)"
+                )
     except Exception as e:
         logger.warning(f"Could not create unique index on award_usage: {e}")
 
@@ -701,21 +717,22 @@ def init_db():
             awards.create_column_by_example(column, example)
     try:
         if "awards" in db.tables:
-            db.query(
-                "CREATE UNIQUE INDEX IF NOT EXISTS idx_awards_slug ON awards (slug)"
-            )
-            db.query(
-                "CREATE INDEX IF NOT EXISTS idx_awards_receiver_created "
-                "ON awards (receiver_uid, created_at)"
-            )
-            db.query(
-                "CREATE INDEX IF NOT EXISTS idx_awards_giver_created "
-                "ON awards (giver_uid, created_at)"
-            )
-            db.query(
-                "CREATE INDEX IF NOT EXISTS idx_awards_generated "
-                "ON awards (receiver_uid, generated_at)"
-            )
+            with db:
+                db.query(
+                    "CREATE UNIQUE INDEX IF NOT EXISTS idx_awards_slug ON awards (slug)"
+                )
+                db.query(
+                    "CREATE INDEX IF NOT EXISTS idx_awards_receiver_created "
+                    "ON awards (receiver_uid, created_at)"
+                )
+                db.query(
+                    "CREATE INDEX IF NOT EXISTS idx_awards_giver_created "
+                    "ON awards (giver_uid, created_at)"
+                )
+                db.query(
+                    "CREATE INDEX IF NOT EXISTS idx_awards_generated "
+                    "ON awards (receiver_uid, generated_at)"
+                )
     except Exception as e:
         logger.warning(f"Could not create awards indexes: {e}")
 
@@ -788,10 +805,11 @@ def init_db():
             user_activity.create_column_by_example(column, example)
     try:
         if "user_activity" in db.tables:
-            db.query(
-                "CREATE UNIQUE INDEX IF NOT EXISTS idx_user_activity_user_action "
-                "ON user_activity (user_uid, action)"
-            )
+            with db:
+                db.query(
+                    "CREATE UNIQUE INDEX IF NOT EXISTS idx_user_activity_user_action "
+                    "ON user_activity (user_uid, action)"
+                )
     except Exception as e:
         logger.warning(f"Could not create unique index on user_activity: {e}")
 
@@ -806,10 +824,11 @@ def init_db():
             user_activity_seen.create_column_by_example(column, example)
     try:
         if "user_activity_seen" in db.tables:
-            db.query(
-                "CREATE UNIQUE INDEX IF NOT EXISTS idx_user_activity_seen_unique "
-                "ON user_activity_seen (user_uid, action, target)"
-            )
+            with db:
+                db.query(
+                    "CREATE UNIQUE INDEX IF NOT EXISTS idx_user_activity_seen_unique "
+                    "ON user_activity_seen (user_uid, action, target)"
+                )
     except Exception as e:
         logger.warning(f"Could not create unique index on user_activity_seen: {e}")
 
@@ -1219,6 +1238,8 @@ def init_db():
             "statistics_tracking_enabled": "1",
             "docs_search_mode": "agent",
             "outbound_proxy_url": "",
+            "devii_lessons_max_per_owner": "500",
+            "devii_lessons_max_age_days": "90",
         }
         for key, value in operational_defaults.items():
             existing = db["site_settings"].find_one(key=key)
@@ -1372,6 +1393,8 @@ def backfill_api_keys() -> int:
         users.create_column_by_example("ai_modifier_sync", 1)
     if not users.has_column("ai_modifier_prompt"):
         users.create_column_by_example("ai_modifier_prompt", DEFAULT_MODIFIER_PROMPT)
+    if not users.has_column("interactions_enabled"):
+        users.create_column_by_example("interactions_enabled", -1)
     if not users.has_column("timezone"):
         users.create_column_by_example("timezone", "")
     if not users.has_column("avatar_seed"):
@@ -1396,6 +1419,10 @@ def backfill_api_keys() -> int:
             "WHERE ai_modifier_prompt IS NULL OR ai_modifier_prompt = ''",
             prompt=DEFAULT_MODIFIER_PROMPT,
         )
+        db.query(
+            "UPDATE users SET interactions_enabled = -1 "
+            "WHERE interactions_enabled IS NULL"
+        )
     import uuid_utils
 
     updated = 0

devplacepy/database/soft_delete.py

@@ -94,11 +94,12 @@ def soft_delete_in(table_name, column, uids, deleted_by, *, stamp=None, **extra)
     for index, (key, value) in enumerate(extra.items()):
         params[f"x{index}"] = value
         extra_sql += f" AND {key} = :x{index}"
-    db.query(
-        f"UPDATE {table_name} SET deleted_at = :dat, deleted_by = :dby "
-        f"WHERE {column} IN ({placeholders}) AND deleted_at IS NULL{extra_sql}",
-        **params,
-    )
+    with db:
+        db.query(
+            f"UPDATE {table_name} SET deleted_at = :dat, deleted_by = :dby "
+            f"WHERE {column} IN ({placeholders}) AND deleted_at IS NULL{extra_sql}",
+            **params,
+        )
     return len(uids)
 
 
@@ -161,11 +162,12 @@ def restore_event(stamp):
                     s=stamp,
                 ).__next__()["n"]
             )
-            db.query(
-                f"UPDATE {table_name} SET deleted_at = NULL, deleted_by = NULL "
-                f"WHERE deleted_at = :s",
-                s=stamp,
-            )
+            with db:
+                db.query(
+                    f"UPDATE {table_name} SET deleted_at = NULL, deleted_by = NULL "
+                    f"WHERE deleted_at = :s",
+                    s=stamp,
+                )
     return restored
 
 
@@ -182,7 +184,8 @@ def purge_event(stamp):
             )
             if rows:
                 purged.append((table_name, rows))
-                db.query(
-                    f"DELETE FROM {table_name} WHERE deleted_at = :s", s=stamp
-                )
+                with db:
+                    db.query(
+                        f"DELETE FROM {table_name} WHERE deleted_at = :s", s=stamp
+                    )
     return purged