Update

.gitignore

@@ -32,3 +32,10 @@ var/
 .coverage
 .coverage.*
 htmlcov/
+
+# local environments and scratch
+.venv/
+tmp/
+*.log
+*.bak
+test.db

CLAUDE.md

@@ -155,7 +155,7 @@ Routers in `devplacepy/routers/` are organised as a **directory tree that mirror
 | `/auth` | auth/ package |
 | `/feed`, `/posts`, `/comments` | flat files |
 | `/projects`, `/projects/{slug}/files`, `/projects/{slug}/containers` | projects/ package - see `routers/projects/CLAUDE.md` |
-| `/profile` | profile/ package (customization, notifications, ai-correction, ai-modifier, telegram, usage) |
+| `/profile` | profile/ package (customization, notifications, ai-correction, ai-modifier, interactions, telegram, usage) |
 | `/messages` | messages.py - see `services/messaging/CLAUDE.md` |
 | `/notifications`, `/votes`, `/reactions`, `/bookmarks`, `/polls`, `/avatar`, `/follow`, `/leaderboard` | flat files |
 | (none) | relations.py - block/mute |
@@ -193,7 +193,7 @@ The CLIENT pipeline (`ContentRenderer.js`, `dp-content`/`dp-title`) is retained
 
 **Emoji shortcodes are the full GitHub/Discord `:name:` set** (~4869 names), generated once from the `emoji` library by `rendering.py` `build_emoji_shortcodes()`; the frontend gets the identical map via the generated `static/js/emoji-shortcodes.js` (regenerate with `devplace emoji-sync` after bumping the dependency, never hand-edit).
 
-**Template em-dash normalization:** the shared `templates.env.template_class` runs every rendered template's final HTML through `normalize_em_dash` - the em-dash character and its HTML entity forms all become a plain hyphen, application-wide. One hook; never re-strip em-dash per template.
+**Em-dash normalization:** `_normalize_dashes` in `rendering.py` replaces all forms (em dash `\u2014`, en dash `\u2013`, and their HTML entities) with a hyphen BEFORE mistune processes the text. Runs inside `_render_content`/`_render_title` which are `@lru_cache`d, so each unique text is normalized once. No per-template overhead.
 
 ### Auth
 

Dockerfile

@@ -31,6 +31,7 @@ RUN pip install --no-cache-dir ".[bots]" \
 EXPOSE 10500
 
 ENV DEVPLACE_WEB_WORKERS=2
+ENV DEVPLACE_TEMPLATE_AUTO_RELOAD=0
 
 HEALTHCHECK --interval=30s --timeout=10s --retries=3 --start-period=10s \
     CMD curl -f http://localhost:10500/ || exit 1

README.md

@@ -109,6 +109,8 @@ Member progression is driven by activity and peer recognition.
 
 - **AI modifier.** Enabled by default and applied synchronously by default. It works like AI content correction, except it runs **only** where the prose you author contains an inline `@ai <instruction>` directive: the configured prompt tells the model to execute that instruction and replace the marked part, removing the `@ai` marker. Text with no `@ai ...` directive is left exactly as written. It is **context-aware**: the model is given a grounding summary of who is asking (your username, role, level, stars, post count, rank, followers, and bio), the current date, and where the directive sits - the post a comment replies to, the conversation a direct message belongs to, the gist's language and code, and so on - so directives like `@ai answer the question above`, `@ai write my bio from my stats`, or `@ai reply to this` work. It uses your own API key for per-user attribution, is fail-soft (the original is kept on any error), and applies across the web UI, the REST and devRant APIs, and Devii, on the same prose fields as correction (posts, projects, gists, comments, direct messages, and your bio). Code and source files are never touched. In direct messages it runs live: typing `@ai <instruction>` in a message executes it and the resolved result appears in the chat for both participants without a reload. You can switch the apply mode to background or disable it on your profile or via the Devii `ai_modifier_set` tool; the settings are saved at `POST /profile/{username}/ai-modifier`. The default instruction is "Execute what is behind `@ai` (the prompt) and replace that part including `@ai`". Successful modifications accumulate per-user running totals - modifications, token counts, cost, and timing/performance (average latency, average speed in tokens per second, and total processing time) - shown on the profile page; token, call, and performance figures are visible to the member, while the dollar figures (total and average cost) are shown to administrators only.
 
+- **Devii interactive widgets.** Administrators set the site default on the Devii service (`devii_interactions_default`, default on). Guests always use that default. Signed-in members inherit it until they override it on their profile or via the Devii `interactions_set` tool (`POST /profile/{username}/interactions`; owner or admin). When enabled, Devii may present decisions with channel-aware controls (`ui_prompt`); when disabled, it falls back to plain numbered menus.
+
 Every AI gateway response (`/openai/v1/*`) also returns per-call `X-Gateway-*` headers with the full token breakdown and the dollar cost of that call, so any client can read its own usage.
 
 ## Code Farm

devplacepy/attachments.py

@@ -444,12 +444,13 @@ def link_attachments(uids, target_type, target_uid):
         return
     placeholders = ",".join(f":p{i}" for i in range(len(flat)))
     params = {f"p{i}": uid for i, uid in enumerate(flat)}
-    db.query(
+    with db:
-        f"UPDATE attachments SET target_type=:tt, target_uid=:tu WHERE uid IN ({placeholders})",
+        db.query(
-        tt=target_type,
+            f"UPDATE attachments SET target_type=:tt, target_uid=:tu WHERE uid IN ({placeholders})",
-        tu=target_uid,
+            tt=target_type,
-        **params,
+            tu=target_uid,
-    )
+            **params,
+        )
 
 
 def set_gitea_asset_id(uid, asset_id):
@@ -617,7 +618,8 @@ def delete_attachments_for(target_type, target_uids):
     for row in rows:
         _unlink_attachment_files(row)
     ids = ",".join(str(row["id"]) for row in rows)
-    db.query(f"DELETE FROM attachments WHERE id IN ({ids})")
+    with db:
+        db.query(f"DELETE FROM attachments WHERE id IN ({ids})")
 
 
 def get_attachments(target_type, target_uid):

devplacepy/cli/devii.py

@@ -1,13 +1,11 @@
 # retoor <retoor@molodetz.nl>
 
 import sys
-from devplacepy.database import get_table
+from devplacepy.database import db, get_table
 from devplacepy.cli._shared import _audit_cli
 
 
 def cmd_devii_reset_quota(args):
-    from devplacepy.database import db
-
     table_name = "devii_usage_ledger"
     if table_name not in db.tables:
         print(f"Table '{table_name}' does not exist, nothing to reset")
@@ -48,6 +46,67 @@ def cmd_devii_reset_quota(args):
     print(f"Reset AI quota for '{args.username}' ({count} ledger rows deleted)")
 
 
+def _active_count() -> int:
+    if "devii_lessons" not in db.tables:
+        return 0
+    return db["devii_lessons"].count(deleted_at=None)
+
+
+def _soft_deleted_count() -> int:
+    if "devii_lessons" not in db.tables:
+        return 0
+    return db["devii_lessons"].count(deleted_at={"!=": None})
+
+
+def cmd_devii_lessons_count(args):
+    active = _active_count()
+    deleted = _soft_deleted_count()
+    print(f"Lessons: {active} active, {deleted} soft-deleted ({active + deleted} total)")
+
+
+def cmd_devii_lessons_clear(args):
+    from devplacepy.services.devii.agentic.lessons import TABLE
+
+    if TABLE not in db.tables:
+        print("No devii_lessons table exists")
+        return
+    active = _active_count()
+    deleted = _soft_deleted_count()
+    total = active + deleted
+    if not args.force:
+        print(f"Will delete {total} lesson(s) ({active} active, {deleted} soft-deleted). Pass --force to confirm.")
+        return
+    db[TABLE].delete()
+    _audit_cli("cli.devii.lessons.clear", "CLI cleared all devii_lessons", metadata={"active": active, "soft_deleted": deleted})
+    print(f"Deleted {total} lesson(s)")
+
+
+def cmd_devii_lessons_prune(args):
+    from devplacepy.services.devii.agentic.lessons import LessonStore, _read_retention_settings
+
+    if "devii_lessons" not in db.tables:
+        print("No devii_lessons table exists")
+        return
+    active_before = _active_count()
+    if args.all_owners:
+        _, max_age = _read_retention_settings(db)
+        store = LessonStore(db, "_global", "_global")
+        pruned = store.prune_all_owners(max_age)
+    elif args.username:
+        user = get_table("users").find_one(username=args.username)
+        if not user:
+            print(f"User '{args.username}' not found")
+            sys.exit(1)
+        _, max_age = _read_retention_settings(db)
+        store = LessonStore(db, "user", user["uid"])
+        pruned = store.prune(max_age)
+    else:
+        print("Provide --all-owners, or --username USER")
+        sys.exit(1)
+    _audit_cli("cli.devii.lessons.prune", "CLI pruned devii_lessons", metadata={"pruned": pruned, "active_before": active_before})
+    print(f"Pruned {pruned} lesson(s) (active before: {active_before}, now: {_active_count()})")
+
+
 def register_devii(subparsers):
     devii = subparsers.add_parser("devii", help="Devii assistant management")
     devii_sub = devii.add_subparsers(title="action", dest="action")
@@ -64,3 +123,18 @@ def register_devii(subparsers):
         "--all", action="store_true", help="Reset every quota (users and guests)"
     )
     devii_reset.set_defaults(func=cmd_devii_reset_quota)
+
+    devii_lessons = devii_sub.add_parser("lessons", help="Manage persisted Devii lesson data")
+    lessons_sub = devii_lessons.add_subparsers(title="sub-action", dest="sub_action")
+
+    lessons_count = lessons_sub.add_parser("count", help="Count active and soft-deleted lessons")
+    lessons_count.set_defaults(func=cmd_devii_lessons_count)
+
+    lessons_prune = lessons_sub.add_parser("prune", help="Soft-delete lessons older than the configured max age")
+    lessons_prune.add_argument("--all-owners", action="store_true", help="Prune across every owner")
+    lessons_prune.add_argument("--username", help="Prune for a specific user")
+    lessons_prune.set_defaults(func=cmd_devii_lessons_prune)
+
+    lessons_clear = lessons_sub.add_parser("clear", help="Hard-delete every devii_lessons row")
+    lessons_clear.add_argument("--force", action="store_true", help="Required to confirm hard deletion")
+    lessons_clear.set_defaults(func=cmd_devii_lessons_clear)

devplacepy/content.py

@@ -13,6 +13,7 @@ from devplacepy.database import (
     resolve_by_slug,
     get_users_by_uids,
     get_vote_counts,
+    STAR_TARGETS,
     get_user_votes,
     get_reactions_by_targets,
     get_user_bookmarks,
@@ -20,6 +21,7 @@ from devplacepy.database import (
     get_poll_for_post,
     update_target_stars,
     clear_user_stars,
+    clear_user_post_count,
     get_target_owner_uid,
     resolve_object_url,
     soft_delete,
@@ -162,6 +164,8 @@ def create_content_item(
             **fields,
         }
     )
+    if table_name == "posts":
+        clear_user_post_count(user["uid"])
     if table_name == "projects":
         from devplacepy.templating import clear_user_projects_cache
 
@@ -615,6 +619,8 @@ def delete_content_item(
     soft_delete_engagement(target_type, [item["uid"]], actor)
     if comment_uids:
         soft_delete_engagement("comment", comment_uids, actor)
+    if target_type == "post":
+        clear_user_post_count(item["user_uid"])
     if target_type == "project":
         from devplacepy.project_files import soft_delete_all_project_files
         from devplacepy.templating import clear_user_projects_cache
@@ -647,7 +653,11 @@ def load_detail(
     if user and item["user_uid"] in get_blocked_uids(user["uid"]):
         return None
     author = get_users_by_uids([item["user_uid"]]).get(item["user_uid"])
-    ups, downs = get_vote_counts([item["uid"]])
+    if target_type in STAR_TARGETS:
+        star_count = item.get("stars") or 0
+    else:
+        ups, downs = get_vote_counts([item["uid"]])
+        star_count = ups.get(item["uid"], 0) - downs.get(item["uid"], 0)
     reactions = (
         get_reactions_by_targets(target_type, [item["uid"]], user).get(
             item["uid"], {"counts": {}, "mine": []}
@@ -664,7 +674,7 @@ def load_detail(
         "item": item,
         "author": author,
         "is_owner": bool(user and user["uid"] == item["user_uid"]),
-        "star_count": ups.get(item["uid"], 0) - downs.get(item["uid"], 0),
+        "star_count": star_count,
         "my_vote": get_user_votes(user["uid"], [item["uid"]]).get(item["uid"], 0)
         if user
         else 0,

devplacepy/customization.py

@@ -36,6 +36,18 @@ def owner_for(request: Request) -> tuple[str, str] | None:
 
 
 def _overrides_for(request: Request) -> dict:
+    cached = getattr(request.state, "_custom_overrides", None)
+    if cached is not None:
+        return cached
+    overrides = _resolve_overrides(request)
+    try:
+        request.state._custom_overrides = overrides
+    except Exception:
+        pass
+    return overrides
+
+
+def _resolve_overrides(request: Request) -> dict:
     if get_setting("customization_enabled", "1") != "1":
         return {"css": "", "js": ""}
     owner = owner_for(request)

devplacepy/database/CLAUDE.md

@@ -99,7 +99,7 @@ if "comments" not in db.tables:
 
 - **NEVER index the bare `deleted_at` column - use a PARTIAL trash index `WHERE deleted_at IS NOT NULL`.** `ensure_soft_delete_columns` creates `idx_<table>_trash ON (deleted_at) WHERE deleted_at IS NOT NULL` (and drops any legacy full `idx_<table>_deleted`). A full `deleted_at` index is a planner hazard: the column is one giant `NULL` bucket plus many unique delete-timestamps, so `sqlite_stat1` mis-estimates `deleted_at IS NULL` as returning ~2 rows and the planner picks that index for live reads, then `USE TEMP B-TREE FOR ORDER BY` to sort the whole live set (the global feed was doing exactly this, with 82% of posts soft-deleted). The partial index serves the admin Trash view (`deleted_at IS NOT NULL`) cheaply and stops poisoning live `IS NULL` queries.
 
-- **For "live, newest-first" listings add a composite or live-partial index that includes the sort column.** A `WHERE deleted_at IS NULL ORDER BY created_at` query needs the ordering in the index or it filesorts. Posts use a partial `idx_posts_live_created ON (created_at) WHERE deleted_at IS NULL` (feed) plus `idx_posts_user_created (user_uid, created_at)` (profile); comments use `idx_comments_target_created (target_type, target_uid, created_at)`; votes use `idx_votes_user_target (user_uid, target_uid)` (the per-user "my_vote" check on every card); notifications/gists/projects use `(user_uid, created_at)`. All were verified to drop the `USE TEMP B-TREE FOR ORDER BY` step.
+- **For "live, newest-first" listings add a composite or live-partial index that includes the sort column.** A `WHERE deleted_at IS NULL ORDER BY created_at` query needs the ordering in the index or it filesorts. Posts use a partial `idx_posts_live_created ON (created_at) WHERE deleted_at IS NULL` (feed) plus `idx_posts_user_created (user_uid, created_at)` (profile); comments use `idx_comments_target_created (target_type, target_uid, created_at)`; votes use `idx_votes_user_target (user_uid, target_uid)` (the per-user "my_vote" check on every card); notifications/gists/projects use `(user_uid, created_at)`; follows use `idx_follows_follower_created (follower_uid, created_at)` + `idx_follows_following_created (following_uid, created_at)` (the followers/following tabs sort newest-first; the legacy single-column follower/following indexes were dropped as redundant prefixes). All were verified to drop the `USE TEMP B-TREE FOR ORDER BY` step.
 
 - **Index the non-`uid` lookup keys too, not just the sort/owner columns.** A demand-vs-supply audit added the last missing single-key lookups: the `resolve_by_slug` hot path filters `slug` on content detail pages, so posts/gists/news/projects each get `idx_<table>_slug (slug)`; `get_setting`/`set_setting` filter `key`, so `idx_site_settings_key (key)`; the container store's `find_one(slug=)`/`find_one(name=)` fallbacks get `idx_instances_slug`/`idx_instances_name`. The DM thread load `find(sender_uid=, receiver_uid=)` gets the covering composites `idx_messages_conversation (sender_uid, receiver_uid)` + `idx_messages_conversation_rev (receiver_uid, sender_uid)` (the read-flag `UPDATE` uses the reverse); the badge-has check gets `idx_badges_user_name (user_uid, badge_name)`; the admin user list `ORDER BY -created_at` gets `idx_users_created_at (created_at)` (the existing `(role, created_at)` cannot serve a full-table created_at sort). All are non-unique so `_index` always creates them even if legacy duplicate data exists. Column sets already resolved to ~1 row by an existing prefix index (votes `+target_type`, game_quests `+kind`, poll_options `position`) are intentionally left uncovered - a trailing column there only adds write cost.
 

devplacepy/database/__init__.py

@@ -5,7 +5,7 @@ from .core import refresh_snapshot, _local_cache_versions, _cache_version_cache,
 from .settings import _settings_cache, get_setting, get_int_setting, set_setting, clear_settings_cache, internal_gateway_key
 from .users import get_users_by_uids, _admins_cache, invalidate_admins_cache, get_admin_uids, set_user_timezone, set_last_seen, get_online_users, get_primary_admin_uid, search_users_by_username
 from .relations import _relations_cache, get_user_relations, get_blocked_uids, get_muted_uids, get_silenced_uids, invalidate_user_relations
-from .pagination import PAGE_SIZE, paginate, interleave_by_author, paginate_diverse, get_user_post_count, build_pagination
+from .pagination import PAGE_SIZE, paginate, interleave_by_author, paginate_diverse, get_user_post_count, clear_user_post_count, build_pagination
 from .soft_delete import SOFT_DELETE_TABLES, ensure_soft_delete_columns, soft_delete, soft_delete_in, restore, purge, list_deleted, count_deleted, restore_event, purge_event
 from .engagement import _comment_count_cache, get_comment_counts_by_post_uids, get_post_counts_by_user_uids, get_vote_counts, get_user_votes, get_reactions_by_targets, get_user_bookmarks, get_polls_by_post_uids, get_poll_for_post
 from .usage import _add_usage, _get_usage, add_correction_usage, get_correction_usage, add_modifier_usage, get_modifier_usage, NEWS_USAGE_KEY, add_news_usage, get_news_usage, ISSUE_USAGE_KEY, add_issue_usage, get_issue_usage, SEO_USAGE_KEY, add_seo_usage, get_seo_usage, AWARD_USAGE_KEY, add_award_usage, get_award_usage
@@ -36,7 +36,7 @@ from .follows import get_follow_counts, get_follow_list, get_following_among
 from .deepsearch import _ds_now, create_deepsearch_session, update_deepsearch_session, get_deepsearch_session, add_deepsearch_message, get_deepsearch_messages, get_cached_deepsearch_url, upsert_deepsearch_url_cache
 from .ranking import VOTABLE_TARGETS, STAR_TARGETS, _authors_cache, _ranked_authors, _rank_map, get_top_authors, get_leaderboard, get_user_rank, get_user_stars, clear_user_stars, update_target_stars, soft_delete_engagement, delete_engagement, get_target_owner_uid
 from .comments import _drop_blocked, _build_comment_items, load_comments, get_recent_comments_by_target_uids, get_recent_comments_by_post_uids, load_comments_by_target_uids
-from .content import resolve_by_slug, resolve_object_url, get_uids_by_username_match, text_search_clause, get_daily_topic, get_featured_news
+from .content import resolve_by_slug, resolve_object_url, get_uids_by_username_match, text_search_clause, get_daily_topic, get_featured_news, get_trending_topics
 from .attachments_data import get_attachments, get_attachments_by_type, get_news_images_by_uids, delete_attachment_record, delete_attachments, _delete_attachment_file, get_user_media, get_deleted_media
 from .stats import _stats_cache, get_site_stats, _analytics_cache, get_platform_analytics, _gist_languages_cache, get_gist_languages
 from .schema import BUG_TABLE_RENAMES, migrate_bug_tables_to_issue_tables, init_db, _refresh_query_planner_stats, OLD_GATEWAY_URL, migrate_ai_gateway_settings, backfill_api_keys, _backfill_gamification
@@ -98,6 +98,7 @@ __all__ = [
     "interleave_by_author",
     "paginate_diverse",
     "get_user_post_count",
+    "clear_user_post_count",
     "build_pagination",
     "SOFT_DELETE_TABLES",
     "ensure_soft_delete_columns",
@@ -225,6 +226,7 @@ __all__ = [
     "text_search_clause",
     "get_daily_topic",
     "get_featured_news",
+    "get_trending_topics",
     "get_attachments",
     "get_attachments_by_type",
     "get_news_images_by_uids",

devplacepy/database/awards.py

@@ -2,6 +2,7 @@
 
 from datetime import datetime, timedelta, timezone
 
+from devplacepy.cache import TTLCache
 from devplacepy.config import (
     AWARD_DISPLAY_HOURS_DEFAULT,
     AWARD_GIVE_COOLDOWN_HOURS_DEFAULT,
@@ -111,10 +112,22 @@ def recompute_user_award_stats(receiver_uid: str) -> None:
     users.update(payload, ["uid"])
 
 
+_prominence_cache = TTLCache(ttl=15, max_size=500)
+
+
 def award_is_prominent(user: dict | None) -> bool:
     if not user or not user.get("last_award_at") or not user.get("last_award_uid"):
         return False
-    award = resolve_by_slug(_awards_table(), user["last_award_uid"])
+    cached = _prominence_cache.get(user["last_award_uid"])
+    if cached is not None:
+        return cached
+    prominent = _compute_prominence(user["last_award_uid"])
+    _prominence_cache.set(user["last_award_uid"], prominent)
+    return prominent
+
+
+def _compute_prominence(award_uid: str) -> bool:
+    award = resolve_by_slug(_awards_table(), award_uid)
     if not award or not award.get("generated_at"):
         return False
     try:

devplacepy/database/comments.py

@@ -126,7 +126,7 @@ def load_comments_by_target_uids(target_type, target_uids, user=None):
     params["tt"] = target_type
     raw = list(
         db.query(
-            f"SELECT * FROM comments WHERE target_type=:tt AND target_uid IN ({placeholders}) ORDER BY created_at",
+            f"SELECT * FROM comments WHERE target_type=:tt AND target_uid IN ({placeholders}) AND deleted_at IS NULL ORDER BY created_at",
             **params,
         )
     )

devplacepy/database/content.py

@@ -1,7 +1,14 @@
 # retoor <retoor@molodetz.nl>
 
+from collections import Counter
+
+from devplacepy.cache import TTLCache
+
 from .core import db, get_table, or_
 
+_daily_topic_cache = TTLCache(ttl=60, max_size=1)
+_trending_cache = TTLCache(ttl=15, max_size=1)
+
 
 def resolve_by_slug(table, slug, include_deleted=False):
     has_soft_delete = table.has_column("deleted_at")
@@ -77,6 +84,15 @@ def text_search_clause(
 
 
 def get_daily_topic():
+    cached = _daily_topic_cache.get("topic")
+    if cached is not None:
+        return cached
+    topic = _load_daily_topic()
+    _daily_topic_cache.set("topic", topic)
+    return topic
+
+
+def _load_daily_topic():
     if "news" in db.tables:
         article = db["news"].find_one(
             status="published", deleted_at=None, order_by=["-synced_at"]
@@ -128,3 +144,24 @@ def get_featured_news(limit=5):
             }
         )
     return articles
+
+
+def get_trending_topics(limit: int = 6) -> list[dict]:
+    cached = _trending_cache.get("topics")
+    if cached is not None:
+        return cached[:limit]
+    if "posts" not in db.tables or "topic" not in db["posts"].columns:
+        return []
+    rows = db.query(
+        "SELECT topic FROM posts WHERE deleted_at IS NULL "
+        "AND topic IS NOT NULL AND topic != '' "
+        "ORDER BY created_at DESC LIMIT 200"
+    )
+    counter: Counter[str] = Counter()
+    for row in rows:
+        topic = (row["topic"] or "").strip()
+        if topic:
+            counter[topic] += 1
+    topics = [{"topic": t, "count": c} for t, c in counter.most_common(limit)]
+    _trending_cache.set("topics", topics)
+    return topics

devplacepy/database/core.py

@@ -76,18 +76,14 @@ def get_cache_version(name: str) -> int:
     try:
         _ensure_cache_state()
         with db:
-            row = next(
+            rows = list(db.query("SELECT name, version FROM cache_state"))
-                iter(
+        versions = {row["name"]: int(row["version"]) for row in rows}
-                    db.query(
-                        "SELECT version FROM cache_state WHERE name = :name", name=name
-                    )
-                ),
-                None,
-            )
-        version = int(row["version"]) if row else 0
     except Exception as e:
         logger.warning(f"Could not read cache version {name}: {e}")
         return 0
+    for key, version in versions.items():
+        _cache_version_cache.set(key, version)
+    version = versions.get(name, 0)
     _cache_version_cache.set(name, version)
     return version
 

devplacepy/database/pagination.py

@@ -1,5 +1,6 @@
 # retoor <retoor@molodetz.nl>
 
+from devplacepy.cache import TTLCache
 from .core import db, get_table
 from .relations import get_blocked_uids
 
@@ -7,6 +8,9 @@ from .relations import get_blocked_uids
 PAGE_SIZE = 25
 
 
+_user_post_count_cache = TTLCache(ttl=15, max_size=2000)
+
+
 def paginate(
     table,
     *clauses,
@@ -74,10 +78,19 @@ def paginate_diverse(
     return interleave_by_author(rows, uid_key=uid_key), next_cursor
 
 
+def clear_user_post_count(user_uid: str) -> None:
+    _user_post_count_cache.pop(user_uid)
+
+
 def get_user_post_count(user_uid: str) -> int:
+    cached = _user_post_count_cache.get(user_uid)
+    if cached is not None:
+        return cached
     if "posts" not in db.tables:
         return 0
-    return get_table("posts").count(user_uid=user_uid, deleted_at=None)
+    count = get_table("posts").count(user_uid=user_uid, deleted_at=None)
+    _user_post_count_cache.set(user_uid, count)
+    return count
 
 
 def build_pagination(page, total, per_page=25):

devplacepy/database/ranking.py

@@ -16,7 +16,7 @@ VOTABLE_TARGETS: dict[str, str] = {
 STAR_TARGETS: set[str] = {"post", "project", "gist"}
 
 
-_authors_cache = TTLCache(ttl=15, max_size=200)
+_authors_cache = TTLCache(ttl=60, max_size=200)
 
 
 _stars_cache = TTLCache(ttl=15, max_size=2000)
@@ -151,17 +151,19 @@ def delete_engagement(target_type: str, target_uids: list) -> None:
     if "reactions" in db.tables:
         placeholders, params = _in_clause(uids)
         params["tt"] = target_type
-        db.query(
+        with db:
-            f"DELETE FROM reactions WHERE target_type=:tt AND target_uid IN ({placeholders})",
+            db.query(
-            **params,
+                f"DELETE FROM reactions WHERE target_type=:tt AND target_uid IN ({placeholders})",
-        )
+                **params,
+            )
     if "bookmarks" in db.tables:
         placeholders, params = _in_clause(uids)
         params["tt"] = target_type
-        db.query(
+        with db:
-            f"DELETE FROM bookmarks WHERE target_type=:tt AND target_uid IN ({placeholders})",
+            db.query(
-            **params,
+                f"DELETE FROM bookmarks WHERE target_type=:tt AND target_uid IN ({placeholders})",
-        )
+                **params,
+            )
     if target_type == "post" and "polls" in db.tables:
         for uid in uids:
             for poll in db["polls"].find(post_uid=uid):

devplacepy/database/remote.py

@@ -0,0 +1,348 @@
+# retoor <retoor@molodetz.nl>
+
+import inspect
+import os
+
+import httpx
+
+from devplacepy.cache import TTLCache
+from devplacepy_services.base.db_codec import (
+    decode_value,
+    encode_args,
+    is_write,
+    is_write_sql,
+)
+
+_SERVICE_URL = os.environ.get("DEVPLACE_DB_SERVICE_URL", "http://127.0.0.1:10601").rstrip("/")
+_INTERNAL_KEY = os.environ.get("DEVPLACE_GATEWAY_INTERNAL_KEY", "").strip()
+_CLIENT: httpx.Client | None = None
+
+# Section 7.3: settings reads tolerate up to 5s staleness. patch_module()
+# generically RPCs every devplacepy.database call, bypassing the local
+# TTL cache get_setting/get_int_setting had in-process - without this,
+# every settings read (rate limiting, maintenance mode, admin dashboards)
+# pays a full HTTP round trip to the database broker.
+_SETTINGS_CACHE_TTL_SECONDS = 5
+_SETTINGS_CACHE = TTLCache(ttl=_SETTINGS_CACHE_TTL_SECONDS, max_size=512)
+_CACHED_SETTINGS_FNS = frozenset({"get_setting", "get_int_setting"})
+
+
+def _service_url() -> str:
+    return os.environ.get("DEVPLACE_DB_SERVICE_URL", _SERVICE_URL).rstrip("/")
+
+
+def _headers() -> dict[str, str]:
+    headers: dict[str, str] = {}
+    key = os.environ.get("DEVPLACE_GATEWAY_INTERNAL_KEY", _INTERNAL_KEY).strip()
+    if key:
+        headers["X-Internal-Key"] = key
+    return headers
+
+
+def _client() -> httpx.Client:
+    global _CLIENT
+    if _CLIENT is None:
+        _CLIENT = httpx.Client(timeout=30.0)
+    return _CLIENT
+
+
+def _post(path: str, body: dict) -> object:
+    response = _client().post(
+        f"{_service_url()}/{path.lstrip('/')}",
+        json=body,
+        headers=_headers(),
+    )
+    if response.status_code >= 400:
+        payload = response.json() if response.content else {}
+        message = payload.get("error", "Database service request failed")
+        raise RuntimeError(message)
+    if not response.content:
+        return None
+    return decode_value(response.json())
+
+
+def _invoke_cached(fn_name: str, args, kwargs):
+    cache_key = f"{fn_name}:{args!r}:{sorted(kwargs.items())!r}"
+    cached = _SETTINGS_CACHE.get(cache_key)
+    if cached is not None:
+        return cached
+    value = _invoke(fn_name, args, kwargs, write=False)
+    _SETTINGS_CACHE.set(cache_key, value)
+    return value
+
+
+def _invoke(fn_name: str, args, kwargs, *, write: bool = False):
+    encoded_args, encoded_kwargs = encode_args(args, kwargs)
+    payload = {
+        "fn": fn_name,
+        "args": encoded_args,
+        "kwargs": encoded_kwargs,
+        "write": write,
+    }
+    result = _post("internal/invoke", payload)
+    if isinstance(result, dict) and "result" in result:
+        return result["result"]
+    return result
+
+
+class RemoteSearchClause:
+    def __init__(self, term, fields, author_field=None):
+        self.term = term.strip()
+        self.fields = tuple(fields)
+        self.author_field = author_field
+
+
+class RemoteUidInClause:
+    def __init__(self, field, uids):
+        self.field = field
+        self.uids = frozenset(uids)
+
+
+class RemoteTable:
+    def __init__(self, db: "RemoteDb", name: str) -> None:
+        self._db = db
+        self._name = name
+        self._column_cache = None
+
+    def __getattr__(self, name: str):
+        def caller(*args, **kwargs):
+            return self._db._table_op(self._name, name, args, kwargs)
+
+        return caller
+
+    def has_column(self, name: str) -> bool:
+        cache = self._column_cache
+        if cache is None:
+            sample = self.find(_limit=1)
+            row = next(iter(sample), None)
+            cache = set(row.keys()) if row else set()
+            self._column_cache = cache
+        return name in cache
+
+    def count(self, **kwargs):
+        return self._db._table_op(self._name, "count", [], kwargs)
+
+    @property
+    def table(self):
+        return self
+
+    @property
+    def exists(self) -> bool:
+        return self._name in self._db.tables
+
+class RemoteDb:
+    def __init__(self) -> None:
+        self._tables_cache: list[str] | None = None
+
+    @property
+    def tables(self) -> list[str]:
+        if self._tables_cache is None:
+            result = _post("internal/db-op", {"op": "tables"})
+            self._tables_cache = list(result or [])
+        return self._tables_cache
+
+    def __getitem__(self, name: str) -> RemoteTable:
+        return RemoteTable(self, name)
+
+    def query(self, sql: str, **params):
+        encoded_args, encoded_kwargs = encode_args((sql,), params)
+        result = _post(
+            "internal/db-op",
+            {
+                "op": "query",
+                "args": encoded_args,
+                "kwargs": encoded_kwargs,
+                "write": is_write_sql(sql),
+            },
+        )
+        return result or []
+
+    def _table_op(self, table: str, method: str, args, kwargs, *, write: bool = False):
+        encoded_args, encoded_kwargs = encode_args(args, kwargs)
+        result = _post(
+            "internal/db-op",
+            {
+                "op": "table_op",
+                "table": table,
+                "method": method,
+                "args": encoded_args,
+                "kwargs": encoded_kwargs,
+                "write": write,
+            },
+        )
+        if method in {"insert", "update", "delete"}:
+            self._tables_cache = None
+        return result
+
+    @property
+    def executable(self):
+        return self
+
+    @property
+    def in_transaction(self) -> bool:
+        return False
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc, tb):
+        return False
+
+
+_LOCAL_REMOTE = frozenset(
+    {
+        "get_table",
+        "refresh_snapshot",
+        "_in_clause",
+        "_now_iso",
+        "text_search_clause",
+    }
+)
+
+
+def _remote_text_search_clause(
+    table, search, fields=("title", "description"), author_field=None
+):
+    term = (search or "").strip()
+    if not term:
+        return None
+    if type(table).__name__ == "RemoteTable":
+        return RemoteSearchClause(term, fields, author_field)
+    from devplacepy.database.content import text_search_clause as local_clause
+
+    return local_clause(table, search, fields, author_field=author_field)
+
+
+def _remote_get_table(name: str):
+    import devplacepy.database.core as core
+
+    return core.db[name]
+
+
+def _remote_refresh_snapshot() -> None:
+    return None
+
+
+def patch_module(module) -> None:
+    import devplacepy.database as db_module
+
+    for name in db_module.__all__:
+        if name in _LOCAL_REMOTE:
+            continue
+        target = getattr(module, name, None)
+        if target is None or not callable(target):
+            continue
+        if inspect.isclass(target):
+            continue
+
+        def make_wrapper(fn_name: str, fn_write: bool):
+            if fn_name in _CACHED_SETTINGS_FNS:
+
+                def wrapper(*args, **kwargs):
+                    return _invoke_cached(fn_name, args, kwargs)
+
+                wrapper.__name__ = fn_name
+                return wrapper
+
+            def wrapper(*args, **kwargs):
+                return _invoke(fn_name, args, kwargs, write=fn_write)
+
+            wrapper.__name__ = fn_name
+            return wrapper
+
+        setattr(module, name, make_wrapper(name, is_write(name)))
+
+
+def activate() -> None:
+    import devplacepy.database.core as core
+
+    core.db = RemoteDb()
+    import devplacepy.database as db_module
+
+    patch_module(db_module)
+    for submodule_name in (
+        "settings",
+        "users",
+        "relations",
+        "pagination",
+        "soft_delete",
+        "engagement",
+        "usage",
+        "awards",
+        "seo_meta",
+        "activity",
+        "customization",
+        "email",
+        "notifications",
+        "forks",
+        "follows",
+        "deepsearch",
+        "ranking",
+        "comments",
+        "content",
+        "attachments_data",
+        "stats",
+        "schema",
+    ):
+        try:
+            submodule = __import__(
+                f"devplacepy.database.{submodule_name}",
+                fromlist=[submodule_name],
+            )
+        except ImportError:
+            continue
+        patch_module(submodule)
+    for external_name in (
+        "devplacepy.services.statistics.tracking",
+        "devplacepy.services.base",
+        "devplacepy.attachments",
+        "devplacepy.project_files",
+    ):
+        try:
+            external = __import__(external_name, fromlist=[external_name.split(".")[-1]])
+        except ImportError:
+            continue
+        if hasattr(external, "db"):
+            external.db = RemoteDb()
+    db_module.db = core.db
+    db_module.get_table = _remote_get_table
+    core.get_table = _remote_get_table
+    db_module.refresh_snapshot = _remote_refresh_snapshot
+    core.refresh_snapshot = _remote_refresh_snapshot
+    db_module.text_search_clause = _remote_text_search_clause
+    import devplacepy.database.content as content_module
+
+    content_module.text_search_clause = _remote_text_search_clause
+    for submodule_name in (
+        "settings",
+        "users",
+        "relations",
+        "pagination",
+        "soft_delete",
+        "engagement",
+        "usage",
+        "awards",
+        "seo_meta",
+        "activity",
+        "customization",
+        "email",
+        "notifications",
+        "forks",
+        "follows",
+        "deepsearch",
+        "ranking",
+        "comments",
+        "content",
+        "attachments_data",
+        "stats",
+        "schema",
+    ):
+        try:
+            submodule = __import__(
+                f"devplacepy.database.{submodule_name}",
+                fromlist=[submodule_name],
+            )
+        except ImportError:
+            continue
+        if hasattr(submodule, "db"):
+            submodule.db = core.db

devplacepy/database/schema.py

@@ -1,6 +1,6 @@
 # retoor <retoor@molodetz.nl>
 
-from .core import DEFAULT_CORRECTION_PROMPT, DEFAULT_MODIFIER_PROMPT, INTERNAL_GATEWAY_URL, _index, _uid_index, db, defaultdict, get_table, logger
+from .core import DEFAULT_CORRECTION_PROMPT, DEFAULT_MODIFIER_PROMPT, INTERNAL_GATEWAY_URL, _drop_index, _index, _uid_index, db, defaultdict, get_table, logger
 from .settings import get_setting, set_setting
 from .soft_delete import SOFT_DELETE_TABLES, ensure_soft_delete_columns
 from .ranking import _authors_cache
@@ -183,8 +183,10 @@ def init_db():
     )
     _index(db, "badges", "idx_badges_user", ["user_uid"])
     _index(db, "badges", "idx_badges_user_name", ["user_uid", "badge_name"])
-    _index(db, "follows", "idx_follows_follower", ["follower_uid"])
+    _drop_index(db, "idx_follows_follower")
-    _index(db, "follows", "idx_follows_following", ["following_uid"])
+    _drop_index(db, "idx_follows_following")
+    _index(db, "follows", "idx_follows_follower_created", ["follower_uid", "created_at"])
+    _index(db, "follows", "idx_follows_following_created", ["following_uid", "created_at"])
     user_relations = get_table("user_relations")
     for column, example in (
         ("uid", ""),
@@ -363,9 +365,10 @@ def init_db():
         if not conversations.has_column("channel"):
             conversations.create_column_by_example("channel", "main")
         try:
-            db.query(
+            with db:
-                "UPDATE devii_conversations SET channel='main' WHERE channel IS NULL"
+                db.query(
-            )
+                    "UPDATE devii_conversations SET channel='main' WHERE channel IS NULL"
+                )
         except Exception as e:  # noqa: BLE001
             logger.warning(f"Could not backfill devii_conversations.channel: {e}")
     _index(
@@ -391,6 +394,7 @@ def init_db():
         db, "devii_tasks", "idx_devii_tasks_due", ["enabled", "status", "next_run_at"]
     )
     _index(db, "devii_lessons", "idx_devii_lessons_owner", ["owner_kind", "owner_id"])
+    _index(db, "devii_lessons", "idx_devii_lessons_owner_created", ["owner_kind", "owner_id", "created_at"])
     _index(
         db, "devii_virtual_tools", "idx_devii_vtools_owner", ["owner_kind", "owner_id"]
     )
@@ -430,6 +434,12 @@ def init_db():
         "idx_gw_usage_endpoint_time",
         ["endpoint", "created_at"],
     )
+    _index(
+        db,
+        "gateway_usage_ledger",
+        "idx_gw_usage_appref_time",
+        ["app_reference", "created_at"],
+    )
     _index(db, "gateway_concurrency_samples", "idx_gw_conc_time", ["created_at"])
     jobs_table = get_table("jobs")
     for column, example in (
@@ -559,10 +569,11 @@ def init_db():
             correction_usage.create_column_by_example(column, example)
     try:
         if "correction_usage" in db.tables:
-            db.query(
+            with db:
-                "CREATE UNIQUE INDEX IF NOT EXISTS idx_correction_usage_user "
+                db.query(
-                "ON correction_usage (user_uid)"
+                    "CREATE UNIQUE INDEX IF NOT EXISTS idx_correction_usage_user "
-            )
+                    "ON correction_usage (user_uid)"
+                )
     except Exception as e:
         logger.warning(f"Could not create unique index on correction_usage: {e}")
 
@@ -582,10 +593,11 @@ def init_db():
             modifier_usage.create_column_by_example(column, example)
     try:
         if "modifier_usage" in db.tables:
-            db.query(
+            with db:
-                "CREATE UNIQUE INDEX IF NOT EXISTS idx_modifier_usage_user "
+                db.query(
-                "ON modifier_usage (user_uid)"
+                    "CREATE UNIQUE INDEX IF NOT EXISTS idx_modifier_usage_user "
-            )
+                    "ON modifier_usage (user_uid)"
+                )
     except Exception as e:
         logger.warning(f"Could not create unique index on modifier_usage: {e}")
 
@@ -605,10 +617,11 @@ def init_db():
             news_usage.create_column_by_example(column, example)
     try:
         if "news_usage" in db.tables:
-            db.query(
+            with db:
-                "CREATE UNIQUE INDEX IF NOT EXISTS idx_news_usage_user "
+                db.query(
-                "ON news_usage (user_uid)"
+                    "CREATE UNIQUE INDEX IF NOT EXISTS idx_news_usage_user "
-            )
+                    "ON news_usage (user_uid)"
+                )
     except Exception as e:
         logger.warning(f"Could not create unique index on news_usage: {e}")
 
@@ -628,10 +641,11 @@ def init_db():
             issue_usage.create_column_by_example(column, example)
     try:
         if "issue_usage" in db.tables:
-            db.query(
+            with db:
-                "CREATE UNIQUE INDEX IF NOT EXISTS idx_issue_usage_user "
+                db.query(
-                "ON issue_usage (user_uid)"
+                    "CREATE UNIQUE INDEX IF NOT EXISTS idx_issue_usage_user "
-            )
+                    "ON issue_usage (user_uid)"
+                )
     except Exception as e:
         logger.warning(f"Could not create unique index on issue_usage: {e}")
 
@@ -651,10 +665,11 @@ def init_db():
             seo_usage.create_column_by_example(column, example)
     try:
         if "seo_usage" in db.tables:
-            db.query(
+            with db:
-                "CREATE UNIQUE INDEX IF NOT EXISTS idx_seo_usage_user "
+                db.query(
-                "ON seo_usage (user_uid)"
+                    "CREATE UNIQUE INDEX IF NOT EXISTS idx_seo_usage_user "
-            )
+                    "ON seo_usage (user_uid)"
+                )
     except Exception as e:
         logger.warning(f"Could not create unique index on seo_usage: {e}")
 
@@ -674,10 +689,11 @@ def init_db():
             award_usage.create_column_by_example(column, example)
     try:
         if "award_usage" in db.tables:
-            db.query(
+            with db:
-                "CREATE UNIQUE INDEX IF NOT EXISTS idx_award_usage_user "
+                db.query(
-                "ON award_usage (user_uid)"
+                    "CREATE UNIQUE INDEX IF NOT EXISTS idx_award_usage_user "
-            )
+                    "ON award_usage (user_uid)"
+                )
     except Exception as e:
         logger.warning(f"Could not create unique index on award_usage: {e}")
 
@@ -701,21 +717,22 @@ def init_db():
             awards.create_column_by_example(column, example)
     try:
         if "awards" in db.tables:
-            db.query(
+            with db:
-                "CREATE UNIQUE INDEX IF NOT EXISTS idx_awards_slug ON awards (slug)"
+                db.query(
-            )
+                    "CREATE UNIQUE INDEX IF NOT EXISTS idx_awards_slug ON awards (slug)"
-            db.query(
+                )
-                "CREATE INDEX IF NOT EXISTS idx_awards_receiver_created "
+                db.query(
-                "ON awards (receiver_uid, created_at)"
+                    "CREATE INDEX IF NOT EXISTS idx_awards_receiver_created "
-            )
+                    "ON awards (receiver_uid, created_at)"
-            db.query(
+                )
-                "CREATE INDEX IF NOT EXISTS idx_awards_giver_created "
+                db.query(
-                "ON awards (giver_uid, created_at)"
+                    "CREATE INDEX IF NOT EXISTS idx_awards_giver_created "
-            )
+                    "ON awards (giver_uid, created_at)"
-            db.query(
+                )
-                "CREATE INDEX IF NOT EXISTS idx_awards_generated "
+                db.query(
-                "ON awards (receiver_uid, generated_at)"
+                    "CREATE INDEX IF NOT EXISTS idx_awards_generated "
-            )
+                    "ON awards (receiver_uid, generated_at)"
+                )
     except Exception as e:
         logger.warning(f"Could not create awards indexes: {e}")
 
@@ -788,10 +805,11 @@ def init_db():
             user_activity.create_column_by_example(column, example)
     try:
         if "user_activity" in db.tables:
-            db.query(
+            with db:
-                "CREATE UNIQUE INDEX IF NOT EXISTS idx_user_activity_user_action "
+                db.query(
-                "ON user_activity (user_uid, action)"
+                    "CREATE UNIQUE INDEX IF NOT EXISTS idx_user_activity_user_action "
-            )
+                    "ON user_activity (user_uid, action)"
+                )
     except Exception as e:
         logger.warning(f"Could not create unique index on user_activity: {e}")
 
@@ -806,10 +824,11 @@ def init_db():
             user_activity_seen.create_column_by_example(column, example)
     try:
         if "user_activity_seen" in db.tables:
-            db.query(
+            with db:
-                "CREATE UNIQUE INDEX IF NOT EXISTS idx_user_activity_seen_unique "
+                db.query(
-                "ON user_activity_seen (user_uid, action, target)"
+                    "CREATE UNIQUE INDEX IF NOT EXISTS idx_user_activity_seen_unique "
-            )
+                    "ON user_activity_seen (user_uid, action, target)"
+                )
     except Exception as e:
         logger.warning(f"Could not create unique index on user_activity_seen: {e}")
 
@@ -1219,6 +1238,8 @@ def init_db():
             "statistics_tracking_enabled": "1",
             "docs_search_mode": "agent",
             "outbound_proxy_url": "",
+            "devii_lessons_max_per_owner": "500",
+            "devii_lessons_max_age_days": "90",
         }
         for key, value in operational_defaults.items():
             existing = db["site_settings"].find_one(key=key)
@@ -1372,6 +1393,8 @@ def backfill_api_keys() -> int:
         users.create_column_by_example("ai_modifier_sync", 1)
     if not users.has_column("ai_modifier_prompt"):
         users.create_column_by_example("ai_modifier_prompt", DEFAULT_MODIFIER_PROMPT)
+    if not users.has_column("interactions_enabled"):
+        users.create_column_by_example("interactions_enabled", -1)
     if not users.has_column("timezone"):
         users.create_column_by_example("timezone", "")
     if not users.has_column("avatar_seed"):
@@ -1396,6 +1419,10 @@ def backfill_api_keys() -> int:
             "WHERE ai_modifier_prompt IS NULL OR ai_modifier_prompt = ''",
             prompt=DEFAULT_MODIFIER_PROMPT,
         )
+        db.query(
+            "UPDATE users SET interactions_enabled = -1 "
+            "WHERE interactions_enabled IS NULL"
+        )
     import uuid_utils
 
     updated = 0

devplacepy/database/soft_delete.py

@@ -94,11 +94,12 @@ def soft_delete_in(table_name, column, uids, deleted_by, *, stamp=None, **extra)
     for index, (key, value) in enumerate(extra.items()):
         params[f"x{index}"] = value
         extra_sql += f" AND {key} = :x{index}"
-    db.query(
+    with db:
-        f"UPDATE {table_name} SET deleted_at = :dat, deleted_by = :dby "
+        db.query(
-        f"WHERE {column} IN ({placeholders}) AND deleted_at IS NULL{extra_sql}",
+            f"UPDATE {table_name} SET deleted_at = :dat, deleted_by = :dby "
-        **params,
+            f"WHERE {column} IN ({placeholders}) AND deleted_at IS NULL{extra_sql}",
-    )
+            **params,
+        )
     return len(uids)
 
 
@@ -161,11 +162,12 @@ def restore_event(stamp):
                     s=stamp,
                 ).__next__()["n"]
             )
-            db.query(
+            with db:
-                f"UPDATE {table_name} SET deleted_at = NULL, deleted_by = NULL "
+                db.query(
-                f"WHERE deleted_at = :s",
+                    f"UPDATE {table_name} SET deleted_at = NULL, deleted_by = NULL "
-                s=stamp,
+                    f"WHERE deleted_at = :s",
-            )
+                    s=stamp,
+                )
     return restored
 
 
@@ -182,7 +184,8 @@ def purge_event(stamp):
             )
             if rows:
                 purged.append((table_name, rows))
-                db.query(
+                with db:
-                    f"DELETE FROM {table_name} WHERE deleted_at = :s", s=stamp
+                    db.query(
-                )
+                        f"DELETE FROM {table_name} WHERE deleted_at = :s", s=stamp
+                    )
     return purged

devplacepy/db_client.py

@@ -0,0 +1,29 @@
+# retoor <retoor@molodetz.nl>
+
+import os
+
+
+def _activate() -> None:
+    if os.environ.get("DEVPLACE_DB_SERVICE") == "1":
+        return
+    if os.environ.get("DEVPLACE_REMOTE_DB") == "1":
+        from devplacepy.database.remote import activate
+
+        activate()
+
+
+_activate()
+
+import devplacepy.database as _database
+
+
+def _remote_table(table) -> bool:
+    return type(table).__name__ == "RemoteTable"
+
+
+def __getattr__(name: str):
+    return getattr(_database, name)
+
+
+def __dir__():
+    return sorted(name for name in dir(_database) if not name.startswith("_"))

devplacepy/docs_api/groups/gateway.py

@@ -5,7 +5,6 @@ from .._shared import endpoint, field
 GROUP = {
         "slug": "gateway",
         "title": "OpenAI Gateway",
-        "admin": True,
         "intro": """
 # OpenAI Gateway
 
@@ -37,6 +36,26 @@ generic model `molodetz-img-small`, which the gateway maps to the configured ima
 Flux 1.1 Pro by default). Cost is tracked per call with a flat per-image price when the upstream
 returns no native cost.
 
+## Quick start
+
+Copy the command below and paste it into a terminal. If you are signed in the `{{ api_key }}`
+and `{{ app_reference }}` placeholders are already filled in with your own values; otherwise
+replace them with the API key from your [profile](/profile) page and any application identifier.
+
+```bash
+curl -X POST "{{ base }}/openai/v1/chat/completions" \
+  -H "Authorization: Bearer {{ api_key }}" \
+  -H "X-App-Reference: {{ app_reference }}" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "molodetz",
+    "messages": [{"role": "user", "content": "Hello, how are you?"}]
+  }'
+```
+
+The response carries `X-Gateway-*` headers with token counts and dollar cost for the call.
+For streaming, add `"stream": true` to the JSON body.
+
 ## Model routing and providers
 
 On top of the single default upstream above, an administrator can register additional named
@@ -90,6 +109,17 @@ Dollar costs use the upstream's native `cost` field when it returns one
 model route, falling back to the prices configured on the `openai` service when no route matches. The
 denied paths that make no upstream call (embeddings or image generation disabled) return no usage headers.
 
+## Request header `X-App-Reference`
+
+Clients **SHOULD** send an `X-App-Reference` header to identify themselves for cost attribution.
+The value is a free-form slug (max 30 characters, letters, digits, `_`, `.`, `-`). When missing or
+invalid, the gateway defaults to `default`. The value is recorded in every usage ledger row and can
+be queried alongside owner-kind and owner-id to attribute spending per application.
+
+```
+X-App-Reference: devplace-devii-v-1-0-0
+```
+
 Administrators enable and configure this gateway under [Background Services](/docs/services.html)
 (the `openai` service).
 
@@ -104,7 +134,7 @@ for signing DevPlace's own requests.
                 path="/openai/v1/chat/completions",
                 title="Chat completions",
                 summary="OpenAI-compatible chat completion. Supports streaming.",
-                auth="user",
+                auth="public",
                 encoding="json",
                 params=[
                     field(
@@ -113,7 +143,7 @@ for signing DevPlace's own requests.
                         "string",
                         False,
                         "gpt-4o-mini",
-                        "Model id. When it matches a configured model route the gateway forwards to that route's provider and upstream model; otherwise it uses the default upstream model.",
+                        "Model id. When it matches a configured model route the gateway forwards to that route's provider and upstream model; otherwise it falls back to the configured default upstream model.",
                     ),
                     field(
                         "messages",
@@ -144,7 +174,7 @@ for signing DevPlace's own requests.
                 path="/openai/v1/embeddings",
                 title="Embeddings",
                 summary="OpenAI-compatible text embeddings. Request model molodetz~embed.",
-                auth="user",
+                auth="public",
                 encoding="json",
                 params=[
                     field(
@@ -175,6 +205,7 @@ for signing DevPlace's own requests.
                 notes=[
                     "Returns `503` when the gateway service is not running or embeddings are disabled.",
                     "Every response carries the `X-Gateway-*` token and dollar-cost headers (see Per-call cost and usage headers above).",
+                    "If `model` matches a configured embed model route it is forwarded to that route's provider and upstream model; otherwise it falls back to the configured default embedding model.",
                 ],
             ),
             endpoint(
@@ -183,7 +214,7 @@ for signing DevPlace's own requests.
                 path="/openai/v1/images/generations",
                 title="Image generation",
                 summary="OpenAI-compatible image generation. Request model molodetz-img-small.",
-                auth="user",
+                auth="public",
                 encoding="json",
                 params=[
                     field(
@@ -222,6 +253,7 @@ for signing DevPlace's own requests.
                 notes=[
                     "Returns `503` when the gateway service is not running or image generation is disabled.",
                     "Every response carries the `X-Gateway-*` token and dollar-cost headers (see Per-call cost and usage headers above).",
+                    "If `model` matches a configured image model route it is forwarded to that route's provider and upstream model; otherwise it falls back to the configured default image model.",
                 ],
             ),
             endpoint(
@@ -230,7 +262,7 @@ for signing DevPlace's own requests.
                 path="/openai/v1/{path}",
                 title="Passthrough",
                 summary="Any other /v1 path is forwarded to the upstream as-is.",
-                auth="user",
+                auth="public",
                 interactive=False,
                 params=[
                     field(

devplacepy/docs_api/groups/profiles.py

@@ -136,7 +136,7 @@ four ways to sign requests.
                         "textarea",
                         False,
                         "Leave literary as is, only do punctuation and casing",
-                        "Correction instruction, up to 2000 characters.",
+                        "Correction instruction, up to 20000 characters.",
                     ),
                 ],
                 sample_response={
@@ -150,6 +150,53 @@ four ways to sign requests.
                     },
                 },
             ),
+            endpoint(
+                id="profile-interactions",
+                method="POST",
+                path="/profile/{username}/interactions",
+                title="Configure Devii interactive widgets",
+                summary="Enable or disable CA-IWP interactive prompts (ui_prompt) for this account, or reset to the administrator default. Guests always use the site default. Admins may target any user.",
+                auth="user",
+                encoding="form",
+                destructive=True,
+                params=[
+                    field(
+                        "username",
+                        "path",
+                        "string",
+                        True,
+                        "bob_test",
+                        "Profile owner. Must be yourself unless you are an admin.",
+                    ),
+                    field(
+                        "enabled",
+                        "form",
+                        "boolean",
+                        False,
+                        "true",
+                        "true to enable interactive widgets, false to disable. Ignored when reset is true.",
+                    ),
+                    field(
+                        "reset",
+                        "form",
+                        "boolean",
+                        False,
+                        "false",
+                        "true to clear the user override and inherit the administrator default.",
+                    ),
+                ],
+                sample_response={
+                    "ok": True,
+                    "redirect": "/profile/bob_test",
+                    "data": {
+                        "url": "/profile/bob_test",
+                        "enabled": True,
+                        "source": "user",
+                        "default": True,
+                        "override": True,
+                    },
+                },
+            ),
             endpoint(
                 id="profile-ai-modifier",
                 method="POST",
@@ -190,7 +237,7 @@ four ways to sign requests.
                         "textarea",
                         False,
                         "Execute what is behind `@ai` (the prompt) and replace that part including `@ai`",
-                        "Modifier instruction, up to 2000 characters.",
+                        "Modifier instruction, up to 20000 characters.",
                     ),
                 ],
                 sample_response={

devplacepy/docs_api/render.py

@@ -43,5 +43,6 @@ def render_group(slug, base, username, api_key):
         "{{ base }}": base,
         "{{ username }}": username or "YOUR_USERNAME",
         "{{ api_key }}": api_key or "YOUR_API_KEY",
+        "{{ app_reference }}": f"user-{username}-app-v-1-0-0" if username else "user-app-v-13.37.0",
     }
     return _substitute(group, replacements)

devplacepy/docs_devrant.py

@@ -237,7 +237,7 @@ DEVRANT_GROUPS = {
             encoding="form",
             params=[
                 field("rant_id", "path", type="int", required=True, example="1", description="Rant id."),
-                field("comment", "body", type="textarea", required=True, example="Great rant!", description="Comment text, 1-1000 chars."),
+                field("comment", "body", type="textarea", required=True, example="Great rant!", description="Comment text, 1-125000 chars."),
             ],
             sample_response={"success": True},
         ),

devplacepy/main.py

@@ -37,8 +37,10 @@ from devplacepy.database import (
     get_user_post_count,
     get_user_stars,
     get_blocked_uids,
+    get_top_authors,
+    get_trending_topics,
 )
-from devplacepy.templating import templates
+from devplacepy.templating import templates, jinja_unread_count
 from devplacepy.cache import TTLCache
 from devplacepy.responses import respond, wants_json, json_error
 from devplacepy.schemas import LandingOut, ValidationErrorOut
@@ -465,7 +467,8 @@ app.include_router(game.router, prefix="/game")
 
 @app.middleware("http")
 async def refresh_db_snapshot(request: Request, call_next):
-    refresh_snapshot()
+    if not request.url.path.startswith(("/static", "/avatar")):
+        refresh_snapshot()
     return await call_next(request)
 
 
@@ -608,7 +611,7 @@ async def response_timing(request: Request, call_next):
     return response
 
 
-app.add_middleware(GZipMiddleware, minimum_size=512, compresslevel=6)
+app.add_middleware(GZipMiddleware, minimum_size=512, compresslevel=5)
 
 
 _home_cache = TTLCache(ttl=int(os.environ.get("DEVPLACE_HOME_CACHE_TTL", "60")), max_size=4)
@@ -700,6 +703,14 @@ async def landing(request: Request):
         breadcrumbs=[],
         schemas=[website_schema(base)],
     )
+    user_xp = user.get("xp", 0) or 0 if user else 0
+    user_level = user.get("level", 1) or 1 if user else 1
+    xp_progress_pct = (user_xp % 100) if user_xp else 0
+    unread_count = jinja_unread_count(user["uid"]) if user else 0
+
+    top_contributors = get_top_authors(5) if not blocked else []
+    trending_topics = get_trending_topics(6) if not blocked else []
+
     return respond(
         request,
         "landing.html",
@@ -710,8 +721,14 @@ async def landing(request: Request):
             "is_authenticated": bool(user),
             "user_post_count": get_user_post_count(user["uid"]) if user else 0,
             "user_stars": get_user_stars(user["uid"]) if user else 0,
+            "user_xp": user_xp,
+            "user_level": user_level,
+            "xp_progress_pct": xp_progress_pct,
+            "unread_count": unread_count,
             "landing_articles": landing_articles,
             "landing_posts": landing_posts,
+            "top_contributors": top_contributors,
+            "trending_topics": trending_topics,
         },
         model=LandingOut,
     )

devplacepy/models.py

@@ -158,7 +158,7 @@ class PostEditForm(BaseModel):
 
 
 class CommentForm(BaseModel):
-    content: str = Field(min_length=3, max_length=1000)
+    content: str = Field(min_length=3, max_length=125000)
     target_uid: str = Field(default="", max_length=36)
     post_uid: str = Field(default="", max_length=36)
     target_type: Literal["post", "project", "news", "issue", "gist"] = "post"
@@ -173,7 +173,7 @@ class CommentForm(BaseModel):
 
 
 class CommentEditForm(BaseModel):
-    content: str = Field(min_length=3, max_length=1000)
+    content: str = Field(min_length=3, max_length=125000)
 
 
 class ProjectForm(BaseModel):
@@ -258,13 +258,18 @@ class NotificationDefaultForm(BaseModel):
 class AiCorrectionForm(BaseModel):
     enabled: bool = False
     sync: bool = False
-    prompt: str = Field(default=DEFAULT_CORRECTION_PROMPT, max_length=2000)
+    prompt: str = Field(default=DEFAULT_CORRECTION_PROMPT, max_length=20000)
 
 
 class AiModifierForm(BaseModel):
     enabled: bool = False
     sync: bool = False
-    prompt: str = Field(default=DEFAULT_MODIFIER_PROMPT, max_length=2000)
+    prompt: str = Field(default=DEFAULT_MODIFIER_PROMPT, max_length=20000)
+
+
+class InteractionsForm(BaseModel):
+    enabled: bool = True
+    reset: bool = False
 
 
 class TelegramPairForm(BaseModel):

devplacepy/rendering.py

@@ -45,6 +45,9 @@ def write_emoji_module(path: Path = EMOJI_JS_PATH) -> int:
 
 EMOJI_MAP = build_emoji_shortcodes()
 
+_WIDGET_RE = re.compile(r"<dp-widget>(.*?)</dp-widget>", re.DOTALL | re.IGNORECASE)
+_WIDGET_PH = "\x00WIDGET_{}\x00"
+
 _SHORTCODE_RE = re.compile(r":([A-Za-z0-9_+\-]+):")
 _YOUTUBE_RE = re.compile(
     r"(?:https?://)?(?:www\.)?"
@@ -102,7 +105,17 @@ _content_markdown = mistune.create_markdown(
 
 
 def _normalize_dashes(text: str) -> str:
-    return text.replace("\u2014", "-")
+    text = text.replace("\u2014", "-")
+    text = text.replace("\u2013", "-")
+    text = text.replace("&mdash;", "-")
+    text = text.replace("&ndash;", "-")
+    text = text.replace("&#8212;", "-")
+    text = text.replace("&#8211;", "-")
+    text = text.replace("&#x2014;", "-")
+    text = text.replace("&#X2014;", "-")
+    text = text.replace("&#x2013;", "-")
+    text = text.replace("&#X2013;", "-")
+    return text
 
 
 def _replace_shortcodes(text: str) -> str:
@@ -252,16 +265,42 @@ def _render_title(text: str) -> str:
     return _keep_inline(_content_markdown(text))
 
 
-def render_content(text) -> Markup:
+def _extract_widgets(text: str) -> tuple[str, list[str]]:
-    if not text:
+    widgets: list[str] = []
-        return Markup("")
+    def _replacer(m: re.Match) -> str:
-    return Markup(_render_content(str(text)))
+        widgets.append(m.group(1))
+        return _WIDGET_PH.format(len(widgets) - 1)
+    return _WIDGET_RE.sub(_replacer, text), widgets
 
 
-def render_title(text) -> Markup:
+def _reinsert_widgets(text: str, widgets: list[str]) -> str:
+    for i, widget in enumerate(widgets):
+        text = text.replace(_WIDGET_PH.format(i), widget)
+    return text
+
+
+def render_content(text, author_is_admin: bool = False) -> Markup:
     if not text:
         return Markup("")
-    return Markup(_render_title(str(text)))
+    text_str = str(text)
+    if author_is_admin and _WIDGET_RE.search(text_str):
+        modified, widgets = _extract_widgets(text_str)
+        rendered = _render_content(modified)
+        result = _reinsert_widgets(rendered, widgets)
+        return Markup(result)
+    return Markup(_render_content(text_str))
+
+
+def render_title(text, author_is_admin: bool = False) -> Markup:
+    if not text:
+        return Markup("")
+    text_str = str(text)
+    if author_is_admin and _WIDGET_RE.search(text_str):
+        modified, widgets = _extract_widgets(text_str)
+        rendered = _render_title(modified)
+        result = _reinsert_widgets(rendered, widgets)
+        return Markup(result)
+    return Markup(_render_title(text_str))
 
 
 def content_preview(text, length: int = 60) -> str:

devplacepy/routers/CLAUDE.md

@@ -14,7 +14,7 @@ Prefixes are wired in `main.py`:
 | `/comments` | comments.py |
 | `/projects` | projects/ package - `index.py` (listing/detail/create/delete plus owner-only visibility toggles `POST /{slug}/private` and `POST /{slug}/readonly`, and async `POST /{slug}/fork`), `files.py`, and the `containers/` subpackage (below). `main.py` mounts the whole `/projects` tree from this one package. See `routers/projects/CLAUDE.md` for the deep detail on this tree |
 | `/projects/{slug}/files` | projects/files.py - per-project virtual filesystem (CRUD dirs/files, upload, inline edit, plus line-range ops: `lines` read, `replace-lines`, `insert-lines`, `delete-lines`, `append` for large text files) |
-| `/profile` | profile/ package - `index.py` (page/search/followers/update/api-key; the **Posts** tab shows only the 10 most recent posts, while the sidebar `posts_count` stat always uses the real `count()` so it is never capped to the rendered 10), `customization.py`, `notifications.py`, `ai_correction.py` (`POST /{username}/ai-correction`, owner-or-admin AI content correction toggle + prompt), `ai_modifier.py` (`POST /{username}/ai-modifier`, owner-or-admin AI modifier toggle + prompt), `telegram.py` (`POST /{username}/telegram`, owner-or-admin Telegram pairing-code request/unpair), `usage.py` (`_ai_quota`, re-exported from the package) |
+| `/profile` | profile/ package - `index.py` (page/search/followers/update/api-key; the **Posts** tab shows only the 10 most recent posts, while the sidebar `posts_count` stat always uses the real `count()` so it is never capped to the rendered 10), `customization.py`, `notifications.py`, `ai_correction.py` (`POST /{username}/ai-correction`, owner-or-admin AI content correction toggle + prompt), `ai_modifier.py` (`POST /{username}/ai-modifier`, owner-or-admin AI modifier toggle + prompt), `interactions.py` (`POST /{username}/interactions`, owner-or-admin Devii interactive widgets preference; admin default via `devii_interactions_default`), `telegram.py` (`POST /{username}/telegram`, owner-or-admin Telegram pairing-code request/unpair), `usage.py` (`_ai_quota`, re-exported from the package) |
 | `/messages` | messages.py - real-time DM chat. `GET ""` (page; the conversation sidebar is one windowed SQL aggregate and an opened thread loads only the newest `CONVERSATION_MESSAGE_LIMIT` = 500 messages - keep these queries bounded), `GET /search`, `POST /send` (no-JS fallback), and `WS /messages/ws` (NOT lock-owner gated - accepts on every worker; closes `1008` for guests). The WS adds live bidirectional delivery, typing, read receipts, and in-process presence on top of the existing `messages` table; both the WS `send` and HTTP `POST /send` route through the shared `services/messaging/persist.py` `persist_message` choke point (identical audit/notification/mention) and then through `_finalize_and_broadcast`, which awaits any pending SYNC AI correction/modifier futures (DM content runs through both, so typing `@ai <instruction>` executes live) and broadcasts the FINAL corrected/modified content (the WS path passes `request=websocket` and awaits directly since HTTP middleware does not run for websockets; the frontend swaps the sender's optimistic bubble to the authoritative content). Same-worker delivery is instant via the per-worker `message_hub`; cross-worker delivery is filled by `services/messaging/relay.py` `message_relay` (per-worker ~1s DB poll on the `messages.id` watermark, deduped against the hub's delivered set). See `devplacepy/services/messaging/CLAUDE.md` |
 | `/notifications` | notifications.py |
 | `/votes` | votes.py |

devplacepy/routers/avatar.py

@@ -17,15 +17,14 @@ _CACHE_CONTROL = f"public, max-age={SECONDS_PER_DAY}, immutable"
 
 @router.get("/{style}/{seed}")
 async def avatar_proxy(request: Request, style: str, seed: str, size: int = 128):
-    cache_key = f"{seed}:{size}"
+    etag = '"' + hashlib.md5(f"{seed}:{size}".encode("utf-8")).hexdigest() + '"'
-    etag = '"' + hashlib.md5(cache_key.encode("utf-8")).hexdigest() + '"'
     headers = {"ETag": etag, "Cache-Control": _CACHE_CONTROL}
 
     if request.headers.get("if-none-match") == etag:
         return Response(status_code=304, headers=headers)
 
-    svg = _cache.get(cache_key)
+    svg = _cache.get(seed)
     if svg is None:
         svg = generate_avatar_svg(seed)
-        _cache.set(cache_key, svg)
+        _cache.set(seed, svg)
     return Response(content=svg, media_type="image/svg+xml", headers=headers)

devplacepy/routers/devii.py

@@ -185,7 +185,10 @@ async def clippy_proxy(request: Request):
         return JSONResponse({"error": "Devii is unavailable"}, status_code=503)
     cfg = svc.effective_config()
     body = await request.body()
-    headers = {"Content-Type": "application/json"}
+    headers = {
+        "Content-Type": "application/json",
+        "X-App-Reference": "devplace-devii-v-1-0-0",
+    }
     if cfg.get("devii_ai_key"):
         headers["Authorization"] = f"Bearer {cfg['devii_ai_key']}"
     async with stealth.stealth_async_client(timeout=45.0) as client:
@@ -260,6 +263,8 @@ async def devii_ws(websocket: WebSocket):
                 if command == "reset":
                     await session.reset()
                     continue
+                if await session.try_answer_interaction(text):
+                    continue
                 if svc.quota_exceeded(owner_kind, owner_id, owner_is_admin):
                     limit = svc.daily_limit_for(owner_kind, owner_is_admin)
                     audit.record_system(
@@ -302,6 +307,11 @@ async def devii_ws(websocket: WebSocket):
                 )
             elif kind in ("avatar_result", "client_result"):
                 session.resolve_query(str(data.get("id", "")), data.get("result"))
+            elif kind == "interaction_result":
+                session.resolve_interaction(
+                    str(data.get("id", data.get("interaction_id", ""))),
+                    data.get("result") or data,
+                )
     except WebSocketDisconnect:
         pass
     except Exception:  # noqa: BLE001 - never let the socket loop crash the worker

devplacepy/routers/gists.py

@@ -57,6 +57,7 @@ LANGUAGES = [
     ("yaml", "YAML"),
     ("json", "JSON"),
     ("markdown", "Markdown"),
+    ("markdown_rendered", "Markdown Rendered"),
     ("swift", "Swift"),
     ("php", "PHP"),
     ("ruby", "Ruby"),

devplacepy/routers/messages.py

@@ -231,7 +231,7 @@ async def send_message(request: Request, data: Annotated[MessageForm, Depends(js
 async def broadcast_message(
     sender: dict, message: dict, client_id: Optional[str] = None
 ) -> None:
-    frame = message_frame(message, sender.get("username", ""), client_id)
+    frame = message_frame(message, sender.get("username", ""), client_id, sender_role=sender.get("role"))
     message_hub.mark_delivered(message["uid"])
     targets = [message["sender_uid"], message["receiver_uid"]]
     await message_hub.send_to_users(targets, frame)

devplacepy/routers/profile/__init__.py

@@ -6,6 +6,7 @@ from devplacepy.routers.profile import (
     avatar,
     award,
     customization,
+    interactions,
     notifications,
     telegram,
 )
@@ -17,6 +18,7 @@ router.include_router(customization.router)
 router.include_router(notifications.router)
 router.include_router(ai_correction.router)
 router.include_router(ai_modifier.router)
+router.include_router(interactions.router)
 router.include_router(avatar.router)
 router.include_router(telegram.router)
 

devplacepy/routers/profile/index.py

@@ -12,6 +12,7 @@ from devplacepy.database import (
     get_notification_prefs,
     get_user_stars,
     get_user_rank,
+    get_user_post_count,
     get_comment_counts_by_post_uids,
     get_reactions_by_targets,
     get_user_bookmarks,
@@ -211,9 +212,7 @@ async def profile_page(
         )
         for g in gists_raw:
             gists.append({"gist": g, "time_ago": time_ago(g["created_at"])})
-    posts_count = get_table("posts").count(
+    posts_count = get_user_post_count(profile_user["uid"])
-        user_uid=profile_user["uid"], deleted_at=None
-    )
 
     activities = []
     if tab == "activity":
@@ -302,6 +301,18 @@ async def profile_page(
         if is_owner
         else None
     )
+    from devplacepy.services.devii.interaction import prefs as interaction_prefs
+
+    interactions_snap = (
+        interaction_prefs.snapshot("user", profile_user["uid"], profile_user)
+        if is_owner
+        else {
+            "enabled": True,
+            "source": None,
+            "default": True,
+            "override": None,
+        }
+    )
     from devplacepy.services.telegram import store as telegram_store
 
     telegram_paired = (
@@ -402,6 +413,10 @@ async def profile_page(
             "ai_modifier_enabled": ai_modifier_enabled,
             "ai_modifier_sync": ai_modifier_sync,
             "ai_modifier_prompt": ai_modifier_prompt,
+            "interactions_enabled": interactions_snap["enabled"],
+            "interactions_source": interactions_snap["source"],
+            "interactions_default": interactions_snap["default"],
+            "interactions_override": interactions_snap["override"],
             "telegram_paired": telegram_paired,
             "notif_telegram_paired": notif_telegram_paired,
             "can_manage_customization": can_manage_customization,

devplacepy/routers/profile/interactions.py

@@ -0,0 +1,61 @@
+# retoor <retoor@molodetz.nl>
+
+import logging
+from typing import Annotated
+
+from fastapi import Depends, APIRouter, Request
+
+from devplacepy.models import InteractionsForm
+from devplacepy.responses import action_result
+from devplacepy.services.audit import record as audit
+from devplacepy.services.devii.interaction import prefs
+from devplacepy.routers.profile._shared import resolve_customization_target
+from devplacepy.dependencies import json_or_form
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+
+@router.post("/{username}/interactions")
+async def set_interactions(
+    request: Request,
+    username: str,
+    data: Annotated[InteractionsForm, Depends(json_or_form(InteractionsForm))],
+):
+    target, denied = resolve_customization_target(request, username)
+    if denied is not None:
+        return denied
+    if data.reset:
+        snap = prefs.set_user_pref(target["uid"], None)
+        summary = f"reset interactive widgets to admin default for {target['username']}"
+        new_value = -1
+    else:
+        snap = prefs.set_user_pref(target["uid"], bool(data.enabled))
+        summary = (
+            f"{'enabled' if data.enabled else 'disabled'} interactive widgets "
+            f"for {target['username']}"
+        )
+        new_value = 1 if data.enabled else 0
+    logger.info(summary)
+    audit.record(
+        request,
+        "profile.interactions",
+        target_type="user",
+        target_uid=target["uid"],
+        target_label=target["username"],
+        new_value=new_value,
+        summary=summary,
+        links=[audit.target("user", target["uid"], target["username"])],
+    )
+    url = f"/profile/{target['username']}"
+    return action_result(
+        request,
+        url,
+        data={
+            "url": url,
+            "enabled": snap["enabled"],
+            "source": snap["source"],
+            "default": snap["default"],
+            "override": snap["override"],
+        },
+    )

devplacepy/routers/xmlrpc.py

@@ -4,6 +4,7 @@ import logging
 
 import httpx
 from fastapi import APIRouter, Request
+from starlette.requests import ClientDisconnect
 from starlette.responses import PlainTextResponse, Response
 
 from devplacepy.config import XMLRPC_BIND, XMLRPC_PORT
@@ -49,7 +50,11 @@ async def info(request: Request, path: str = "") -> PlainTextResponse:
 @router.post("/")
 @router.post("/{path:path}")
 async def proxy(request: Request, path: str = "") -> Response:
-    body = await request.body()
+    try:
+        body = await request.body()
+    except ClientDisconnect:
+        logger.warning("XML-RPC client disconnected before request body was read")
+        return PlainTextResponse("Client disconnected", status_code=400)
     headers = {
         key: value
         for key, value in request.headers.items()

devplacepy/schemas/auth.py

@@ -40,12 +40,23 @@ class LandingPostOut(_Out):
     slug: str = ""
 
 
+class TrendingTopicOut(_Out):
+    topic: str = ""
+    count: int = 0
+
+
 class LandingOut(_Out):
     is_authenticated: bool = False
     user_post_count: int = 0
     user_stars: int = 0
+    user_xp: int = 0
+    user_level: int = 1
+    xp_progress_pct: int = 0
+    unread_count: int = 0
     landing_articles: list[LandingArticleOut] = []
     landing_posts: list[LandingPostOut] = []
+    top_contributors: list = []
+    trending_topics: list[TrendingTopicOut] = []
 
 
 class DeviiPageOut(_Out):

devplacepy/schemas/profile.py

@@ -50,6 +50,10 @@ class ProfileOut(_Out):
     ai_modifier_enabled: bool = False
     ai_modifier_sync: bool = False
     ai_modifier_prompt: Optional[str] = None
+    interactions_enabled: bool = True
+    interactions_source: Optional[str] = None
+    interactions_default: bool = True
+    interactions_override: Optional[bool] = None
     telegram_paired: bool = False
     notif_telegram_paired: bool = False
     can_manage_customization: bool = False

devplacepy/services/CLAUDE.md

@@ -22,9 +22,9 @@ Generic, lightweight, **fire-and-forget** offload for non-critical side-effects
 
 **Opt-in, default off, server-side.** When a user turns it on (profile **AI content correction** block, owner-only, saved at `POST /profile/{username}/ai-correction`), the prose they author is rewritten by the AI gateway. The per-user `ai_correction_sync` flag (0 = background default, 1 = sync) picks the **apply mode**: background rewrites the stored fields a moment after the write (never slows the request); sync makes the HTTP response wait until the correction is applied.
 
-- **Sync must never block the event loop (the self-deadlock trap).** The correction POSTs to the in-process gateway (`INTERNAL_GATEWAY_URL` = `localhost:{PORT}`), and the hooked content helpers are synchronous on the loop thread, so a blocking inline call self-deadlocks the (single) worker against its own gateway request - on a single worker that loop is the only thing that can serve the gateway request it is waiting on, so a blocking inline call deadlocks the worker until the httpx timeout, then fail-softs (server hangs, no correction). The fix: sync runs `_run_correction` via `loop.run_in_executor` (loop stays free), stashes the future on `request.scope[PENDING_SCOPE_KEY]`, and the `await_pending_corrections` middleware in `main.py` awaits it after the handler. Never reintroduce a blocking inline correction on the loop thread.
+- **Sync must never block the event loop (the self-deadlock trap).** The correction POSTs to the in-process gateway (`INTERNAL_GATEWAY_URL` = `localhost:{PORT}`), and the hooked content helpers are synchronous on the loop thread, so a blocking inline call self-deadlocks the (single) worker against its own gateway request - on a single worker that loop is the only thing that can serve the gateway request it is waiting on, so a blocking inline call deadlocks the worker until the httpx timeout, then fail-softs (server hangs, no correction). The fix: sync runs `_run_correction` on the dedicated `AI_APPLY_EXECUTOR` thread pool (`correction.py`, 4 workers) via `loop.run_in_executor` (loop stays free, and the default executor - shared with `asyncio.to_thread` password hashing at login/signup - is never occupied by blocking AI HTTP calls), stashes the future on `request.scope[PENDING_SCOPE_KEY]`, and the `await_pending_corrections` middleware in `main.py` awaits it after the handler. Never reintroduce a blocking inline correction on the loop thread.
 - **Field registry is the single source of truth.** `CORRECTABLE_FIELDS: dict[str, tuple[str, ...]]` maps each correctable table to its prose columns: `posts` -> `(title, content)`, `projects`/`gists` -> `(title, description)`, `comments`/`messages` -> `(content,)`, `users` -> `(bio,)`. `gists.source_code`, project files, and Gitea issues are intentionally excluded - code and external systems are never corrected.
-- **Choke entrypoint.** `schedule_correction(user, table, uid, request=None)` is the only thing handlers call (every hook passes `request`). It is a no-op unless: a user dict is present, `table` is in the registry, `user["ai_correction_enabled"]` is truthy, and the user has a non-empty `api_key`. In **sync** mode (`user["ai_correction_sync"]`) it calls `_run_inline_awaited`, which - only when a `request.scope` and a running event loop exist - submits `_run_correction` to `loop.run_in_executor` (off the loop thread) and stashes the future on `request.scope[PENDING_SCOPE_KEY]` for the middleware to await; if there is no request/loop it returns False and falls back to background. In **background** mode (default) it `background.submit`s `_run_correction` (per-worker queue; also runs inline under `DEVPLACE_DISABLE_SERVICES=1`). The same `_run_correction` worker function runs in all cases.
+- **Choke entrypoint.** `schedule_correction(user, table, uid, request=None)` is the only thing handlers call (every hook passes `request`). It is a no-op unless: a user dict is present, `table` is in the registry, `user["ai_correction_enabled"]` is truthy, and the user has a non-empty `api_key`. In **sync** mode (`user["ai_correction_sync"]`) it calls `_run_inline_awaited`, which - only when a `request.scope` and a running event loop exist - submits `_run_correction` to `loop.run_in_executor(AI_APPLY_EXECUTOR, ...)` (off the loop thread, on the dedicated AI pool) and stashes the future on `request.scope[PENDING_SCOPE_KEY]` for the middleware to await; if there is no request/loop it returns False and falls back to background. In **background** mode (default) it `background.submit`s `_run_correction` (per-worker queue; also runs inline under `DEVPLACE_DISABLE_SERVICES=1`). The same `_run_correction` worker function runs in all cases.
 - **The hooks (covers UI + REST + Devii + devRant in one place):** `content.create_content_item` (posts/projects/gists), `content.edit_content_item`, `content.create_comment_record`, `content.edit_comment_record`, `services/messaging/persist.persist_message` (DMs - sender is the user), `routers/profile/index.update_profile` (bio), and the two devRant direct-update edit paths (`routers/devrant/rants.edit_rant`, `routers/devrant/comments.edit_comment`). devRant create paths route through the shared content cores, so they are already hooked. Code-only and external paths (project files, Gitea) are deliberately not hooked.
 - **The gateway call is fail-soft.** `correct_text(api_key, prompt, text)` is synchronous (runs on the background worker thread), POSTs to `INTERNAL_GATEWAY_URL` with `model=INTERNAL_MODEL` via `stealth.stealth_sync_client`, authenticated with the user's own `Bearer` api_key (per-user attribution). It returns the ORIGINAL text on any error, empty output, or suspiciously large output (`len > len(text) * MAX_GROWTH_FACTOR + 200`, rejecting hallucinated expansion). `_run_correction` reads the row back, corrects each registry field that has non-blank text, writes only changed fields via `table.update(updates, ["uid"])` without touching `updated_at` (auto-correction is not a user edit) or the slug (slugs are permanent), and `clear_user_cache(user_uid)` when `table == "users"` so the corrected bio re-caches.
 - **Per-user usage aggregation (only on success).** `correct_text` returns `(text, usage)`; `usage` is parsed from the gateway's `X-Gateway-*` response headers (`_usage_from_headers`) whenever the upstream call returned 200 (cost was incurred, even if the corrected output was rejected), else `None` on any failure. `_usage_from_headers` captures the token and cost headers PLUS the timing headers `X-Gateway-Upstream-Latency-Ms` and `X-Gateway-Total-Latency-Ms` (as `upstream_latency_ms`/`total_latency_ms`), so each call's timing is metered. `_run_correction` accumulates the per-field `usage` into one `totals` dict and, when `totals["calls"] > 0`, makes ONE call to `database.add_correction_usage(user_uid, totals)` (a single `totals` dict, not positional args) - so a 2-field content item is a single aggregated write, and a failed/empty correction records nothing. `add_correction_usage` (and `add_modifier_usage`) delegate to the shared `database._add_usage(usage_table, user_uid, totals)`: a single atomic `INSERT ... ON CONFLICT(user_uid) DO UPDATE SET col = col + excluded.col` upsert against the `correction_usage` table (per-user running SUMS: `calls`/`prompt_tokens`/`completion_tokens`/`total_tokens`/`cost_usd`/`upstream_latency_ms`/`total_latency_ms`/`updated_at`, unique index `idx_correction_usage_user`, the two latency columns REAL default 0.0, all ensured in `init_db`). It is a derived counter table (NOT in `SOFT_DELETE_TABLES`, like `gateway_usage_ledger`) and is deliberately separate from `users` so accumulating never invalidates the auth/user cache. `database.get_correction_usage(user_uid)` (via `_get_usage`) returns the stored sums PLUS computed averages: `avg_tokens` (total_tokens/calls), `avg_upstream_latency_ms`, `avg_total_latency_ms`, `avg_tokens_per_second` (completion_tokens over total upstream seconds), and `avg_cost_usd`.
@@ -57,7 +57,7 @@ This section covers only the shared machinery. The individual services built on
 
 In production (`make prod`, 2 workers) only the lock-holding worker runs services, but an admin request can hit either worker. State therefore lives in the DB, not process memory:
 - **Desired/config state** in `site_settings` (via `get_setting`/`set_setting`): `service_<name>_enabled` (`"0"`/`"1"` - also the boot flag), `service_<name>_command` (`"<verb>:<counter>"`, verbs `run`/`clear`), `service_<name>_log_size`, plus each declared config field's own key.
-- **Observed state** in the `service_state` table (one row per service): `status`, `last_run`, `next_run`, `started_at`, `heartbeat`, `logs` (JSON), `updated_at`. Written by the supervising worker, read by any worker.
+- **Observed state** in the `service_state` table (one row per service): `status`, `last_run`, `next_run`, `started_at`, `heartbeat`, `logs` (JSON), `updated_at`. Written by the supervising worker, read by any worker. The heartbeat persists every `PERSIST_SECONDS` (8s, under the 15s `STALE_SECONDS` liveness window) and caches the row id so a persist is one UPDATE, not a `find_one` + UPDATE. `collect_metrics()` runs on its own slower `METRICS_SECONDS` cadence (15s default, per-class overridable - `AuditService` uses 300s because its metric is a `COUNT(*)` over the ever-growing audit table); between refreshes the last snapshot is re-persisted, and a `force` persist (transitions, run-now) always recomputes. Keep expensive aggregates out of the 1s tick: put them in `collect_metrics` and, if still heavy, raise the subclass `METRICS_SECONDS`.
 
 `main.py` registers every service in **all** workers (so `describe_all()` works anywhere) but only the lock worker calls `service_manager.supervise()`.
 
@@ -118,7 +118,7 @@ devplace devii reset-quota --all        # Reset every quota (users and guests)
 
 `uvicorn --workers N` = N independent processes sharing only the filesystem and SQLite DB. Module-global caches/counters are per-process, so a local `clear()` is invisible to siblings. Full reference: admin docs `Production -> Multi-worker and concurrency` (`templates/docs/production-concurrency.html`). Enforce these:
 
-- **In-process caches are never authoritative across workers.** Coordinate invalidation through the `cache_state` version table in `database.py`: call `bump_cache_version(name)` at every write path and `sync_local_cache(name, cache)` before every read path. Existing names: `auth` (guards `_user_cache`; bumped by `clear_user_cache`/`clear_session_cache` on logout, ban, role/password change), `settings` (guards `_settings_cache`; bumped by `set_setting`/`clear_settings_cache`), `relations` (`_relations_cache`), `customizations` (`_customizations_cache`), `notif_prefs` (`_notification_prefs_cache`), `gateway_routing` (`_ROUTING_CACHE`), and **`admins`** (guards `_admins_cache`, which memoizes `get_admin_uids()` / `get_primary_admin_uid()`). The `_cache_version_cache` (`ttl=1`) caches the version reads themselves, so the bump is seen by other workers within ~1s on their next read (the originating worker is immediate); the version read/bump fail open (logged, never raise). When you add a per-process cache whose staleness matters, give it a name and wire both calls - do not invent a second invalidation mechanism.
+- **In-process caches are never authoritative across workers.** Coordinate invalidation through the `cache_state` version table in `database.py`: call `bump_cache_version(name)` at every write path and `sync_local_cache(name, cache)` before every read path. Existing names: `auth` (guards `_user_cache`; bumped by `clear_user_cache`/`clear_session_cache` on logout, ban, role/password change - a version bump makes EVERY worker clear its WHOLE `_user_cache`, so display-only refreshes (XP/level in `award_xp`, AI-corrected bio) call `clear_user_cache(uid, propagate=False)` for a local pop without the global bump; identity/authz changes (logout, ban, role, password, api-key/token revoke) MUST keep the default propagate=True), `settings` (guards `_settings_cache`; bumped by `set_setting`/`clear_settings_cache`), `relations` (`_relations_cache`), `customizations` (`_customizations_cache`), `notif_prefs` (`_notification_prefs_cache`), `gateway_routing` (`_ROUTING_CACHE`), and **`admins`** (guards `_admins_cache`, which memoizes `get_admin_uids()` / `get_primary_admin_uid()`). The `_cache_version_cache` (`ttl=1`) caches the version reads themselves, so the bump is seen by other workers within ~1s on their next read (the originating worker is immediate); the version read/bump fail open (logged, never raise). When you add a per-process cache whose staleness matters, give it a name and wire both calls - do not invent a second invalidation mechanism.
 - **Admin-set cache invariant (load-bearing).** `_admins_cache` makes `get_admin_uids()` / `get_primary_admin_uid()` (hit on the projects-listing visibility filter, `_owner_is_admin`, and every primary-admin gate: `/dbapi`, backup-archive download) a dict lookup instead of a per-call `SELECT`. It is NOT the authorization gate - `is_admin(user)`/`require_admin` read the role off the user object (independently invalidated via `clear_user_cache`), so a stale admin set cannot grant access. But **any code that writes `users.role` MUST call `database.invalidate_admins_cache()`** (clears local + bumps the `admins` version), exactly like the soft-delete and `with db:` rules. Current role-write sites all do: `routers/admin/users.py` (role change), `cli.py` (`role set`), and `utils._create_account` (first user -> Admin). Omitting the call leaves the admin set stale for up to the 300s TTL.
 - **Deterministic / eventual caches skip version-sync on purpose.** Two caches need no `cache_state` name because correctness does not depend on cross-worker freshness: (1) `docs_prose._render_markdown` is an `@lru_cache` on the **static** prose source string (pure function of template content, so identical on every worker; changes only on deploy), and (2) `main._home_cache` (`TTLCache ttl=60`) memoizes the guest `/` blocks - the featured-news block (identical for everyone) and the latest-posts block **only for the no-block case**; a viewer with a non-empty block set bypasses the cache and is computed fresh, so the original block-filter semantics are preserved byte-for-byte and there is zero cross-user leakage. Worst case is a soft-deleted/edited public post lingering on `/` for <=60s. Use a plain `TTLCache`/`lru_cache` (no version name) ONLY when the value is deterministic or its staleness is purely cosmetic; anything whose staleness affects correctness or permissions MUST use the version-sync pattern above.
 - **Authoritative state lives in the DB**, memory is only a short-TTL accelerator (sessions, the Devii 24h cap via `devii_usage_ledger`, cost counters).
@@ -132,7 +132,7 @@ Read-mostly aggregates that were recomputed per request or per vote sit behind s
 
 | Cache | Where | Key | TTL | Invalidation |
 |---|---|---|---|---|
-| `_authors_cache` | `database/ranking.py` | `ranked`/`rank_map` | 15s | TTL only - `update_target_stars` deliberately does NOT clear it per vote anymore (the old per-vote `clear()` forced a full UNION-JOIN ranking recompute on the next feed/leaderboard/landing request and never propagated cross-worker anyway) |
+| `_authors_cache` | `database/ranking.py` | `ranked`/`rank_map` | 60s | TTL only - `update_target_stars` deliberately does NOT clear it per vote anymore (the old per-vote `clear()` forced a full UNION-JOIN ranking recompute on the next feed/leaderboard/landing request and never propagated cross-worker anyway) |
 | `_stars_cache` | `database/ranking.py` | user uid | 15s | TTL + `clear_user_stars(owner_uid)` from `content.apply_vote`, so a user's own total updates immediately on the voting worker |
 | `_leaderboard_cache` | `services/game/store/farm.py` | `top:{limit}` | 15s | TTL only (the farm scan + Python `farm_score` sort runs at most once per 15s per worker) |
 | `_projects_cache` | `templating.py` | user uid | 10s | TTL + `clear_user_projects_cache(uid)` from the `content.py` project create/delete choke points (in-function import - `templating` imports `content` at module level). `jinja_user_projects` also filters `deleted_at=None` now (the composer dropdown previously listed soft-deleted projects) |

devplacepy/services/access_tokens.py

@@ -90,6 +90,9 @@ def revoke_token(uid: str) -> bool:
         return False
     stamp = datetime.now(timezone.utc).isoformat()
     tokens.update({"id": row["id"], "deleted_at": stamp, "deleted_by": "manual"}, ["id"])
+    from devplacepy.utils.authcache import clear_user_cache
+
+    clear_user_cache(row["user_uid"])
     return True
 
 
@@ -101,6 +104,10 @@ def revoke_all(user_uid: str) -> int:
     for row in list(tokens.find(user_uid=user_uid, deleted_at=None)):
         tokens.update({"id": row["id"], "deleted_at": stamp, "deleted_by": "manual"}, ["id"])
         count += 1
+    if count:
+        from devplacepy.utils.authcache import clear_user_cache
+
+        clear_user_cache(user_uid)
     return count
 
 

devplacepy/services/ai_modifier.py

@@ -61,19 +61,20 @@ def schedule_modification(
     prompt = (user.get("ai_modifier_prompt") or DEFAULT_MODIFIER_PROMPT).strip()
     user_uid = user.get("uid") or ""
     if user.get("ai_modifier_sync") and schedule_pending(
-        _run_modification, request, api_key, prompt, table, uid, user_uid
+        _run_modification, request, api_key, prompt, table, uid, user_uid, row
     ):
         return
     background.submit(_run_modification, api_key, prompt, table, uid, user_uid)
 
 
 def _run_modification(
-    api_key: str, prompt: str, table: str, uid: str, user_uid: str
+    api_key: str, prompt: str, table: str, uid: str, user_uid: str, row: dict | None = None
 ) -> None:
     fields = CORRECTABLE_FIELDS.get(table)
     if not fields:
         return
-    row = get_table(table).find_one(uid=uid)
+    if row is None:
+        row = get_table(table).find_one(uid=uid)
     if not row:
         return
     updates: dict = {}
@@ -97,6 +98,6 @@ def _run_modification(
         if table == "users" and user_uid:
             from devplacepy.utils import clear_user_cache
 
-            clear_user_cache(user_uid)
+            clear_user_cache(user_uid, propagate=False)
     if totals["calls"] and user_uid:
         add_modifier_usage(user_uid, totals)

devplacepy/services/audit/service.py

@@ -18,6 +18,7 @@ class AuditService(BaseService):
     description = "Prunes audit_log rows and their links older than the retention window."
     default_enabled = True
     min_interval = 3600
+    METRICS_SECONDS = 300
     config_fields = [
         ConfigField(
             RETENTION_KEY,

devplacepy/services/audit/store.py

@@ -101,16 +101,15 @@ def insert_event(row: dict) -> str:
         record["created_at"] = now()
     if record.get("via_agent") is None:
         record["via_agent"] = 0
-    get_table(AUDIT_TABLE).insert(record)
+    get_table(AUDIT_TABLE).insert(record, ensure=False)
     return record["uid"]
 
 
 def insert_links(audit_uid: str, links: list[dict]) -> int:
     if not links:
         return 0
-    table = get_table(LINKS_TABLE)
     stamp = now()
-    count = 0
+    records = []
     for link in links:
         record = _empty_link()
         record.update(
@@ -121,9 +120,10 @@ def insert_links(audit_uid: str, links: list[dict]) -> int:
         record["created_at"] = stamp
         if not record.get("object_uid"):
             continue
-        table.insert(record)
+        records.append(record)
-        count += 1
+    if records:
-    return count
+        get_table(LINKS_TABLE).insert_many(records, ensure=False)
+    return len(records)
 
 
 def get_event(uid: str) -> Optional[dict]:

devplacepy/services/base.py

@@ -123,7 +123,8 @@ class BaseService(ABC):
     details = ""
     DEFAULT_LOG_SIZE = 20
     TICK_SECONDS = 1
-    PERSIST_SECONDS = 3
+    PERSIST_SECONDS = 8
+    METRICS_SECONDS = 15
     STALE_SECONDS = 15
 
     def __init__(self, name: str, interval_seconds: int = 3600):
@@ -144,6 +145,9 @@ class BaseService(ABC):
         self._run_pending = False
         self._last_command = None
         self._last_persist = None
+        self._last_metrics = None
+        self._metrics_snapshot = {}
+        self._state_row_id = None
         self.enabled_field = ConfigField(
             self.enabled_key,
             "Enabled",
@@ -234,6 +238,14 @@ class BaseService(ABC):
             logger.warning(f"Could not collect metrics for {self.name}: {e}")
             return {}
 
+    def _current_metrics(self, now, force: bool = False) -> dict:
+        if not force and self._last_metrics is not None:
+            if (now - self._last_metrics).total_seconds() < self.METRICS_SECONDS:
+                return self._metrics_snapshot
+        self._last_metrics = now
+        self._metrics_snapshot = self._safe_metrics()
+        return self._metrics_snapshot
+
     def start_supervisor(self) -> None:
         if self._task is not None:
             return
@@ -344,16 +356,18 @@ class BaseService(ABC):
             "started_at": self._started_at.isoformat() if self._started_at else "",
             "heartbeat": now.isoformat(),
             "logs": json.dumps(list(self.log_buffer)),
-            "metrics": json.dumps(self._safe_metrics()),
+            "metrics": json.dumps(self._current_metrics(now, force=force)),
             "updated_at": now.isoformat(),
         }
         try:
             table = get_table("service_state")
-            existing = table.find_one(name=self.name)
+            if self._state_row_id is None:
-            if existing:
+                existing = table.find_one(name=self.name)
-                table.update({**record, "id": existing["id"]}, ["id"])
+                self._state_row_id = existing["id"] if existing else None
+            if self._state_row_id is not None:
+                table.update({**record, "id": self._state_row_id}, ["id"])
             else:
-                table.insert(record)
+                self._state_row_id = table.insert(record)
         except Exception as e:
             logger.warning(f"Could not persist state for {self.name}: {e}")
 

devplacepy/services/bot/engage.py

@@ -387,7 +387,7 @@ class BotEngageMixin:
             idx = random.randrange(limit)
             target = comments.nth(idx)
             try:
-                text = (await target.inner_text() or "")[:1000]
+                text = (await target.inner_text() or "")[:125000]
             except Exception:
                 continue
             if not text:

devplacepy/services/bot/social.py

@@ -433,7 +433,7 @@ class BotSocialMixin:
         mentioner = ""
         if comment_loc is not None:
             try:
-                parent_text = (await comment_loc.inner_text() or "")[:1000]
+                parent_text = (await comment_loc.inner_text() or "")[:125000]
             except Exception:
                 parent_text = ""
             try:

devplacepy/services/containers/files/bot.py

@@ -72,7 +72,7 @@ MENTION_POLL_SECONDS = int(os.environ.get("MENTION_POLL_SECONDS", "30"))
 DM_POLL_SECONDS = int(os.environ.get("DM_POLL_SECONDS", "10"))
 BOT_USERNAME = os.environ.get("BOT_USERNAME", "")
 
-COMMENT_CHAR_LIMIT = 1000
+COMMENT_CHAR_LIMIT = 125000
 MESSAGE_CHAR_LIMIT = 2000
 PART_SUFFIX_RESERVE = 12
 PART_DELIVERY_DELAY = 0.5

devplacepy/services/containers/files/bot.py.bak

@@ -72,7 +72,7 @@ MENTION_POLL_SECONDS = int(os.environ.get("MENTION_POLL_SECONDS", "30"))
 DM_POLL_SECONDS = int(os.environ.get("DM_POLL_SECONDS", "10"))
 BOT_USERNAME = os.environ.get("BOT_USERNAME", "")
 
-COMMENT_CHAR_LIMIT = 1000
+COMMENT_CHAR_LIMIT = 125000
 MESSAGE_CHAR_LIMIT = 2000
 PART_SUFFIX_RESERVE = 12
 PART_DELIVERY_DELAY = 0.5

devplacepy/services/correction.py

@@ -2,6 +2,8 @@
 
 import asyncio
 import logging
+import threading
+from concurrent.futures import ThreadPoolExecutor
 
 import httpx
 
@@ -30,6 +32,22 @@ CORRECTION_TIMEOUT_SECONDS = 20.0
 MAX_GROWTH_FACTOR = 3
 PENDING_SCOPE_KEY = "devplace_pending_corrections"
 
+AI_APPLY_EXECUTOR = ThreadPoolExecutor(max_workers=4, thread_name_prefix="ai-apply")
+
+_gateway_client: httpx.Client | None = None
+_gateway_client_lock = threading.Lock()
+
+
+def _client() -> httpx.Client:
+    global _gateway_client
+    if _gateway_client is None:
+        with _gateway_client_lock:
+            if _gateway_client is None:
+                _gateway_client = stealth.stealth_sync_client(
+                    timeout=CORRECTION_TIMEOUT_SECONDS
+                )
+    return _gateway_client
+
 
 def _usage_from_headers(response_headers) -> dict:
     parsed = parse_usage_headers(response_headers)
@@ -72,21 +90,25 @@ def gateway_complete(
         ],
         "temperature": 0.1,
     }
-    headers = {"Content-Type": "application/json"}
+    headers = {
+        "Content-Type": "application/json",
+        "X-App-Reference": "devplace-correction-v-1-0-0",
+    }
     if api_key:
         headers["Authorization"] = f"Bearer {api_key}"
     try:
-        with stealth.stealth_sync_client(timeout=timeout) as client:
+        response = _client().post(
-            response = client.post(INTERNAL_GATEWAY_URL, json=payload, headers=headers)
+            INTERNAL_GATEWAY_URL, json=payload, headers=headers, timeout=timeout
-            response.raise_for_status()
+        )
-            usage = _usage_from_headers(response.headers)
+        response.raise_for_status()
-            content = (
+        usage = _usage_from_headers(response.headers)
-                response.json()
+        content = (
-                .get("choices", [{}])[0]
+            response.json()
-                .get("message", {})
+            .get("choices", [{}])[0]
-                .get("content", "")
+            .get("message", {})
-                .strip()
+            .get("content", "")
-            )
+            .strip()
+        )
     except (httpx.HTTPError, ValueError, KeyError, IndexError) as exc:
         logger.warning("AI gateway completion failed, keeping original: %s", exc)
         return text, None
@@ -138,7 +160,7 @@ def schedule_pending(fn, request: object, *args) -> bool:
         loop = asyncio.get_running_loop()
     except RuntimeError:
         return False
-    future = loop.run_in_executor(None, fn, *args)
+    future = loop.run_in_executor(AI_APPLY_EXECUTOR, fn, *args)
     scope.setdefault(PENDING_SCOPE_KEY, []).append(future)
     return True
 
@@ -182,6 +204,6 @@ def _run_correction(
         if table == "users" and user_uid:
             from devplacepy.utils import clear_user_cache
 
-            clear_user_cache(user_uid)
+            clear_user_cache(user_uid, propagate=False)
     if totals["calls"] and user_uid:
         add_correction_usage(user_uid, totals)

devplacepy/services/dbapi/nl2sql.py

@@ -92,7 +92,11 @@ async def _complete(messages: list[dict], api_key: str, model: str) -> str:
         "max_tokens": MAX_TOKENS,
         "temperature": 0.0,
     }
-    headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+        "X-App-Reference": "devplace-dbapi-v-1-0-0",
+    }
     async with stealth.stealth_async_client(timeout=GATEWAY_TIMEOUT_SECONDS) as client:
         response = await client.post(INTERNAL_GATEWAY_URL, json=payload, headers=headers)
     if response.status_code >= 400:

devplacepy/services/deepsearch/chat.py

@@ -82,7 +82,7 @@ class DeepsearchChat:
                     stored_dim,
                 )
                 return []
-        return self.store.hybrid_search(question, query_vector, top_k=CHAT_TOP_K)
+        return await self.store.hybrid_search(question, query_vector, top_k=CHAT_TOP_K)
 
     async def answer(self, question: str, history: list[dict] | None = None) -> ChatAnswer:
         chunks = await self.retrieve(question)

devplacepy/services/deepsearch/embeddings.py

@@ -9,8 +9,8 @@ import re
 import time
 from dataclasses import dataclass, field
 
-from devplacepy import stealth
 from devplacepy.config import INTERNAL_EMBED_MODEL, INTERNAL_EMBED_URL
+from devplacepy.services.deepsearch.llm import gateway_client
 
 logger = logging.getLogger(__name__)
 
@@ -112,13 +112,15 @@ async def embed_texts(
     headers = {
         "Authorization": f"Bearer {api_key}",
         "Content-Type": "application/json",
+        "X-App-Reference": "devplace-deepsearch-v-1-0-0",
     }
     payload = {"model": INTERNAL_EMBED_MODEL, "input": pending_text}
     start = time.monotonic()
     backend = "gateway"
     try:
-        async with stealth.stealth_async_client(timeout=EMBED_TIMEOUT_SECONDS) as client:
+        response = await gateway_client().post(
-            response = await client.post(gateway_url, json=payload, headers=headers)
+            gateway_url, json=payload, headers=headers, timeout=EMBED_TIMEOUT_SECONDS
+        )
         if response.status_code >= 400:
             raise RuntimeError(f"embed gateway returned {response.status_code}")
         data = response.json()

devplacepy/services/deepsearch/llm.py

@@ -5,6 +5,8 @@ from __future__ import annotations
 import logging
 import time
 
+import httpx
+
 from devplacepy import stealth
 from devplacepy.config import INTERNAL_GATEWAY_URL, INTERNAL_MODEL
 
@@ -13,6 +15,15 @@ logger = logging.getLogger(__name__)
 CHAT_TIMEOUT_SECONDS = 120.0
 DEFAULT_MAX_TOKENS = 1200
 
+_gateway_client: httpx.AsyncClient | None = None
+
+
+def gateway_client() -> httpx.AsyncClient:
+    global _gateway_client
+    if _gateway_client is None or _gateway_client.is_closed:
+        _gateway_client = stealth.stealth_async_client(timeout=CHAT_TIMEOUT_SECONDS)
+    return _gateway_client
+
 
 async def request_completion(
     messages: list[dict],
@@ -33,10 +44,12 @@ async def request_completion(
     headers = {
         "Authorization": f"Bearer {api_key}",
         "Content-Type": "application/json",
+        "X-App-Reference": "devplace-deepsearch-v-1-0-0",
     }
     start: float = time.monotonic()
-    async with stealth.stealth_async_client(timeout=timeout) as client:
+    response = await gateway_client().post(
-        response = await client.post(gateway_url, json=payload, headers=headers)
+        gateway_url, json=payload, headers=headers, timeout=timeout
+    )
     elapsed_ms: int = int((time.monotonic() - start) * 1000)
     if response.status_code >= 400:
         raise RuntimeError(f"chat gateway returned {response.status_code}")

devplacepy/services/deepsearch/store.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import asyncio
 import logging
 import math
 import re
@@ -19,6 +20,7 @@ BM25_B = 0.75
 RRF_K = 60.0
 DEFAULT_TOP_K = 8
 CANDIDATE_MULTIPLIER = 4
+CHROMADB_TIMEOUT = 30.0
 
 
 @dataclass
@@ -45,6 +47,20 @@ class VectorStore:
         self._collection = None
         self._dims: int | None = None
 
+    async def _run_sync(self, func, *args, timeout: float = CHROMADB_TIMEOUT):
+        """Run a synchronous ChromaDB call in a thread executor with a timeout."""
+        try:
+            return await asyncio.wait_for(
+                asyncio.to_thread(func, *args), timeout=timeout
+            )
+        except asyncio.TimeoutError:
+            logger.error(
+                "deepsearch ChromaDB operation timed out after %.1fs on collection %s",
+                timeout,
+                self.collection_name,
+            )
+            raise
+
     def _ensure(self):
         if self._collection is not None:
             return self._collection
@@ -71,7 +87,7 @@ class VectorStore:
             return self._dims
         return self._dims
 
-    def add(self, chunks: list[Chunk], vectors: list[list[float]]) -> None:
+    def _add_sync(self, chunks: list[Chunk], vectors: list[list[float]]) -> None:
         if not chunks:
             return
         keep_chunks: list[Chunk] = []
@@ -111,9 +127,17 @@ class VectorStore:
             ],
         )
 
-    def all_chunks(self) -> list[Chunk]:
+    async def add(self, chunks: list[Chunk], vectors: list[list[float]]) -> None:
+        if not chunks:
+            return
+        await self._run_sync(self._add_sync, chunks, vectors)
+
+    def _all_chunks_sync(self, limit: int = 0) -> list[Chunk]:
         collection = self._ensure()
-        data = collection.get(include=["documents", "metadatas"])
+        kwargs: dict = {"include": ["documents", "metadatas"]}
+        if limit > 0:
+            kwargs["limit"] = limit
+        data = collection.get(**kwargs)
         chunks: list[Chunk] = []
         ids = data.get("ids") or []
         documents = data.get("documents") or []
@@ -134,13 +158,17 @@ class VectorStore:
             )
         return chunks
 
-    def count(self) -> int:
+    async def all_chunks(self, limit: int = 1000) -> list[Chunk]:
+        return await self._run_sync(self._all_chunks_sync, limit)
+
+    async def count(self) -> int:
         try:
-            return self._ensure().count()
+            collection = self._ensure()
+            return await self._run_sync(collection.count)
         except Exception:
             return 0
 
-    def vector_search(
+    def _vector_search_sync(
         self, query_vector: list[float], top_k: int, where: dict | None = None
     ) -> list[Chunk]:
         collection = self._ensure()
@@ -173,6 +201,13 @@ class VectorStore:
             )
         return chunks
 
+    async def vector_search(
+        self, query_vector: list[float], top_k: int, where: dict | None = None
+    ) -> list[Chunk]:
+        return await self._run_sync(
+            self._vector_search_sync, query_vector, top_k, where
+        )
+
     def keyword_scores(self, query: str, chunks: list[Chunk]) -> dict[str, float]:
         terms = _tokenize(query)
         if not terms or not chunks:
@@ -205,14 +240,14 @@ class VectorStore:
             scores[chunk.uid] = score
         return scores
 
-    def hybrid_search(
+    async def hybrid_search(
         self,
         query: str,
         query_vector: list[float],
         top_k: int = DEFAULT_TOP_K,
         where: dict | None = None,
     ) -> list[Chunk]:
-        candidates = self.vector_search(
+        candidates = await self.vector_search(
             query_vector, top_k * CANDIDATE_MULTIPLIER, where
         )
         if not candidates:
@@ -234,10 +269,11 @@ class VectorStore:
         candidates.sort(key=lambda chunk: chunk.score, reverse=True)
         return candidates[:top_k]
 
-    def coverage_analytics(self) -> dict:
+    async def coverage_analytics(self, sample_limit: int = 5000) -> dict:
-        chunks = self.all_chunks()
+        chunks = await self.all_chunks(limit=sample_limit)
         if not chunks:
-            return {"chunks": 0, "domains": 0, "sources": 0, "avg_chunk_chars": 0}
+            total = await self.count()
+            return {"chunks": total, "domains": 0, "sources": 0, "avg_chunk_chars": 0}
         domains = {
             urlparse(chunk.metadata.get("url", "")).netloc for chunk in chunks
         }

devplacepy/services/devii/CLAUDE.md

@@ -35,6 +35,12 @@ Conversation history persists to `devii_conversations` (rehydrated on reconnect,
 
 **Per-owner self-learning memory is privacy-critical.** The `LessonStore` (reflect/recall) is **owner-scoped, never shared**: `LessonStore(db, owner_kind, owner_id)` filters every read/write by owner. The hub builds one per session over the same `owned_db` as the task store - the main `db` (table `devii_lessons`) for signed-in users (persistent, isolated, survives restarts) and a fresh `memory_db()` for guests (ephemeral, scoped to that web session). `forget_lessons` (agentic tool -> `LessonStore.clear()`/`delete()`) lets the user purge them; the system prompt forbids storing credentials/secrets in lessons. **Regression to avoid: do NOT share one `LessonStore` across sessions** - that leaked one user's reflected lessons (including credentials) into every other user's recall.
 
+**Lesson retention and deduplication.** Every `add()` deduplicates against existing active lessons via Jaccard similarity (threshold 0.70): a near-duplicate bumps the original's `hits` counter and refreshes `created_at` instead of inserting a new row. A per-owner cap (`devii_lessons_max_per_owner`, default 500, configurable on `/admin/services` and `site_settings`) is enforced on every insert: when exceeded, the oldest lessons are soft-deleted (`deleted_by="retention"`). Age-based pruning runs on `DeviiService.run_once()` (every 60s on the lock-owner worker): lessons older than `devii_lessons_max_age_days` (default 90, configurable) are soft-deleted across all owners. Both settings persist in `site_settings` and are seeded in `init_db`. The `devii_lessons` table is in `SOFT_DELETE_TABLES` for admin Trash restore/purge.
+
+**Quality signals.** Each lesson has a `rating` column (integer, default 0). The `lesson_rate(uid, value)` agentic tool (value 1 for useful, -1 for unhelpful) lets the agent self-rate lessons. In `_rebuild()`, lessons with `rating <= -3` are excluded from the BM25 index and therefore never returned by `recall()`. The `lesson_count()` tool reports active lesson count.
+
+**CLI:** `devplace devii lessons count` reports active/soft-deleted totals; `devplace devii lessons prune --all-owners|--username USER` soft-deletes old lessons; `devplace devii lessons clear --force` hard-deletes all rows. Rate-limiting guard: `run_once` swallows all exceptions so a bad schema never stops housekeeping.
+
 ## Reminders and scheduled tasks (persistent across reboot)
 
 `create_task` queues a self-contained prompt that a fresh agent runs later (`services/devii/tasks/`): `kind=once` (`delay_seconds` for relative, `run_at` UTC for absolute), `interval` (`every_seconds`), or `cron`. The per-session `Scheduler` ticks every 1s and executes due rows through the session's executor, so the result is broadcast to any connected tab and buffered (`type:"task"` frame) when the terminal is closed.
@@ -142,9 +148,57 @@ Devii can partially configure its OWN system message: every system prompt ends w
 
 - **Store** (`behavior/store.py`). `BehaviorStore(db, owner_kind, owner_id)` over `devii_behavior` (one upserted row per owner keyed on `owner_kind`/`owner_id`; `text()` reads, `set()` upserts; index `idx_devii_behavior_owner`). Persistent for users, `memory_db()` for guests (built in `hub.get_or_create` from the shared `owned_db`, like the other owner stores).
 - **Controller** (`behavior/controller.py`). `BehaviorController(store)`, `dispatch("update_behavior", args)` -> `store.set(behavior)`. Built in `DeviiSession` and passed to `Dispatcher(behavior=...)`; the dispatcher routes `handler="behavior"` to it and degrades gracefully ("not available in this context") when unwired (e.g. the standalone CLI, same as `virtual_tools`/`avatar`).
-- **Injection and refresh** (`session.py`). `_compose_system_prompt()` = base prompt (`_system_prompt_for(is_admin)`) + `\n\n` + `BEHAVIOR_HEADER` (+ `\n` + body when non-empty; just the header when empty). Used to seed the `Agent` and the scheduler executor worker, and `_refresh_system_prompt()` rewrites `agent._messages[0]["content"]` **at the top of every `_run_turn`** (next to `_refresh_tools`), so a mid-conversation `update_behavior` takes effect on the following turn and the live system message is never overridden by the stale base.
+- **Injection and refresh** (`session.py`). `_compose_system_prompt()` = base prompt (`_system_prompt_for(is_admin)`) + CA-IWP fragment + live `CHANNEL` block + `\n\n` + `BEHAVIOR_HEADER` (+ `\n` + body when non-empty; just the header when empty). Used to seed the `Agent` and the scheduler executor worker, and `_refresh_system_prompt()` rewrites `agent._messages[0]["content"]` **at the top of every `_run_turn`** (next to `_refresh_tools`), so a mid-conversation `update_behavior` takes effect on the following turn and the live system message is never overridden by the stale base.
 - Registered via `BEHAVIOR_ACTIONS` (`registry.py`); the system-prompt **SELF-CONFIGURED BEHAVIOR (TRUTH RULES)** section in `agent.py` steers when/how to call it.
 
+## Channel-Aware Interactive Widget Protocol (CA-IWP / Speak With Buttons)
+
+Devii can ask the user for decisions through a gated tool surface instead of inventing HTML or channel-specific prose. Spec lives as the CA-IWP document; implementation is Devii-only under `services/devii/interaction/`.
+
+### Layers
+
+| Piece | Role |
+|-------|------|
+| `interaction/capabilities.py` | Maps session channel (`main`/`docs` -> `site-chat`, `telegram`, `cli`) to capability flags + limits; builds the compact `CHANNEL` / `CAPS` / `LIMITS` / `TOOLS` / `INTERACTION` fragment injected every turn. |
+| `interaction/schema.py` | Pydantic validation for `ui_prompt` args (widget catalog: confirm, choice, choice_multi, text, number, date, select, `//`, group). Untrusted model input is sanitized and capped. |
+| `interaction/broker.py` | Validates, assigns `interaction_id`, single-flight open interactions, routes to site / telegram / plain adapters, returns structured `{status, values, meta}`. |
+| `interaction/controller.py` | Dispatches `ui_prompt` / `ui_cancel` / `ui_notify` (`handler="interaction"`). |
+| `interaction/actions.py` | Catalog entries registered in `registry.py` as `INTERACTION_ACTIONS`. |
+| `interaction/markdown.py` + `parse.py` | Dual-surface markdown projection and plain/CLI reply parsers. |
+
+### Tool gating and preferences (admin default + user override)
+
+`DeviiSession._builtin_tools()` filters UI tools through `channel_context().tools`. Docs channel stays search-only (no UI tools). Open interaction (single-flight) exposes only `ui_cancel` plus preference tools.
+
+**Admin default:** Devii service ConfigField `devii_interactions_default` (bool, default on) on `/admin/services`. Guests always use this default.
+
+**User override:** column `users.interactions_enabled` (`-1` = inherit default, `0` = off, `1` = on). New signups insert `-1`. Resolution: `interaction/prefs.py` `effective_for(owner_kind, owner_id)`.
+
+**Surfaces (same fan-out as AI correction):**
+- Devii tools `interactions_get` / `interactions_set` (`requires_auth=True`; set accepts `enabled` or `reset=true` to inherit again). Always offered to signed-in users even when widgets are off, so they can re-enable.
+- HTTP `POST /profile/{username}/interactions` (owner-or-admin, form `InteractionsForm`), profile card + `InteractionsPref.js`, `docs_api` endpoint, audit `profile.interactions`, profile JSON keys on `ProfileOut`.
+- When effective is off, `ui_prompt` / `ui_notify` are absent from the tool list and the controller refuses them; the model must use degraded markdown menus.
+
+### Site-chat path
+
+`ui_prompt` blocks like client tools: session `_interaction_wait` emits `{type:"interaction", id, args}` on the WebSocket; `devii-terminal.js` mounts an `<ai-interaction>` tree via `createElement` (never model HTML), lazy-loads widget modules through `AiAutoload` (`static/js/autoload/AiAutoload.js`, allowlisted tag -> module map only), and replies with `{type:"interaction_result"}`. Router resolves via `session.resolve_interaction`. Custom elements live under `static/js/components/Ai*.js` with per-component CSS under `static/css/components/ai-*.css`. Light DOM only; `Application` boots the shell (`AiInteraction`, `AiStatus`, `AiActions`, `AiHelp`, `AiOption`) and starts the autoloader.
+
+### Telegram path
+
+`TelegramConnection` handles `type:"interaction"`: sends dual-surface plain text plus inline keyboard for confirm / single choice (`callback_data` short tokens). `TelegramBridge` registers a pending future **before** the chat lock so the next message or `callback_query` can resolve mid-turn. Worker `allowed_updates` includes `callback_query`; service routes `type:"callback"` to the bridge.
+
+### Plain / CLI path
+
+Broker uses `present_plain` or emits a plain menu and waits; `parse_plain_reply` accepts y/n, indices, value tokens, cancel.
+
+### System prompt
+
+Every non-docs turn appends `CA_IWP_SYSTEM_FRAGMENT` plus the live channel fragment from `_compose_system_prompt()`. Model rules: prefer `ui_prompt` when gated on; never invent HTML/CE tags; branch on `status` (submitted / cancelled / timeout / superseded / error).
+
+### Tests
+
+Unit coverage under `tests/unit/services/devii/interaction/` (capabilities, schema, markdown, parse, broker, controller) plus session tool-gating assertions and telegram callback emission. Mirror this layout for any extension.
+
 ## Related Devii tool wrappers (customization, container)
 
 Two more Devii-side controllers live under `services/devii/` but their full mechanism is documented in their owning subsystem's file, not here:

devplacepy/services/devii/actions/ai_correction_actions.py

@@ -61,7 +61,7 @@ AI_CORRECTION_ACTIONS: tuple[Action, ...] = (
             ),
             arg(
                 "prompt",
-                "The correction instruction (max 2000 chars). Omit to keep the current one.",
+                "The correction instruction (max 20000 chars). Omit to keep the current one.",
             ),
         ),
     ),

devplacepy/services/devii/actions/ai_modifier_actions.py

@@ -64,7 +64,7 @@ AI_MODIFIER_ACTIONS: tuple[Action, ...] = (
             ),
             arg(
                 "prompt",
-                "The modifier instruction (max 2000 chars). Omit to keep the current one.",
+                "The modifier instruction (max 20000 chars). Omit to keep the current one.",
             ),
         ),
     ),

devplacepy/services/devii/actions/catalog/comments.py

@@ -28,7 +28,7 @@ COMMENTS_ACTIONS: tuple[Action, ...] = (
         summary="Edit the body of one of your own comments",
         params=(
             path("comment_uid", "Uid of the comment."),
-            body("content", "New comment body, 3-1000 characters.", required=True),
+            body("content", "New comment body, 3-125000 characters.", required=True),
         ),
     ),
     Action(

devplacepy/services/devii/actions/dispatcher.py

@@ -116,6 +116,7 @@ _DEVII_MECHANIC_EVENTS = {
     "email_mark": "email.message.flag",
     "email_set_flags": "email.message.flag",
     "telegram_send": "telegram.send",
+    "interactions_set": "profile.interactions",
 }
 
 _DEVII_CONTAINER_EVENTS = {
@@ -268,6 +269,7 @@ class Dispatcher:
         owner_id: str = "",
         virtual_tools: Any = None,
         behavior: Any = None,
+        interaction: Any = None,
     ) -> None:
         self._actions = catalog.by_name()
         self._client = client
@@ -308,6 +310,7 @@ class Dispatcher:
         self._telegram = TelegramSendController(owner_kind, owner_id)
         self._virtual_tools = virtual_tools
         self._behavior = behavior
+        self._interaction = interaction
         self._read_files: set[tuple[str, str]] = set()
 
     @staticmethod
@@ -500,6 +503,15 @@ class Dispatcher:
                 )
             return await self._behavior.dispatch(action.name, arguments)
 
+        if action.handler == "interaction":
+            if self._interaction is None:
+                return error_result(
+                    ToolInputError(
+                        "Interactive prompts are not available in this context."
+                    )
+                )
+            return await self._interaction.dispatch(action.name, arguments)
+
         if action.handler == "virtual_tool":
             if self._virtual_tools is None:
                 return error_result(

devplacepy/services/devii/actions/spec.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
+from functools import lru_cache
 from typing import Any, Literal
 
 ParamLocation = Literal["path", "query", "body", "file"]
@@ -49,6 +50,7 @@ class Action:
         "virtual_tool",
         "email",
         "telegram",
+        "interaction",
     ] = "http"
     freeform_body: bool = False
     ajax: bool = False
@@ -117,16 +119,27 @@ class Catalog:
     def tool_schemas(self) -> list[dict[str, Any]]:
         return [action.tool_schema() for action in self.actions]
 
+    @lru_cache(maxsize=16)
+    def _schemas_for(
+        self,
+        authenticated: bool,
+        is_admin: bool,
+        is_primary_admin: bool,
+    ) -> tuple[dict[str, Any], ...]:
+        return tuple(
+            action.tool_schema()
+            for action in self.actions
+            if (authenticated or not action.requires_auth)
+            and (is_admin or not action.requires_admin)
+            and (is_primary_admin or not action.requires_primary_admin)
+        )
+
     def tool_schemas_for(
         self,
         authenticated: bool,
         is_admin: bool = False,
         is_primary_admin: bool = False,
     ) -> list[dict[str, Any]]:
-        return [
+        return list(
-            action.tool_schema()
+            self._schemas_for(bool(authenticated), bool(is_admin), bool(is_primary_admin))
-            for action in self.actions
+        )
-            if (authenticated or not action.requires_auth)
-            and (is_admin or not action.requires_admin)
-            and (is_primary_admin or not action.requires_primary_admin)
-        ]

devplacepy/services/devii/agentic/actions.py

@@ -150,6 +150,34 @@ AGENTIC_ACTIONS: tuple[Action, ...] = (
             ),
         ),
     ),
+    Action(
+        name="lesson_rate",
+        method="LOCAL",
+        path="",
+        summary="Vote on a lesson's quality to help the system keep the best ones and drop bad ones",
+        description=(
+            "Rate a lesson recorded by reflect(). A positive vote (1) marks it as useful; "
+            "a negative vote (-1) marks it as unhelpful. Lessons with a cumulative rating of "
+            "-3 or lower are excluded from recall() results. Use this when you notice a lesson "
+            "was particularly helpful or misleading."
+        ),
+        handler="agentic",
+        requires_auth=False,
+        params=(
+            arg("uid", "The uid of the lesson to rate, from a recall() result.", required=True),
+            arg("value", "1 for useful, -1 for unhelpful.", required=True, kind="integer"),
+        ),
+    ),
+    Action(
+        name="lesson_count",
+        method="LOCAL",
+        path="",
+        summary="Report how many learned lessons are stored for this session or account",
+        description="Returns the number of active lessons with an optional per-tag breakdown.",
+        handler="agentic",
+        requires_auth=False,
+        params=(),
+    ),
     Action(
         name="eval",
         method="LOCAL",

devplacepy/services/devii/agentic/compaction.py

@@ -6,13 +6,17 @@ import json
 import logging
 from typing import Any
 
+from ..text import normalize_newlines
+
 logger = logging.getLogger("devii.agentic.compaction")
 
 SUMMARY_INPUT_CAP = 600_000
 SUMMARY_PROMPT = (
     "Summarize the following assistant conversation segment as a concise factual log of "
     "actions taken, tools called, entities created or changed, conclusions reached, and "
-    "outstanding work. Keep identifiers, slugs, uids, and decisions verbatim. Maximum 800 words.\n\n"
+    "outstanding work. Keep identifiers, slugs, uids, and decisions verbatim. "
+    "Write plain markdown with real line breaks (never the two-character sequence \\n). "
+    "Use headings and bullet lists where helpful. Maximum 800 words.\n\n"
     "---\n\n"
 )
 
@@ -32,6 +36,37 @@ def find_compaction_split(messages: list[dict[str, Any]], keep_tail: int) -> int
     return 1
 
 
+def _segment_plain(messages: list[dict[str, Any]]) -> str:
+    parts: list[str] = []
+    for message in messages:
+        role = str(message.get("role") or "unknown")
+        content = message.get("content")
+        if isinstance(content, str) and content.strip():
+            parts.append(f"{role}:\n{normalize_newlines(content)}")
+            continue
+        if isinstance(content, list):
+            chunks: list[str] = []
+            for part in content:
+                if isinstance(part, dict) and part.get("type") == "text":
+                    chunks.append(str(part.get("text") or ""))
+                elif isinstance(part, str):
+                    chunks.append(part)
+            text = normalize_newlines("\n".join(c for c in chunks if c))
+            if text.strip():
+                parts.append(f"{role}:\n{text}")
+            continue
+        tool_calls = message.get("tool_calls")
+        if tool_calls:
+            names = []
+            for call in tool_calls:
+                fn = (call or {}).get("function") or {}
+                name = fn.get("name") or "tool"
+                names.append(str(name))
+            if names:
+                parts.append(f"{role}: called {', '.join(names)}")
+    return "\n\n".join(parts)
+
+
 async def compact_messages(
     llm: Any, messages: list[dict[str, Any]], keep_tail: int
 ) -> list[dict[str, Any]]:
@@ -46,16 +81,24 @@ async def compact_messages(
     if not middle:
         return messages
 
-    segment = json.dumps(middle, default=str)[:SUMMARY_INPUT_CAP]
+    segment = _segment_plain(middle)[:SUMMARY_INPUT_CAP]
+    if not segment.strip():
+        segment = json.dumps(middle, default=str)[:SUMMARY_INPUT_CAP]
     try:
         summary = await llm.summarize(SUMMARY_PROMPT + segment)
     except Exception:  # noqa: BLE001 - compaction must never break the loop
         logger.exception("Compaction summary failed; keeping full context")
         return messages
 
+    summary = normalize_newlines(summary or "").strip()
+    if not summary:
+        return messages
     logger.info("Compacted %d messages into a summary", len(middle))
     return [
         system_message,
-        {"role": "assistant", "content": f"[compacted earlier turns]\n\n{summary}"},
+        {
+            "role": "assistant",
+            "content": f"[compacted earlier turns]\n\n{summary}",
+        },
         *tail,
     ]

devplacepy/services/devii/agentic/controller.py

@@ -63,6 +63,8 @@ class AgenticController:
             "reflect": self._reflect,
             "recall": self._recall,
             "forget_lessons": self._forget,
+            "lesson_rate": self._lesson_rate,
+            "lesson_count": self._lesson_count,
             "verify": self._verify,
             "delegate": self._delegate,
             "eval": self._eval,
@@ -162,6 +164,31 @@ class AgenticController:
             ensure_ascii=False,
         )
 
+    async def _lesson_rate(self, arguments: dict[str, Any]) -> str:
+        uid = str(arguments.get("uid", "")).strip()
+        if not uid:
+            raise ToolInputError("lesson_rate requires a lesson uid.")
+        raw_value = arguments.get("value")
+        if raw_value is None:
+            raise ToolInputError("lesson_rate requires a value (1 or -1).")
+        try:
+            value = int(raw_value)
+        except (ValueError, TypeError):
+            raise ToolInputError("lesson_rate value must be an integer (1 or -1).") from None
+        if value not in (1, -1):
+            raise ToolInputError("lesson_rate value must be 1 (useful) or -1 (unhelpful).")
+        ok = self._lessons.rate(uid, value)
+        if not ok:
+            raise ToolInputError(f"No lesson found with uid '{uid}'.")
+        return json.dumps({"status": "success", "lesson_uid": uid, "rated": value}, ensure_ascii=False)
+
+    async def _lesson_count(self, arguments: dict[str, Any]) -> str:
+        count = self._lessons.count()
+        return json.dumps(
+            {"status": "success", "active_lessons": count},
+            ensure_ascii=False,
+        )
+
     async def _verify(self, arguments: dict[str, Any]) -> str:
         summary = str(arguments.get("summary", "")).strip()
         if not summary:

devplacepy/services/devii/agentic/lessons.py

@@ -7,6 +7,7 @@ import logging
 import math
 import re
 import uuid
+from datetime import timedelta
 from typing import Any
 
 from ..tasks.schedule import now_utc, to_iso
@@ -18,6 +19,11 @@ TOKEN_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*|\d+")
 BM25_K1 = 1.5
 BM25_B = 0.75
 
+DEDUP_JACCARD_THRESHOLD = 0.70
+DEFAULT_MAX_PER_OWNER = 500
+DEFAULT_MAX_AGE_DAYS = 90
+LOW_QUALITY_THRESHOLD = -3
+
 
 def tokenize(text: str) -> list[str]:
     tokens = TOKEN_RE.findall((text or "").lower())
@@ -27,6 +33,33 @@ def tokenize(text: str) -> list[str]:
     return list(dict.fromkeys(tokens + extra))
 
 
+def _jaccard(tokens_a: set[str], tokens_b: set[str]) -> float:
+    if not tokens_a and not tokens_b:
+        return 0.0
+    if not tokens_a or not tokens_b:
+        return 0.0
+    return len(tokens_a & tokens_b) / len(tokens_a | tokens_b)
+
+
+def _read_retention_settings(db: Any) -> tuple[int, int]:
+    max_per = DEFAULT_MAX_PER_OWNER
+    max_age = DEFAULT_MAX_AGE_DAYS
+    if "site_settings" not in db.tables:
+        return max_per, max_age
+    for row in db["site_settings"].find(key={"in": ["devii_lessons_max_per_owner", "devii_lessons_max_age_days"]}):
+        if row["key"] == "devii_lessons_max_per_owner":
+            try:
+                max_per = int(row["value"])
+            except (ValueError, TypeError):
+                pass
+        elif row["key"] == "devii_lessons_max_age_days":
+            try:
+                max_age = int(row["value"])
+            except (ValueError, TypeError):
+                pass
+    return max_per, max_age
+
+
 class LessonStore:
     def __init__(self, db: Any, owner_kind: str, owner_id: str) -> None:
         self._db = db
@@ -39,9 +72,10 @@ class LessonStore:
         self._idf: dict[str, float] = {}
         self._avgdl = 0.0
         self._n = 0
+        self._ensure_columns()
         self._ensure_indexes()
 
-    def _ensure_indexes(self) -> None:
+    def _ensure_columns(self) -> None:
         if TABLE not in self._db.tables:
             return
         table = self._db[TABLE]
@@ -49,7 +83,15 @@ class LessonStore:
             table.create_column_by_example("deleted_at", "")
         if not table.has_column("deleted_by"):
             table.create_column_by_example("deleted_by", "")
+        if not table.has_column("rating"):
+            table.create_column_by_example("rating", 0)
+
+    def _ensure_indexes(self) -> None:
+        if TABLE not in self._db.tables:
+            return
+        table = self._db[TABLE]
         table.create_index(["owner_kind", "owner_id"])
+        table.create_index(["owner_kind", "owner_id", "created_at"])
 
     @property
     def _table(self) -> Any:
@@ -64,17 +106,99 @@ class LessonStore:
             return 0
         return self._table.count(deleted_at=None, **self._scope)
 
+    def _row_text(self, row: dict[str, Any]) -> str:
+        return " ".join(
+            str(row.get(field) or "")
+            for field in ("observation", "conclusion", "next_action", "tags")
+        )
+
+    def _find_similar(self, text: str, threshold: float = DEDUP_JACCARD_THRESHOLD) -> dict[str, Any] | None:
+        query_tokens = set(tokenize(text))
+        if not query_tokens:
+            return None
+        all_rows = self.all()
+        best_row: dict[str, Any] | None = None
+        best_score = 0.0
+        for row in all_rows:
+            row_tokens = set(tokenize(self._row_text(row)))
+            score = _jaccard(query_tokens, row_tokens)
+            if score > best_score and score >= threshold:
+                best_score = score
+                best_row = row
+        return best_row
+
+    def _enforce_cap(self, max_per_owner: int) -> int:
+        soft_deleted = 0
+        while True:
+            current = self.count()
+            if current <= max_per_owner:
+                break
+            excess = current - max_per_owner
+            rows = list(
+                self._table.find(
+                    deleted_at=None,
+                    order_by=["created_at"],
+                    _limit=excess,
+                    **self._scope,
+                )
+            )
+            if not rows:
+                break
+            now = to_iso(now_utc())
+            for row in rows:
+                self._table.update(
+                    {
+                        "id": row["id"],
+                        "deleted_at": now,
+                        "deleted_by": "retention",
+                    },
+                    ["id"],
+                )
+                soft_deleted += 1
+            self._dirty = True
+        if soft_deleted:
+            logger.info(
+                "Retention cap pruned %d lesson(s) for owner=%s/%s",
+                soft_deleted,
+                self._owner_kind,
+                self._owner_id,
+            )
+        return soft_deleted
+
     def add(
         self, observation: str, conclusion: str, next_action: str, tags: str = ""
     ) -> dict[str, Any]:
+        text = " ".join([observation, conclusion, next_action, tags])
+        similar = self._find_similar(text)
+        if similar and similar.get("id") is not None:
+            hits = (similar.get("hits") or 0) + 1
+            self._table.update(
+                {
+                    "id": similar["id"],
+                    "hits": hits,
+                    "created_at": to_iso(now_utc()),
+                },
+                ["id"],
+            )
+            self._dirty = True
+            logger.info(
+                "Lesson deduplicated owner=%s/%s hits=%d",
+                self._owner_kind,
+                self._owner_id,
+                hits,
+            )
+            return {**similar, "hits": hits, "deduplicated": True}
+
+        uid = uuid.uuid4().hex
         record = {
-            "uid": uuid.uuid4().hex,
+            "uid": uid,
             "observation": observation,
             "conclusion": conclusion,
             "next_action": next_action,
             "tags": tags,
             "created_at": to_iso(now_utc()),
             "hits": 0,
+            "rating": 0,
             "deleted_at": None,
             "deleted_by": None,
             **self._scope,
@@ -84,6 +208,8 @@ class LessonStore:
         logger.info(
             "Lesson stored owner=%s/%s tags=%s", self._owner_kind, self._owner_id, tags
         )
+        max_per, _ = _read_retention_settings(self._db)
+        self._enforce_cap(max_per)
         return record
 
     def all(self) -> list[dict[str, Any]]:
@@ -118,17 +244,108 @@ class LessonStore:
         )
         return n
 
+    def rate(self, uid: str, value: int) -> bool:
+        if TABLE not in self._db.tables:
+            return False
+        row = self._table.find_one(uid=uid, deleted_at=None, **self._scope)
+        if not row:
+            return False
+        current = row.get("rating") or 0
+        self._table.update(
+            {"id": row["id"], "rating": current + value},
+            ["id"],
+        )
+        self._dirty = True
+        logger.info(
+            "Lesson %s rated %+d (now %d) owner=%s/%s",
+            uid,
+            value,
+            current + value,
+            self._owner_kind,
+            self._owner_id,
+        )
+        return True
+
+    def prune(self, max_age_days: int | None = None) -> int:
+        if TABLE not in self._db.tables:
+            return 0
+        if max_age_days is None:
+            _, max_age_days = _read_retention_settings(self._db)
+        cutoff = now_utc() - timedelta(days=max_age_days)
+        cutoff_iso = to_iso(cutoff)
+        rows = list(
+            self._table.find(
+                deleted_at=None,
+                created_at={"<": cutoff_iso},
+                **self._scope,
+            )
+        )
+        if not rows:
+            return 0
+        now = to_iso(now_utc())
+        soft_deleted = 0
+        for row in rows:
+            self._table.update(
+                {
+                    "id": row["id"],
+                    "deleted_at": now,
+                    "deleted_by": "retention",
+                },
+                ["id"],
+            )
+            soft_deleted += 1
+        if soft_deleted:
+            self._dirty = True
+            logger.info(
+                "Pruned %d old lesson(s) for owner=%s/%s",
+                soft_deleted,
+                self._owner_kind,
+                self._owner_id,
+            )
+        return soft_deleted
+
+    def prune_all_owners(self, max_age_days: int | None = None) -> int:
+        if TABLE not in self._db.tables:
+            return 0
+        if max_age_days is None:
+            _, max_age_days = _read_retention_settings(self._db)
+        cutoff = now_utc() - timedelta(days=max_age_days)
+        cutoff_iso = to_iso(cutoff)
+        rows = list(
+            self._table.find(
+                deleted_at=None,
+                created_at={"<": cutoff_iso},
+            )
+        )
+        if not rows:
+            return 0
+        now = to_iso(now_utc())
+        soft_deleted = 0
+        for row in rows:
+            self._table.update(
+                {
+                    "id": row["id"],
+                    "deleted_at": now,
+                    "deleted_by": "retention",
+                },
+                ["id"],
+            )
+            soft_deleted += 1
+        if soft_deleted:
+            logger.info("Pruned %d old lesson(s) across all owners", soft_deleted)
+        return soft_deleted
+
     def _rebuild(self) -> None:
         rows = self.all()
         df: collections.Counter = collections.Counter()
         docs: list[dict[str, Any]] = []
         tf_list: list[collections.Counter] = []
-        dl: list[int] = []
+        dl_list: list[int] = []
         for row in rows:
-            text = " ".join(
+            rating = row.get("rating") or 0
-                str(row.get(field) or "")
+            if rating <= LOW_QUALITY_THRESHOLD:
-                for field in ("observation", "conclusion", "next_action", "tags")
+                continue
-            )
+            text = self._row_text(row)
             tokens = tokenize(text)
             if not tokens:
                 continue
@@ -137,12 +354,12 @@ class LessonStore:
                 df[term] += 1
             docs.append(row)
             tf_list.append(tf)
-            dl.append(len(tokens))
+            dl_list.append(len(tokens))
         self._docs = docs
         self._tf = tf_list
-        self._dl = dl
+        self._dl = dl_list
         self._n = len(docs)
-        self._avgdl = sum(dl) / max(self._n, 1)
+        self._avgdl = sum(dl_list) / max(self._n, 1)
         self._idf = {
             term: math.log((self._n - freq + 0.5) / (freq + 0.5) + 1)
             for term, freq in df.items()

devplacepy/services/devii/ai_correction/controller.py

@@ -70,7 +70,7 @@ class AiCorrectionController:
         if prompt_raw is None:
             prompt = user.get("ai_correction_prompt") or DEFAULT_CORRECTION_PROMPT
         else:
-            prompt = str(prompt_raw).strip()[:2000] or DEFAULT_CORRECTION_PROMPT
+            prompt = str(prompt_raw).strip()[:20000] or DEFAULT_CORRECTION_PROMPT
         get_table("users").update(
             {
                 "uid": self._owner_id,

devplacepy/services/devii/ai_modifier/controller.py

@@ -70,7 +70,7 @@ class AiModifierController:
         if prompt_raw is None:
             prompt = user.get("ai_modifier_prompt") or DEFAULT_MODIFIER_PROMPT
         else:
-            prompt = str(prompt_raw).strip()[:2000] or DEFAULT_MODIFIER_PROMPT
+            prompt = str(prompt_raw).strip()[:20000] or DEFAULT_MODIFIER_PROMPT
         get_table("users").update(
             {
                 "uid": self._owner_id,

devplacepy/services/devii/config.py

@@ -178,6 +178,7 @@ FIELD_PRICE_CACHE_HIT = "devii_price_cache_hit"
 FIELD_PRICE_CACHE_MISS = "devii_price_cache_miss"
 FIELD_PRICE_OUTPUT = "devii_price_output"
 FIELD_ALLOW_EVAL = "devii_allow_eval"
+FIELD_INTERACTIONS_DEFAULT = "devii_interactions_default"
 FIELD_BROWSER_CONTROL = "devii_browser_control"
 FIELD_TIMEOUT = "devii_timeout"
 FIELD_FETCH_TIMEOUT = "devii_fetch_timeout"

devplacepy/services/devii/interaction/__init__.py

@@ -0,0 +1,37 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from .capabilities import (
+    CHANNEL_SITE_CHAT,
+    CHANNEL_TELEGRAM,
+    CHANNEL_CLI,
+    CHANNEL_API,
+    CHANNEL_UNKNOWN,
+    ChannelContext,
+    channel_id_for_session,
+    context_for,
+    fragment_for,
+    interactions_enabled,
+)
+from .controller import InteractionController
+from . import prefs
+from .schema import InteractionRequest, InteractionResult, validate_prompt_args
+
+__all__ = [
+    "CHANNEL_SITE_CHAT",
+    "CHANNEL_TELEGRAM",
+    "CHANNEL_CLI",
+    "CHANNEL_API",
+    "CHANNEL_UNKNOWN",
+    "ChannelContext",
+    "channel_id_for_session",
+    "context_for",
+    "fragment_for",
+    "interactions_enabled",
+    "InteractionController",
+    "InteractionRequest",
+    "InteractionResult",
+    "validate_prompt_args",
+    "prefs",
+]

devplacepy/services/devii/interaction/actions.py

@@ -0,0 +1,130 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from ..actions.spec import Action, Param
+
+
+def arg(
+    name: str, description: str, required: bool = False, kind: str = "string"
+) -> Param:
+    return Param(
+        name=name,
+        location="body",
+        description=description,
+        required=required,
+        type=kind,
+    )
+
+
+UI = (
+    "Presents a channel-aware interactive prompt (CA-IWP). The host renders the best "
+    "affordance for the current channel (site custom elements, Telegram inline keys, or "
+    "plain numbered menus). Blocks until the user submits, cancels, or the timeout fires. "
+    "Never invent HTML or custom-element tags in your prose - call this tool instead."
+)
+
+INTERACTION_ACTIONS: tuple[Action, ...] = (
+    Action(
+        name="ui_prompt",
+        method="LOCAL",
+        path="",
+        summary="Ask the user an interactive question with widgets (confirm, choice, text, form)",
+        description=(
+            UI
+            + " Prefer this for decisions and short forms. Returns structured "
+            "{interaction_id, status, values, meta}. Branch on status: submitted, cancelled, "
+            "timeout, superseded, or error. Keep labels short and value tokens stable."
+        ),
+        handler="interaction",
+        requires_auth=False,
+        params=(
+            arg("title", "Short question title (≤120 chars).", required=True),
+            arg("description", "Optional help prose (≤500 chars)."),
+            arg(
+                "widgets",
+                "Ordered list of widget objects. Types: confirm, choice, choice_multi, "
+                "text, number, date, select, // (help), group. Each input needs name+label; "
+                "choice types need options[{value,label}].",
+                required=True,
+                kind="array",
+            ),
+            arg("submit_label", "Primary action label (default Confirm)."),
+            arg("cancel_label", "Cancel action label (default Cancel)."),
+            arg("cancelable", "Whether the user may cancel (default true).", kind="boolean"),
+            arg(
+                "timeout_sec",
+                "Optional timeout in seconds (0 = none, max 86400).",
+                kind="integer",
+            ),
+            arg("id", "Optional stable interaction id (a-z, digits, hyphens)."),
+        ),
+    ),
+    Action(
+        name="ui_cancel",
+        method="LOCAL",
+        path="",
+        summary="Cancel the open interactive prompt",
+        description="Closes the current open interaction (or the one named by id) with status cancelled.",
+        handler="interaction",
+        requires_auth=False,
+        params=(
+            arg("id", "Optional interaction id to cancel; defaults to the open one."),
+            arg("reason", "Optional cancel reason."),
+        ),
+    ),
+    Action(
+        name="ui_notify",
+        method="LOCAL",
+        path="",
+        summary="Show a non-blocking ephemeral status on channels that support it",
+        description=(
+            "Sends a short status line to the user without waiting. Only effective when the "
+            "channel advertises ephemeral_status (site-chat). Elsewhere it is skipped."
+        ),
+        handler="interaction",
+        requires_auth=False,
+        params=(arg("text", "Status text to show (≤200 chars).", required=True),),
+    ),
+    Action(
+        name="interactions_get",
+        method="LOCAL",
+        path="",
+        summary="Show whether interactive widgets (ui_prompt) are enabled for the user",
+        description=(
+            "Returns the effective interactive-widgets preference: enabled (bool), source "
+            "('user' override or 'default'), the admin default, and the user's override if set. "
+            "The site default is set by administrators on the Devii service; users may override "
+            "it on their profile or with interactions_set."
+        ),
+        handler="interaction",
+        requires_auth=True,
+        read_only=True,
+    ),
+    Action(
+        name="interactions_set",
+        method="LOCAL",
+        path="",
+        summary="Enable, disable, or reset interactive widgets for the user",
+        description=(
+            "Sets whether this account accepts interactive ui_prompt widgets. Pass enabled=true "
+            "or false to override the admin default; pass reset=true (or omit enabled and set "
+            "reset) to clear the override and inherit the admin default again. Guests cannot "
+            "set a preference."
+        ),
+        handler="interaction",
+        requires_auth=True,
+        params=(
+            arg(
+                "enabled",
+                "true to enable interactive widgets, false to disable them. Omit when reset=true.",
+                kind="boolean",
+            ),
+            arg(
+                "reset",
+                "true to clear the user override and inherit the administrator default again.",
+                kind="boolean",
+            ),
+        ),
+    ),
+)

devplacepy/services/devii/interaction/broker.py

@@ -0,0 +1,513 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from typing import Any, Awaitable, Callable, Optional
+
+import uuid_utils
+
+from .capabilities import (
+    CHANNEL_CLI,
+    CHANNEL_SITE_CHAT,
+    CHANNEL_TELEGRAM,
+    ChannelContext,
+    channel_id_for_session,
+    context_for,
+)
+from .markdown import project_markdown, project_plain_menu
+from .parse import parse_plain_reply
+from .schema import (
+    InteractionRequest,
+    InteractionResult,
+    count_input_fields,
+    validate_prompt_args,
+)
+
+logger = logging.getLogger("devii.interaction")
+
+EmitFn = Callable[[dict[str, Any]], Awaitable[None]]
+WaitFn = Callable[[str, dict[str, Any], float], Awaitable[Any]]
+TelegramPresentFn = Callable[[dict[str, Any]], Awaitable[Any]]
+PlainPresentFn = Callable[[str], Awaitable[str]]
+
+
+class InteractionBroker:
+    def __init__(
+        self,
+        session_channel: str = "main",
+        *,
+        owner_kind: str = "guest",
+        owner_id: str = "",
+        emit: Optional[EmitFn] = None,
+        wait_site: Optional[WaitFn] = None,
+        present_telegram: Optional[TelegramPresentFn] = None,
+        present_plain: Optional[PlainPresentFn] = None,
+    ) -> None:
+        self._session_channel = session_channel
+        self._channel_id = channel_id_for_session(session_channel)
+        self._owner_kind = owner_kind
+        self._owner_id = owner_id
+        self._emit = emit
+        self._wait_site = wait_site
+        self._present_telegram = present_telegram
+        self._present_plain = present_plain
+        self._open: dict[str, dict[str, Any]] = {}
+        self._lock = asyncio.Lock()
+        self._single_flight = True
+
+    @property
+    def channel_id(self) -> str:
+        return self._channel_id
+
+    def set_channel_id(self, channel_id: str) -> None:
+        self._channel_id = channel_id
+
+    def open_id(self) -> str | None:
+        if not self._open:
+            return None
+        return next(iter(self._open))
+
+    def open_request(self) -> InteractionRequest | None:
+        open_id = self.open_id()
+        if not open_id:
+            return None
+        payload = self._open.get(open_id) or {}
+        request = payload.get("request_model")
+        if isinstance(request, InteractionRequest):
+            return request
+        raw = payload.get("request")
+        if isinstance(raw, dict):
+            try:
+                return InteractionRequest.model_validate(raw)
+            except Exception:
+                return None
+        return None
+
+    def answer_text(self, text: str) -> dict[str, Any] | None:
+        open_id = self.open_id()
+        if not open_id:
+            return None
+        request = self.open_request()
+        if request is None:
+            return {
+                "handled": True,
+                "status": "error",
+                "error": "Open interaction has no parseable schema.",
+                "interaction_id": open_id,
+            }
+        status, values, error = parse_plain_reply(request, text)
+        if status == "error":
+            return {
+                "handled": True,
+                "status": "error",
+                "error": error or "Could not parse reply.",
+                "interaction_id": open_id,
+            }
+        result = {
+            "status": status,
+            "interaction_id": open_id,
+            "values": values if status == "submitted" else {},
+            "meta": {
+                "adapter": self._channel_id,
+                "degraded": True,
+                "via": "text",
+            },
+            "error": error,
+        }
+        return {
+            "handled": True,
+            "status": status,
+            "interaction_id": open_id,
+            "result": result,
+        }
+
+    def channel_context(self) -> ChannelContext:
+        owner_kind = getattr(self, "_owner_kind", "guest")
+        owner_id = getattr(self, "_owner_id", "")
+        return context_for(
+            self._channel_id,
+            open_interaction_id=self.open_id(),
+            owner_kind=owner_kind,
+            owner_id=owner_id,
+        )
+
+    def bind(
+        self,
+        *,
+        emit: Optional[EmitFn] = None,
+        wait_site: Optional[WaitFn] = None,
+        present_telegram: Optional[TelegramPresentFn] = None,
+        present_plain: Optional[PlainPresentFn] = None,
+    ) -> None:
+        if emit is not None:
+            self._emit = emit
+        if wait_site is not None:
+            self._wait_site = wait_site
+        if present_telegram is not None:
+            self._present_telegram = present_telegram
+        if present_plain is not None:
+            self._present_plain = present_plain
+
+    async def prompt(self, arguments: dict[str, Any]) -> InteractionResult:
+        request = validate_prompt_args(arguments)
+        async with self._lock:
+            if self._single_flight and self._open:
+                for open_id in list(self._open):
+                    await self._finish(
+                        open_id,
+                        status="superseded",
+                        values={},
+                        error=None,
+                    )
+            interaction_id = request.id or f"ix-{uuid_utils.uuid7().hex[:8]}"
+            if not interaction_id.startswith("ix-") and not request.id:
+                interaction_id = f"ix-{interaction_id}"
+            started = time.monotonic()
+            payload = {
+                "interaction_id": interaction_id,
+                "request": request.model_dump(),
+                "request_model": request,
+                "markdown": project_markdown(request, interaction_id),
+                "plain": project_plain_menu(request),
+                "channel_id": self._channel_id,
+                "started": started,
+                "future": asyncio.get_event_loop().create_future(),
+            }
+            self._open[interaction_id] = payload
+            logger.info(
+                "interaction open id=%s channel=%s widgets=%d",
+                interaction_id,
+                self._channel_id,
+                count_input_fields(request.widgets),
+            )
+
+        try:
+            result = await self._present(interaction_id, request, payload)
+        except Exception as exc:
+            logger.exception("interaction failed id=%s", interaction_id)
+            result = InteractionResult(
+                interaction_id=interaction_id,
+                status="error",
+                channel_id=self._channel_id,
+                values={},
+                meta={"adapter": self._channel_id, "degraded": True},
+                error=str(exc),
+            )
+            await self._resolve_local(interaction_id, result)
+            return result
+
+        latency_ms = int((time.monotonic() - started) * 1000)
+        if "latency_ms" not in result.meta:
+            result.meta["latency_ms"] = latency_ms
+        if "adapter" not in result.meta:
+            result.meta["adapter"] = self._channel_id
+        self._open.pop(interaction_id, None)
+        logger.info(
+            "interaction closed id=%s status=%s latency_ms=%s",
+            interaction_id,
+            result.status,
+            result.meta.get("latency_ms"),
+        )
+        return result
+
+    async def cancel(
+        self, interaction_id: str = "", reason: str = "cancelled"
+    ) -> InteractionResult:
+        target = interaction_id or self.open_id()
+        if not target or target not in self._open:
+            return InteractionResult(
+                interaction_id=target or "",
+                status="error",
+                channel_id=self._channel_id,
+                error="No open interaction to cancel.",
+            )
+        status = reason if reason in (
+            "cancelled",
+            "timeout",
+            "superseded",
+            "error",
+            "channel_changed",
+        ) else "cancelled"
+        result = InteractionResult(
+            interaction_id=target,
+            status=status,  # type: ignore[arg-type]
+            channel_id=self._channel_id,
+            values={},
+            meta={"adapter": self._channel_id, "reason": reason},
+        )
+        await self._resolve_local(target, result)
+        return result
+
+    def resolve(
+        self,
+        interaction_id: str,
+        *,
+        status: str,
+        values: dict[str, Any] | None = None,
+        error: str | None = None,
+        meta: dict[str, Any] | None = None,
+    ) -> bool:
+        payload = self._open.get(interaction_id)
+        if payload is None:
+            return False
+        result = InteractionResult(
+            interaction_id=interaction_id,
+            status=status,  # type: ignore[arg-type]
+            channel_id=self._channel_id,
+            values=values or {},
+            meta=meta or {"adapter": self._channel_id},
+            error=error,
+        )
+        future = payload.get("future")
+        if future is not None and not future.done():
+            future.set_result(result)
+        return True
+
+    async def _present(
+        self,
+        interaction_id: str,
+        request: InteractionRequest,
+        payload: dict[str, Any],
+    ) -> InteractionResult:
+        timeout = float(request.timeout_sec or 0)
+        if self._wait_site is not None and self._channel_id in (
+            CHANNEL_SITE_CHAT,
+            CHANNEL_TELEGRAM,
+        ):
+            return await self._present_site(interaction_id, request, payload, timeout)
+        if self._channel_id == CHANNEL_TELEGRAM:
+            return await self._present_telegram_path(
+                interaction_id, request, payload, timeout
+            )
+        return await self._present_plain_path(
+            interaction_id, request, payload, timeout
+        )
+
+    async def _present_site(
+        self,
+        interaction_id: str,
+        request: InteractionRequest,
+        payload: dict[str, Any],
+        timeout: float,
+    ) -> InteractionResult:
+        if self._wait_site is None:
+            return await self._present_plain_path(
+                interaction_id, request, payload, timeout, degraded=True
+            )
+        frame = {
+            "interaction_id": interaction_id,
+            "channel": self._channel_id,
+            "title": request.title,
+            "description": request.description,
+            "widgets": [w.model_dump() for w in request.widgets],
+            "submit_label": request.submit_label,
+            "cancel_label": request.cancel_label,
+            "cancelable": request.cancelable,
+            "timeout_sec": request.timeout_sec,
+            "markdown": payload["markdown"],
+            "plain": payload["plain"],
+            "request": request.model_dump(),
+        }
+        try:
+            raw = await self._wait_site(interaction_id, frame, timeout)
+        except asyncio.TimeoutError:
+            return InteractionResult(
+                interaction_id=interaction_id,
+                status="timeout",
+                channel_id=self._channel_id,
+                meta={
+                    "adapter": self._channel_id,
+                    "degraded": self._channel_id != CHANNEL_SITE_CHAT,
+                },
+            )
+        return self._normalize_site_result(
+            interaction_id,
+            raw,
+            adapter=self._channel_id,
+            degraded=self._channel_id != CHANNEL_SITE_CHAT,
+        )
+
+    async def _present_telegram_path(
+        self,
+        interaction_id: str,
+        request: InteractionRequest,
+        payload: dict[str, Any],
+        timeout: float,
+    ) -> InteractionResult:
+        if self._present_telegram is not None:
+            try:
+                raw = await self._present_telegram(
+                    {
+                        "interaction_id": interaction_id,
+                        "request": request.model_dump(),
+                        "markdown": payload["markdown"],
+                        "plain": payload["plain"],
+                        "timeout_sec": request.timeout_sec,
+                    }
+                )
+                return self._normalize_site_result(
+                    interaction_id, raw, adapter=CHANNEL_TELEGRAM, degraded=True
+                )
+            except asyncio.TimeoutError:
+                return InteractionResult(
+                    interaction_id=interaction_id,
+                    status="timeout",
+                    channel_id=self._channel_id,
+                    meta={"adapter": CHANNEL_TELEGRAM, "degraded": True},
+                )
+            except Exception as exc:
+                logger.exception("telegram adapter failed, falling back to plain")
+                payload["plain_error"] = str(exc)
+        return await self._present_plain_path(
+            interaction_id, request, payload, timeout, degraded=True
+        )
+
+    async def _present_plain_path(
+        self,
+        interaction_id: str,
+        request: InteractionRequest,
+        payload: dict[str, Any],
+        timeout: float,
+        *,
+        degraded: bool = False,
+    ) -> InteractionResult:
+        menu = payload["plain"]
+        if self._present_plain is not None:
+            try:
+                if timeout > 0:
+                    reply = await asyncio.wait_for(
+                        self._present_plain(menu), timeout=timeout
+                    )
+                else:
+                    reply = await self._present_plain(menu)
+            except asyncio.TimeoutError:
+                return InteractionResult(
+                    interaction_id=interaction_id,
+                    status="timeout",
+                    channel_id=self._channel_id,
+                    meta={
+                        "adapter": self._channel_id or CHANNEL_CLI,
+                        "degraded": True,
+                    },
+                )
+            status, values, error = parse_plain_reply(request, reply)
+            return InteractionResult(
+                interaction_id=interaction_id,
+                status=status,  # type: ignore[arg-type]
+                channel_id=self._channel_id,
+                values=values,
+                meta={
+                    "adapter": self._channel_id or CHANNEL_CLI,
+                    "degraded": True,
+                },
+                error=error,
+            )
+        future: asyncio.Future = payload["future"]
+        if self._emit is not None:
+            await self._emit(
+                {
+                    "type": "interaction",
+                    "id": interaction_id,
+                    "mode": "plain",
+                    "text": menu,
+                    "markdown": payload["markdown"],
+                }
+            )
+        try:
+            if timeout > 0:
+                result = await asyncio.wait_for(future, timeout=timeout)
+            else:
+                result = await future
+            if isinstance(result, InteractionResult):
+                result.meta.setdefault("degraded", degraded or True)
+                return result
+            return self._normalize_site_result(
+                interaction_id, result, adapter=self._channel_id, degraded=True
+            )
+        except asyncio.TimeoutError:
+            return InteractionResult(
+                interaction_id=interaction_id,
+                status="timeout",
+                channel_id=self._channel_id,
+                meta={"adapter": self._channel_id, "degraded": True},
+            )
+
+    def _normalize_site_result(
+        self,
+        interaction_id: str,
+        raw: Any,
+        *,
+        adapter: str = CHANNEL_SITE_CHAT,
+        degraded: bool = False,
+    ) -> InteractionResult:
+        if isinstance(raw, InteractionResult):
+            return raw
+        if not isinstance(raw, dict):
+            return InteractionResult(
+                interaction_id=interaction_id,
+                status="error",
+                channel_id=self._channel_id,
+                error="Invalid interaction result payload.",
+                meta={"adapter": adapter, "degraded": degraded},
+            )
+        if raw.get("error") and not raw.get("status"):
+            return InteractionResult(
+                interaction_id=interaction_id,
+                status="error",
+                channel_id=self._channel_id,
+                error=str(raw.get("error")),
+                meta={"adapter": adapter, "degraded": degraded},
+            )
+        status = str(raw.get("status") or "submitted")
+        if status not in (
+            "submitted",
+            "cancelled",
+            "timeout",
+            "superseded",
+            "error",
+            "channel_changed",
+        ):
+            status = "error"
+        values = raw.get("values") if isinstance(raw.get("values"), dict) else {}
+        meta = raw.get("meta") if isinstance(raw.get("meta"), dict) else {}
+        meta.setdefault("adapter", adapter)
+        meta.setdefault("degraded", degraded)
+        return InteractionResult(
+            interaction_id=str(raw.get("interaction_id") or interaction_id),
+            status=status,  # type: ignore[arg-type]
+            channel_id=self._channel_id,
+            values=values,
+            meta=meta,
+            error=str(raw["error"]) if raw.get("error") else None,
+        )
+
+    async def _finish(
+        self,
+        interaction_id: str,
+        *,
+        status: str,
+        values: dict[str, Any],
+        error: str | None,
+    ) -> None:
+        result = InteractionResult(
+            interaction_id=interaction_id,
+            status=status,  # type: ignore[arg-type]
+            channel_id=self._channel_id,
+            values=values,
+            meta={"adapter": self._channel_id},
+            error=error,
+        )
+        await self._resolve_local(interaction_id, result)
+
+    async def _resolve_local(
+        self, interaction_id: str, result: InteractionResult
+    ) -> None:
+        payload = self._open.pop(interaction_id, None)
+        if payload is None:
+            return
+        future = payload.get("future")
+        if future is not None and not future.done():
+            future.set_result(result)

devplacepy/services/devii/interaction/capabilities.py

@@ -0,0 +1,217 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+CHANNEL_SITE_CHAT = "site-chat"
+CHANNEL_TELEGRAM = "telegram"
+CHANNEL_CLI = "cli"
+CHANNEL_API = "api"
+CHANNEL_UNKNOWN = "unknown"
+
+UI_TOOL_NAMES = frozenset(
+    {
+        "ui_prompt",
+        "ui_cancel",
+        "ui_notify",
+        "interactions_get",
+        "interactions_set",
+    }
+)
+
+DEFAULT_LIMITS = {
+    "max_options": 32,
+    "max_fields": 24,
+    "max_label_chars": 200,
+    "max_help_chars": 500,
+    "callback_data_bytes": 0,
+}
+
+_SITE_CAPS = {
+    "rich_widgets": True,
+    "markdown": True,
+    "confirm": True,
+    "choice_single": True,
+    "choice_multi": True,
+    "text_input": True,
+    "number_input": True,
+    "date_input": True,
+    "file_input": False,
+    "inline_buttons": True,
+    "polls": False,
+    "web_app": False,
+    "streaming_partial": True,
+    "ephemeral_status": True,
+}
+
+_TELEGRAM_CAPS = {
+    "rich_widgets": False,
+    "markdown": True,
+    "confirm": True,
+    "choice_single": True,
+    "choice_multi": True,
+    "text_input": True,
+    "number_input": True,
+    "date_input": True,
+    "file_input": False,
+    "inline_buttons": True,
+    "polls": True,
+    "web_app": False,
+    "streaming_partial": False,
+    "ephemeral_status": False,
+}
+
+_PLAIN_CAPS = {
+    "rich_widgets": False,
+    "markdown": True,
+    "confirm": True,
+    "choice_single": True,
+    "choice_multi": True,
+    "text_input": True,
+    "number_input": True,
+    "date_input": True,
+    "file_input": False,
+    "inline_buttons": False,
+    "polls": False,
+    "web_app": False,
+    "streaming_partial": False,
+    "ephemeral_status": False,
+}
+
+_EMPTY_CAPS = {key: False for key in _SITE_CAPS}
+
+_MATRICES: dict[str, dict[str, bool]] = {
+    CHANNEL_SITE_CHAT: _SITE_CAPS,
+    CHANNEL_TELEGRAM: _TELEGRAM_CAPS,
+    CHANNEL_CLI: _PLAIN_CAPS,
+    CHANNEL_API: _PLAIN_CAPS,
+    CHANNEL_UNKNOWN: _EMPTY_CAPS,
+}
+
+_SESSION_CHANNEL_MAP = {
+    "main": CHANNEL_SITE_CHAT,
+    "docs": CHANNEL_SITE_CHAT,
+    "telegram": CHANNEL_TELEGRAM,
+    "cli": CHANNEL_CLI,
+    "api": CHANNEL_API,
+}
+
+
+@dataclass
+class ChannelContext:
+    channel_id: str
+    capabilities: dict[str, bool] = field(default_factory=dict)
+    limits: dict[str, int] = field(default_factory=dict)
+    tools: list[str] = field(default_factory=list)
+    open_interaction_id: str | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "channel_id": self.channel_id,
+            "capabilities": dict(self.capabilities),
+            "limits": dict(self.limits),
+            "tools": list(self.tools),
+            "open_interaction_id": self.open_interaction_id,
+        }
+
+
+def channel_id_for_session(session_channel: str) -> str:
+    return _SESSION_CHANNEL_MAP.get(str(session_channel or "").strip().lower(), CHANNEL_UNKNOWN)
+
+
+def capabilities_for(channel_id: str) -> dict[str, bool]:
+    return dict(_MATRICES.get(channel_id, _EMPTY_CAPS))
+
+
+def limits_for(channel_id: str) -> dict[str, int]:
+    limits = dict(DEFAULT_LIMITS)
+    if channel_id == CHANNEL_TELEGRAM:
+        limits["callback_data_bytes"] = 64
+        limits["max_options"] = 16
+    return limits
+
+
+def interactions_enabled(
+    owner_kind: str = "guest", owner_id: str = ""
+) -> bool:
+    from .prefs import effective_for
+
+    return effective_for(owner_kind, owner_id)
+
+
+def context_for(
+    channel_id: str,
+    *,
+    open_interaction_id: str | None = None,
+    tools_allowed: bool = True,
+    owner_kind: str = "guest",
+    owner_id: str = "",
+) -> ChannelContext:
+    caps = capabilities_for(channel_id)
+    limits = limits_for(channel_id)
+    tools: list[str] = []
+    pref_tools = ["interactions_get", "interactions_set"]
+    if owner_kind == "user":
+        tools.extend(pref_tools)
+    if (
+        tools_allowed
+        and interactions_enabled(owner_kind, owner_id)
+        and channel_id in (CHANNEL_SITE_CHAT, CHANNEL_TELEGRAM, CHANNEL_CLI, CHANNEL_API)
+    ):
+        if open_interaction_id:
+            tools = [t for t in tools if t in pref_tools] + ["ui_cancel"]
+        else:
+            tools = list(dict.fromkeys(tools + ["ui_prompt", "ui_cancel"]))
+            if caps.get("ephemeral_status"):
+                tools.append("ui_notify")
+    return ChannelContext(
+        channel_id=channel_id,
+        capabilities=caps,
+        limits=limits,
+        tools=tools,
+        open_interaction_id=open_interaction_id,
+    )
+
+
+def fragment_for(ctx: ChannelContext) -> str:
+    active_caps = [name for name, on in ctx.capabilities.items() if on]
+    caps_text = ",".join(active_caps) if active_caps else "none"
+    tools_text = ",".join(ctx.tools) if ctx.tools else "none"
+    interaction = (
+        f"open:{ctx.open_interaction_id}" if ctx.open_interaction_id else "none"
+    )
+    return (
+        f"CHANNEL: {ctx.channel_id}\n"
+        f"CAPS: {caps_text}\n"
+        f"LIMITS: options≤{ctx.limits.get('max_options', 32)} "
+        f"fields≤{ctx.limits.get('max_fields', 24)}\n"
+        f"TOOLS: {tools_text}\n"
+        f"INTERACTION: {interaction}"
+    )
+
+
+CA_IWP_SYSTEM_FRAGMENT = (
+    "# Interactive asks (CA-IWP)\n"
+    "You are channel-aware. Current channel and capabilities are injected each turn as "
+    "CHANNEL / CAPS / LIMITS / TOOLS / INTERACTION.\n"
+    "HARD RULES when TOOLS includes ui_prompt (this is the interactive mode):\n"
+    "1. You MUST call ui_prompt for every decision, confirm, choice, short form, yes/no, "
+    "pick-one, multi-select, text/number/date ask, or demo of interactive widgets. Do not "
+    "simulate buttons in markdown when ui_prompt is available.\n"
+    "2. Do NOT fall back to numbered lists, fake [Yes]/[No] prose, or 'I would show a widget' "
+    "while ui_prompt is in TOOLS. Only use degraded markdown menus when ui_prompt is absent "
+    "from TOOLS.\n"
+    "3. Never invent HTML, JavaScript, or custom-element tags in your visible reply. "
+    "The host renders UI from tool args.\n"
+    "4. Never claim buttons/widgets exist on a channel that lacks them; match wording to CHANNEL.\n"
+    "5. One open interaction at a time. Wait for the ui_prompt tool result before continuing.\n"
+    "6. The user may answer with the on-screen controls OR by typing a clear answer "
+    "(yes/no, option label/number, free text field value, DD/MM/YYYY, or cancel). "
+    "If they type a new request instead of an answer, treat the interaction as cancelled "
+    "and continue with their new request.\n"
+    "7. On status=cancelled or timeout, do not pretend the user agreed. Branch explicitly.\n"
+    "8. Keep tool arguments compact: short labels, stable value tokens, no essays inside options.\n"
+    "9. Do not use ui_prompt for long-form research answers or pure information delivery."
+)

devplacepy/services/devii/interaction/controller.py

@@ -0,0 +1,131 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any
+
+from devplacepy.services.devii.errors import ToolInputError
+
+from .broker import InteractionBroker
+from .capabilities import interactions_enabled
+from . import prefs
+from .schema import sanitize_text
+
+logger = logging.getLogger("devii.interaction")
+
+
+class InteractionController:
+    def __init__(
+        self,
+        broker: InteractionBroker,
+        owner_kind: str = "guest",
+        owner_id: str = "",
+    ) -> None:
+        self._broker = broker
+        self._owner_kind = owner_kind
+        self._owner_id = owner_id
+
+    @property
+    def broker(self) -> InteractionBroker:
+        return self._broker
+
+    async def dispatch(self, name: str, arguments: dict[str, Any]) -> str:
+        if name == "interactions_get":
+            return self._pref_get()
+        if name == "interactions_set":
+            return self._pref_set(arguments or {})
+        if name in ("ui_prompt", "ui_notify") and not interactions_enabled(
+            self._owner_kind, self._owner_id
+        ):
+            raise ToolInputError(
+                "Interactive widgets are disabled for this account. "
+                "Enable them with interactions_set or on the profile, or use a short numbered menu."
+            )
+        if name == "ui_prompt":
+            return await self._prompt(arguments)
+        if name == "ui_cancel":
+            return await self._cancel(arguments)
+        if name == "ui_notify":
+            return await self._notify(arguments)
+        raise ToolInputError(f"Unknown interaction tool: {name}")
+
+    def _require_user(self) -> None:
+        if self._owner_kind != "user" or not self._owner_id:
+            raise ToolInputError(
+                "Interactive-widget preferences are only available for signed-in users."
+            )
+
+    def _pref_get(self) -> str:
+        self._require_user()
+        data = prefs.snapshot(self._owner_kind, self._owner_id)
+        return json.dumps({"status": "success", **data}, ensure_ascii=False)
+
+    def _pref_set(self, arguments: dict[str, Any]) -> str:
+        self._require_user()
+        reset = arguments.get("reset")
+        if isinstance(reset, bool):
+            do_reset = reset
+        else:
+            do_reset = str(reset or "").strip().lower() in ("true", "1", "yes", "on")
+        if do_reset:
+            data = prefs.set_user_pref(self._owner_id, None)
+            return json.dumps({"status": "success", **data}, ensure_ascii=False)
+        if "enabled" not in arguments or arguments.get("enabled") is None:
+            raise ToolInputError(
+                "Pass enabled=true/false to override, or reset=true to inherit the admin default."
+            )
+        raw = arguments.get("enabled")
+        if isinstance(raw, bool):
+            enabled = raw
+        else:
+            enabled = str(raw).strip().lower() in ("true", "1", "yes", "on")
+        data = prefs.set_user_pref(self._owner_id, enabled)
+        return json.dumps({"status": "success", **data}, ensure_ascii=False)
+
+    async def _prompt(self, arguments: dict[str, Any]) -> str:
+        ctx = self._broker.channel_context()
+        if "ui_prompt" not in ctx.tools and self._broker.open_id():
+            raise ToolInputError(
+                "An interaction is already open. Wait for it, or call ui_cancel first."
+            )
+        if "ui_prompt" not in ctx.tools:
+            raise ToolInputError(
+                "ui_prompt is not available. Enable interactive widgets with interactions_set "
+                "or use a short numbered markdown menu."
+            )
+        result = await self._broker.prompt(arguments or {})
+        return json.dumps(result.to_dict(), ensure_ascii=False)
+
+    async def _cancel(self, arguments: dict[str, Any]) -> str:
+        interaction_id = sanitize_text(arguments.get("id", ""), 64)
+        reason = sanitize_text(arguments.get("reason", "cancelled"), 40) or "cancelled"
+        result = await self._broker.cancel(interaction_id, reason=reason)
+        return json.dumps(result.to_dict(), ensure_ascii=False)
+
+    async def _notify(self, arguments: dict[str, Any]) -> str:
+        text = sanitize_text(arguments.get("text", ""), 200)
+        if not text:
+            raise ToolInputError("'text' is required for ui_notify.")
+        ctx = self._broker.channel_context()
+        if not ctx.capabilities.get("ephemeral_status"):
+            return json.dumps(
+                {
+                    "status": "skipped",
+                    "channel_id": ctx.channel_id,
+                    "message": "ephemeral status is not supported on this channel",
+                },
+                ensure_ascii=False,
+            )
+        emit = getattr(self._broker, "_emit", None)
+        if emit is not None:
+            await emit({"type": "status", "text": text})
+        return json.dumps(
+            {
+                "status": "ok",
+                "channel_id": ctx.channel_id,
+                "text": text,
+            },
+            ensure_ascii=False,
+        )

devplacepy/services/devii/interaction/markdown.py

@@ -0,0 +1,106 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from .schema import InteractionRequest, WidgetModel, flatten_widgets
+
+
+def project_markdown(request: InteractionRequest, interaction_id: str) -> str:
+    lines: list[str] = [f"### {request.title}", ""]
+    if request.description:
+        lines.append(request.description)
+        lines.append("")
+    for widget in request.widgets:
+        lines.extend(_widget_lines(widget))
+    actions = f"[{request.submit_label}]"
+    if request.cancelable:
+        actions += f" [{request.cancel_label}]"
+    lines.append("")
+    lines.append(actions)
+    lines.append(f"<!-- ai-interaction:{interaction_id} -->")
+    return "\n".join(lines).strip() + "\n"
+
+
+def project_plain_menu(request: InteractionRequest) -> str:
+    lines: list[str] = [request.title]
+    if request.description:
+        lines.append(request.description)
+    lines.append("")
+    widgets = [
+        w
+        for w in flatten_widgets(request.widgets)
+        if w.type != "//"
+    ]
+    if len(widgets) == 1 and widgets[0].type == "confirm":
+        lines.append(f"Reply: yes | no | cancel")
+        return "\n".join(lines).strip() + "\n"
+    if len(widgets) == 1 and widgets[0].type in ("choice", "select"):
+        options = widgets[0].options[:12]
+        lines.append(f"{widgets[0].label}")
+        for index, option in enumerate(options, start=1):
+            lines.append(f"{index}. {option.label} (`{option.value}`)")
+        lines.append("")
+        lines.append(f"Reply with a number (1-{len(options)}) or `cancel`.")
+        return "\n".join(lines).strip() + "\n"
+    for widget in widgets:
+        if widget.type in ("choice", "choice_multi", "select"):
+            lines.append(f"{widget.label}:")
+            for index, option in enumerate(widget.options[:12], start=1):
+                lines.append(f"  {index}. {option.label} (`{option.value}`)")
+        elif widget.type == "confirm":
+            lines.append(f"{widget.label}: yes | no")
+        else:
+            lines.append(f"{widget.label}: (free text)")
+    lines.append("")
+    lines.append("Reply with values, or `cancel`.")
+    return "\n".join(lines).strip() + "\n"
+
+
+def _widget_lines(widget: WidgetModel, indent: str = "") -> list[str]:
+    if widget.type == "//":
+        return [f"{indent}{widget.text or widget.label}", ""]
+    if widget.type == "group":
+        lines = [f"{indent}**{widget.label}**", ""]
+        for child in widget.widgets:
+            lines.extend(_widget_lines(child, indent + "  "))
+        return lines
+    req = "required" if widget.required else "optional"
+    lines = [f"{indent}- [ ] **{widget.name}** ({req}) — {widget.label}"]
+    if widget.help:
+        lines.append(f"{indent}  _{widget.help}_")
+    if widget.type == "confirm":
+        default = widget.default
+        yes = "•" if default is True else " "
+        no = "•" if default is False else " "
+        lines.append(f"{indent}  - ({yes}) `true` — Yes")
+        lines.append(f"{indent}  - ({no}) `false` — No")
+    elif widget.type in ("choice", "select"):
+        default = str(widget.default) if widget.default is not None else ""
+        for option in widget.options:
+            mark = "•" if option.value == default else " "
+            desc = f" — {option.description}" if option.description else ""
+            lines.append(
+                f"{indent}  - ({mark}) `{option.value}` — {option.label}{desc}"
+            )
+    elif widget.type == "choice_multi":
+        defaults = set(widget.default or []) if isinstance(widget.default, list) else set()
+        for option in widget.options:
+            mark = "x" if option.value in defaults else " "
+            desc = f" — {option.description}" if option.description else ""
+            lines.append(
+                f"{indent}  - [{mark}] `{option.value}` — {option.label}{desc}"
+            )
+    elif widget.type == "text":
+        extra = f", max {widget.max_length}" if widget.max_length else ""
+        lines.append(f"{indent}  - _free text{extra}_")
+    elif widget.type == "number":
+        parts = []
+        if widget.min is not None:
+            parts.append(f"min {widget.min}")
+        if widget.max is not None:
+            parts.append(f"max {widget.max}")
+        hint = (", " + ", ".join(parts)) if parts else ""
+        lines.append(f"{indent}  - _number{hint}_")
+    elif widget.type == "date":
+        lines.append(f"{indent}  - _date DD/MM/YYYY_")
+    return lines

devplacepy/services/devii/interaction/parse.py

@@ -0,0 +1,187 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import re
+from datetime import datetime
+from typing import Any
+
+from .schema import InteractionRequest, WidgetModel, flatten_widgets
+
+YES = frozenset({"y", "yes", "true", "1", "ok", "confirm", "ja", "oui", "j"})
+NO = frozenset({"n", "no", "false", "0", "nee", "non"})
+CANCEL = frozenset({"cancel", "c", "abort", "quit", "annuleren", "stop"})
+DATE_FORMATS = (
+    "%d/%m/%Y",
+    "%d-%m-%Y",
+    "%d.%m.%Y",
+    "%d/%m/%y",
+    "%d-%m-%y",
+    "%Y-%m-%d",
+)
+
+
+def parse_plain_reply(
+    request: InteractionRequest, text: str
+) -> tuple[str, dict[str, Any], str | None]:
+    raw = (text or "").strip()
+    if not raw:
+        return "error", {}, "Empty reply."
+    lower = raw.lower()
+    if lower in CANCEL:
+        return "cancelled", {}, None
+
+    widgets = [w for w in flatten_widgets(request.widgets) if w.type != "//"]
+    if not widgets:
+        return "submitted", {}, None
+
+    if len(widgets) == 1:
+        return _parse_single(widgets[0], raw, strict=True)
+
+    if not _looks_like_multi_answer(raw, widgets):
+        return "error", {}, "Not a structured answer for the open form."
+
+    values: dict[str, Any] = {}
+    chunks = [part.strip() for part in re.split(r"[;\n]+", raw) if part.strip()]
+    if len(chunks) == 1 and "," in chunks[0] and len(widgets) > 1:
+        chunks = [part.strip() for part in chunks[0].split(",") if part.strip()]
+    if len(chunks) != len(widgets):
+        return (
+            "error",
+            {},
+            f"Expected {len(widgets)} values (one per field), got {len(chunks)}.",
+        )
+    for widget, chunk in zip(widgets, chunks):
+        status, partial, err = _parse_single(widget, chunk, strict=True)
+        if status != "submitted":
+            return status, {}, err
+        values.update(partial)
+    return "submitted", values, None
+
+
+def _looks_like_multi_answer(raw: str, widgets: list[WidgetModel]) -> bool:
+    if ";" in raw or "\n" in raw:
+        return True
+    if "," in raw and len(widgets) > 1:
+        return True
+    return False
+
+
+def _parse_single(
+    widget: WidgetModel, raw: str, *, strict: bool = False
+) -> tuple[str, dict[str, Any], str | None]:
+    text = raw.strip()
+    lower = text.lower()
+    if lower in CANCEL:
+        return "cancelled", {}, None
+    name = widget.name
+    if widget.type == "confirm":
+        if lower in YES:
+            return "submitted", {name: True}, None
+        if lower in NO:
+            return "submitted", {name: False}, None
+        return "error", {}, "Reply yes/no (or ja/nee)."
+    if widget.type in ("choice", "select"):
+        value = _match_option(widget, text)
+        if value is None:
+            return "error", {}, f"Unknown option for {name}."
+        return "submitted", {name: value}, None
+    if widget.type == "choice_multi":
+        parts = [p.strip() for p in re.split(r"[,\s]+", text) if p.strip()]
+        values: list[str] = []
+        for part in parts:
+            matched = _match_option(widget, part)
+            if matched is None:
+                return "error", {}, f"Unknown option {part!r} for {name}."
+            if matched not in values:
+                values.append(matched)
+        if widget.required and not values:
+            return "error", {}, f"{name} requires at least one option."
+        return "submitted", {name: values}, None
+    if widget.type == "number":
+        if strict and not re.fullmatch(r"[+-]?\d+(?:[.,]\d+)?", text):
+            return "error", {}, f"{name} must be a number."
+        normalized = text.replace(",", ".")
+        try:
+            number = float(normalized) if "." in normalized else int(normalized)
+        except ValueError:
+            return "error", {}, f"{name} must be a number."
+        if widget.min is not None and number < widget.min:
+            return "error", {}, f"{name} must be ≥ {widget.min}."
+        if widget.max is not None and number > widget.max:
+            return "error", {}, f"{name} must be ≤ {widget.max}."
+        return "submitted", {name: number}, None
+    if widget.type == "date":
+        if strict and not _looks_like_date(text):
+            return "error", {}, f"{name} must be a date as DD/MM/YYYY."
+        european = _parse_date_european(text)
+        if european is None:
+            return "error", {}, f"{name} must be a date as DD/MM/YYYY."
+        return "submitted", {name: european}, None
+    if widget.type == "text":
+        if strict and _looks_like_command(text):
+            return "error", {}, f"{name} does not look like a field answer."
+        if widget.max_length and len(text) > widget.max_length:
+            return "error", {}, f"{name} exceeds max length {widget.max_length}."
+        if widget.required and not text:
+            return "error", {}, f"{name} is required."
+        return "submitted", {name: text}, None
+    return "error", {}, f"Unsupported widget type {widget.type}."
+
+
+def _looks_like_date(text: str) -> bool:
+    return bool(
+        re.fullmatch(r"\d{1,2}[/.-]\d{1,2}[/.-]\d{2,4}", text.strip())
+        or re.fullmatch(r"\d{4}-\d{2}-\d{2}", text.strip())
+    )
+
+
+def _looks_like_command(text: str) -> bool:
+    value = text.strip().lower()
+    if not value:
+        return False
+    if value.startswith("/"):
+        return True
+    starters = (
+        "show me ",
+        "laat ",
+        "geef ",
+        "toon ",
+        "please ",
+        "can you ",
+        "could you ",
+        "i want ",
+        "ik wil ",
+        "volgende",
+        "next ",
+    )
+    return any(value.startswith(prefix) for prefix in starters)
+
+
+def _parse_date_european(text: str) -> str | None:
+    value = text.strip()
+    for fmt in DATE_FORMATS:
+        try:
+            return datetime.strptime(value, fmt).strftime("%d/%m/%Y")
+        except ValueError:
+            continue
+    return None
+
+
+def _match_option(widget: WidgetModel, text: str) -> str | None:
+    raw = text.strip()
+    if raw.isdigit():
+        index = int(raw)
+        if 1 <= index <= len(widget.options):
+            option = widget.options[index - 1]
+            if not option.disabled:
+                return option.value
+    lower = raw.lower()
+    for option in widget.options:
+        if option.disabled:
+            continue
+        if option.value == raw or option.value.lower() == lower:
+            return option.value
+        if option.label.lower() == lower:
+            return option.value
+    return None

devplacepy/services/devii/interaction/prefs.py

@@ -0,0 +1,119 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from typing import Any
+
+INHERIT = -1
+FIELD_DEFAULT = "devii_interactions_default"
+FIELD_LEGACY = "devii_interactions_enabled"
+
+
+def _truthy(raw: Any, default: bool = True) -> bool:
+    if raw is None:
+        return default
+    if isinstance(raw, bool):
+        return raw
+    if isinstance(raw, (int, float)):
+        return raw != 0
+    text = str(raw).strip().lower()
+    if text in ("",):
+        return default
+    return text not in ("0", "false", "off", "no")
+
+
+def admin_default() -> bool:
+    try:
+        from devplacepy.database import get_setting
+
+        raw = get_setting(FIELD_DEFAULT, "")
+        if raw == "" or raw is None:
+            raw = get_setting(FIELD_LEGACY, "1")
+        return _truthy(raw, True)
+    except Exception:
+        return True
+
+
+def _coerce_user_pref(raw: Any) -> int | None:
+    if raw is None:
+        return None
+    try:
+        value = int(raw)
+    except (TypeError, ValueError):
+        return None
+    if value == INHERIT:
+        return None
+    if value in (0, 1):
+        return value
+    return None
+
+
+def user_pref_raw(user: dict[str, Any] | None) -> int | None:
+    if not user:
+        return None
+    return _coerce_user_pref(user.get("interactions_enabled"))
+
+
+def effective_for_user(user: dict[str, Any] | None) -> bool:
+    pref = user_pref_raw(user)
+    if pref is None:
+        return admin_default()
+    return bool(pref)
+
+
+def effective_for(owner_kind: str, owner_id: str = "") -> bool:
+    if owner_kind != "user" or not owner_id:
+        return admin_default()
+    try:
+        from devplacepy.database import get_table
+
+        user = get_table("users").find_one(uid=owner_id)
+    except Exception:
+        return admin_default()
+    return effective_for_user(user)
+
+
+def snapshot(owner_kind: str, owner_id: str = "", user: dict[str, Any] | None = None) -> dict[str, Any]:
+    default = admin_default()
+    if owner_kind != "user":
+        return {
+            "enabled": default,
+            "source": "default",
+            "default": default,
+            "override": None,
+        }
+    if user is None and owner_id:
+        try:
+            from devplacepy.database import get_table
+
+            user = get_table("users").find_one(uid=owner_id)
+        except Exception:
+            user = None
+    pref = user_pref_raw(user)
+    if pref is None:
+        return {
+            "enabled": default,
+            "source": "default",
+            "default": default,
+            "override": None,
+        }
+    return {
+        "enabled": bool(pref),
+        "source": "user",
+        "default": default,
+        "override": bool(pref),
+    }
+
+
+def set_user_pref(user_uid: str, enabled: bool | None) -> dict[str, Any]:
+    from devplacepy.database import get_table
+    from devplacepy.utils import clear_user_cache
+
+    value = INHERIT if enabled is None else (1 if enabled else 0)
+    get_table("users").update(
+        {"uid": user_uid, "interactions_enabled": value},
+        ["uid"],
+    )
+    clear_user_cache(user_uid)
+    user = get_table("users").find_one(uid=user_uid) or {"uid": user_uid}
+    return snapshot("user", user_uid, user)

devplacepy/services/devii/interaction/schema.py

@@ -0,0 +1,260 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import re
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field, field_validator, model_validator
+
+from .capabilities import DEFAULT_LIMITS
+
+WIDGET_TYPES = (
+    "confirm",
+    "choice",
+    "choice_multi",
+    "text",
+    "number",
+    "date",
+    "select",
+    "//",
+    "group",
+)
+
+VALUE_RE = re.compile(r"^[a-z0-9_][a-z0-9_-]{0,63}$")
+NAME_RE = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]{0,63}$")
+ID_RE = re.compile(r"^[a-z][a-z0-9-]{0,63}$")
+CONTROL_CHARS = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")
+
+
+def sanitize_text(value: Any, max_len: int) -> str:
+    text = CONTROL_CHARS.sub("", str(value if value is not None else ""))
+    text = " ".join(text.split())
+    if len(text) > max_len:
+        text = text[:max_len]
+    return text
+
+
+class OptionModel(BaseModel):
+    value: str
+    label: str
+    description: str = ""
+    disabled: bool = False
+
+    @field_validator("value")
+    @classmethod
+    def _value(cls, v: str) -> str:
+        text = sanitize_text(v, 64).lower().replace(" ", "_")
+        if not VALUE_RE.match(text):
+            raise ValueError(
+                "option value must match ^[a-z0-9_][a-z0-9_-]{0,63}$"
+            )
+        return text
+
+    @field_validator("label")
+    @classmethod
+    def _label(cls, v: str) -> str:
+        text = sanitize_text(v, 80)
+        if not text:
+            raise ValueError("option label is required")
+        return text
+
+    @field_validator("description")
+    @classmethod
+    def _description(cls, v: str) -> str:
+        return sanitize_text(v, 120)
+
+
+class WidgetModel(BaseModel):
+    type: Literal[
+        "confirm",
+        "choice",
+        "choice_multi",
+        "text",
+        "number",
+        "date",
+        "select",
+        "//",
+        "group",
+    ]
+    name: str = ""
+    label: str = ""
+    help: str = ""
+    text: str = ""
+    required: bool = True
+    default: Any = None
+    display: Literal["auto", "radio", "select", "buttons"] = "auto"
+    options: list[OptionModel] = Field(default_factory=list)
+    widgets: list[WidgetModel] = Field(default_factory=list)
+    max_length: int | None = None
+    placeholder: str = ""
+    pattern: str = ""
+    min: float | None = None
+    max: float | None = None
+    step: float | None = None
+    min_selected: int | None = None
+    max_selected: int | None = None
+
+    @field_validator("name")
+    @classmethod
+    def _name(cls, v: str) -> str:
+        text = sanitize_text(v, 64)
+        return text
+
+    @field_validator("label", "help", "text", "placeholder", "pattern")
+    @classmethod
+    def _strings(cls, v: str) -> str:
+        return sanitize_text(v, 500)
+
+    @model_validator(mode="after")
+    def _shape(self) -> WidgetModel:
+        if self.type == "//":
+            if not self.text and not self.label:
+                raise ValueError("help widget (//) requires text")
+            if not self.text:
+                self.text = self.label
+            return self
+        if self.type == "group":
+            if not self.label:
+                raise ValueError("group requires label")
+            if not self.widgets:
+                raise ValueError("group requires nested widgets")
+            if len(self.widgets) > 24:
+                raise ValueError("group may contain at most 24 widgets")
+            return self
+        if not self.name or not NAME_RE.match(self.name):
+            raise ValueError(
+                f"widget name required and must match ^[a-zA-Z_][a-zA-Z0-9_]{{0,63}}$ (got {self.name!r})"
+            )
+        if not self.label:
+            raise ValueError(f"widget '{self.name}' requires label")
+        self.label = sanitize_text(self.label, 200)
+        self.help = sanitize_text(self.help, 500)
+        if self.type in ("choice", "choice_multi", "select"):
+            if not self.options:
+                raise ValueError(f"widget '{self.name}' requires options")
+            if len(self.options) > DEFAULT_LIMITS["max_options"]:
+                raise ValueError(
+                    f"widget '{self.name}' has too many options (max {DEFAULT_LIMITS['max_options']})"
+                )
+        if self.type == "select":
+            self.display = "select"
+        return self
+
+
+class InteractionRequest(BaseModel):
+    title: str
+    description: str = ""
+    widgets: list[WidgetModel]
+    submit_label: str = "Confirm"
+    cancel_label: str = "Cancel"
+    cancelable: bool = True
+    timeout_sec: int = 0
+    id: str = ""
+
+    @field_validator("title")
+    @classmethod
+    def _title(cls, v: str) -> str:
+        text = sanitize_text(v, 120)
+        if not text:
+            raise ValueError("title is required")
+        return text
+
+    @field_validator("description")
+    @classmethod
+    def _description(cls, v: str) -> str:
+        return sanitize_text(v, 500)
+
+    @field_validator("submit_label", "cancel_label")
+    @classmethod
+    def _labels(cls, v: str) -> str:
+        return sanitize_text(v, 40) or "Confirm"
+
+    @field_validator("timeout_sec")
+    @classmethod
+    def _timeout(cls, v: int) -> int:
+        n = int(v or 0)
+        if n < 0 or n > 86400:
+            raise ValueError("timeout_sec must be between 0 and 86400")
+        return n
+
+    @field_validator("id")
+    @classmethod
+    def _id(cls, v: str) -> str:
+        text = sanitize_text(v, 64).lower()
+        if text and not ID_RE.match(text):
+            raise ValueError("id must match ^[a-z][a-z0-9-]{0,63}$")
+        return text
+
+    @model_validator(mode="after")
+    def _widgets(self) -> InteractionRequest:
+        if not self.widgets:
+            raise ValueError("widgets must contain at least one item")
+        if len(self.widgets) > DEFAULT_LIMITS["max_fields"]:
+            raise ValueError(
+                f"at most {DEFAULT_LIMITS['max_fields']} top-level widgets"
+            )
+        depth = _max_depth(self.widgets)
+        if depth > 3:
+            raise ValueError("widget nesting depth must be ≤ 3")
+        return self
+
+
+class InteractionResult(BaseModel):
+    interaction_id: str
+    status: Literal[
+        "submitted", "cancelled", "timeout", "superseded", "error", "channel_changed"
+    ]
+    channel_id: str
+    values: dict[str, Any] = Field(default_factory=dict)
+    meta: dict[str, Any] = Field(default_factory=dict)
+    error: str | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "interaction_id": self.interaction_id,
+            "status": self.status,
+            "channel_id": self.channel_id,
+            "values": {} if self.status != "submitted" else dict(self.values),
+            "meta": dict(self.meta),
+            "error": self.error,
+        }
+
+
+def _max_depth(widgets: list[WidgetModel], depth: int = 1) -> int:
+    deepest = depth
+    for widget in widgets:
+        if widget.type == "group" and widget.widgets:
+            deepest = max(deepest, _max_depth(widget.widgets, depth + 1))
+    return deepest
+
+
+def validate_prompt_args(arguments: dict[str, Any]) -> InteractionRequest:
+    try:
+        return InteractionRequest.model_validate(arguments or {})
+    except Exception as exc:
+        from devplacepy.services.devii.errors import ToolInputError
+
+        raise ToolInputError(f"Invalid ui_prompt arguments: {exc}") from exc
+
+
+def count_input_fields(widgets: list[WidgetModel]) -> int:
+    total = 0
+    for widget in widgets:
+        if widget.type == "//":
+            continue
+        if widget.type == "group":
+            total += count_input_fields(widget.widgets)
+        else:
+            total += 1
+    return total
+
+
+def flatten_widgets(widgets: list[WidgetModel]) -> list[WidgetModel]:
+    out: list[WidgetModel] = []
+    for widget in widgets:
+        if widget.type == "group":
+            out.extend(flatten_widgets(widget.widgets))
+        else:
+            out.append(widget)
+    return out

devplacepy/services/devii/registry.py

@@ -19,6 +19,7 @@ from .actions.notification_actions import NOTIFICATION_ACTIONS
 from .actions.rsearch_actions import RSEARCH_ACTIONS
 from .actions.spec import Catalog
 from .actions.telegram_actions import TELEGRAM_ACTIONS
+from .interaction.actions import INTERACTION_ACTIONS
 from .virtual_tools.actions import VIRTUAL_TOOL_ACTIONS
 from .agentic.actions import AGENTIC_ACTIONS
 from .tasks.actions import TASK_ACTIONS
@@ -42,6 +43,7 @@ CATALOG = Catalog(
     + AI_MODIFIER_ACTIONS
     + EMAIL_ACTIONS
     + TELEGRAM_ACTIONS
+    + INTERACTION_ACTIONS
     + VIRTUAL_TOOL_ACTIONS
 )
 

devplacepy/services/devii/service.py

@@ -126,6 +126,16 @@ class DeviiService(BaseService):
             "raw JavaScript execution; read-only tools (context, discover, read) are unaffected.",
             group="Agent",
         ),
+        ConfigField(
+            config.FIELD_INTERACTIONS_DEFAULT,
+            "Interactive widgets default",
+            type="bool",
+            default=True,
+            help="Site default for CA-IWP interactive prompts (ui_prompt). Guests always use "
+            "this default. Signed-in users inherit it until they override it on their profile "
+            "or with the interactions_set tool.",
+            group="Agent",
+        ),
         ConfigField(
             config.FIELD_USER_DAILY_USD,
             "Max USD per user / 24h",
@@ -235,6 +245,24 @@ class DeviiService(BaseService):
             help="Connection/read timeout for IMAP and SMTP calls.",
             group="Email",
         ),
+        ConfigField(
+            "devii_lessons_max_per_owner",
+            "Max lessons per owner",
+            type="int",
+            default=500,
+            minimum=0,
+            help="Soft cap on active lessons per user/guest. When exceeded, the oldest lessons are pruned. 0 disables the cap.",
+            group="Memory",
+        ),
+        ConfigField(
+            "devii_lessons_max_age_days",
+            "Max lesson age (days)",
+            type="int",
+            default=90,
+            minimum=1,
+            help="Lessons older than this are soft-deleted on the periodic housekeeping pass.",
+            group="Memory",
+        ),
     ]
 
     def __init__(self):
@@ -316,12 +344,28 @@ class DeviiService(BaseService):
         ensured = self._ensure_task_schedulers()
         pruned = hub.ledger.prune(48)
         removed = await hub.gc_idle()
-        if pruned or removed or ensured:
+        lessons_pruned = self._prune_old_lessons()
+        if pruned or removed or ensured or lessons_pruned:
             self.log(
                 f"Housekeeping: ensured {ensured} task scheduler(s), pruned {pruned} "
-                f"ledger rows, closed {removed} idle sessions"
+                f"ledger rows, pruned {lessons_pruned} lessons, closed {removed} idle sessions"
             )
 
+    def _prune_old_lessons(self) -> int:
+        try:
+            from devplacepy.database import db
+            from .agentic.lessons import LessonStore, _read_retention_settings
+
+            _, max_age = _read_retention_settings(db)
+            store = LessonStore(db, "_global", "_global")
+            pruned = store.prune_all_owners(max_age)
+            if pruned:
+                logger.info("Devii lessons housekeeping: pruned %d across all owners", pruned)
+            return pruned
+        except Exception:
+            logger.exception("Devii lessons housekeeping failed")
+            return 0
+
     def _ensure_task_schedulers(self) -> int:
         from devplacepy.database import db
         from devplacepy.utils import is_admin, is_primary_admin

devplacepy/services/devii/session/core.py

@@ -19,10 +19,18 @@ from ..config import Settings
 from ..cost import CostTracker
 from ..cost.tracker import Pricing
 from ..http_client import PlatformClient
+from ..interaction import InteractionController
+from ..interaction.broker import InteractionBroker
+from ..interaction.capabilities import (
+    CA_IWP_SYSTEM_FRAGMENT,
+    UI_TOOL_NAMES,
+    fragment_for,
+)
 from ..llm import LLMClient
 from ..registry import CATALOG
 from ..tasks import Scheduler, TaskController, TaskStore
 from ..virtual_tools import VirtualToolController, VirtualToolStore
+from ..text import normalize_newlines
 from ._helpers import _format_offset, _now_iso, _repair_history
 from .prompts import (
     BEHAVIOR_HEADER,
@@ -98,6 +106,14 @@ class DeviiSession:
         )
         self._behavior_store = behavior_store
         self.behavior = BehaviorController(behavior_store)
+        self.interaction_broker = InteractionBroker(
+            session_channel=channel,
+            owner_kind=owner_kind,
+            owner_id=owner_id,
+        )
+        self.interaction = InteractionController(
+            self.interaction_broker, owner_kind=owner_kind, owner_id=owner_id
+        )
         self.dispatcher = Dispatcher(
             CATALOG,
             self.client,
@@ -113,6 +129,7 @@ class DeviiSession:
             owner_id=owner_id,
             virtual_tools=self.virtual_tools,
             behavior=self.behavior,
+            interaction=self.interaction,
         )
         self.tools = self._builtin_tools()
         self._system_prompt = (
@@ -164,8 +181,16 @@ class DeviiSession:
         self._turn_epoch = 0
 
     def restore_history(self, messages: list[dict[str, Any]]) -> None:
-        if messages:
+        if not messages:
-            self.agent._messages = messages
+            return
+        repaired: list[dict[str, Any]] = []
+        for message in messages:
+            item = dict(message)
+            content = item.get("content")
+            if isinstance(content, str):
+                item["content"] = normalize_newlines(content)
+            repaired.append(item)
+        self.agent._messages = repaired
 
     def history(self) -> list[dict[str, Any]]:
         visible: list[dict[str, Any]] = []
@@ -174,11 +199,12 @@ class DeviiSession:
             content = message.get("content")
             if role not in ("user", "assistant") or not content:
                 continue
-            if (
+            if not isinstance(content, str):
-                role == "user"
+                continue
-                and isinstance(content, str)
+            content = normalize_newlines(content).strip()
-                and content.startswith(INTERNAL_PREFIXES)
+            if not content:
-            ):
+                continue
+            if role == "user" and content.startswith(INTERNAL_PREFIXES):
                 continue
             visible.append({"role": role, "content": content})
         return visible
@@ -228,6 +254,10 @@ class DeviiSession:
         self._disconnected.clear()
         self.avatar.bind(self._avatar_request)
         self.browser.bind(self._client_request)
+        self.interaction_broker.bind(
+            emit=self._emit_interaction,
+            wait_site=self._interaction_wait,
+        )
         self.ensure_scheduler_started()
         if self._buffer:
             pending = self._buffer
@@ -248,6 +278,14 @@ class DeviiSession:
             self._disconnected.set()
             self.avatar.unbind()
             self.browser.unbind()
+            open_id = self.interaction_broker.open_id()
+            if open_id:
+                self.interaction_broker.resolve(
+                    open_id,
+                    status="cancelled",
+                    values={},
+                    error="Browser disconnected.",
+                )
         logger.info(
             "Session %s/%s detached (%d conns)",
             self.owner_kind,
@@ -457,7 +495,13 @@ class DeviiSession:
                 for s in schemas
                 if s.get("function", {}).get("name") in DOCS_TOOLS
             ]
-        return schemas
+        allowed = set(self.interaction_broker.channel_context().tools)
+        return [
+            s
+            for s in schemas
+            if s.get("function", {}).get("name") not in UI_TOOL_NAMES
+            or s.get("function", {}).get("name") in allowed
+        ]
 
     def _refresh_tools(self) -> None:
         if self.channel == "docs":
@@ -471,7 +515,14 @@ class DeviiSession:
             return self._system_prompt
         body = self._behavior_store.text().strip()
         section = BEHAVIOR_HEADER if not body else f"{BEHAVIOR_HEADER}\n{body}"
-        return f"{self._system_prompt}\n\n{self._clock_line()}\n\n{section}"
+        channel_block = fragment_for(self.interaction_broker.channel_context())
+        return (
+            f"{self._system_prompt}\n\n"
+            f"{CA_IWP_SYSTEM_FRAGMENT}\n\n"
+            f"{channel_block}\n\n"
+            f"{self._clock_line()}\n\n"
+            f"{section}"
+        )
 
     def _clock_line(self) -> str:
         from datetime import datetime, timedelta, timezone
@@ -745,6 +796,144 @@ class DeviiSession:
             if len(self._buffer) > 100:
                 self._buffer = self._buffer[-100:]
 
+    async def _emit_interaction(self, payload: dict[str, Any]) -> None:
+        await self._emit(payload, buffer=False)
+
+    async def _interaction_wait(
+        self, interaction_id: str, frame: dict[str, Any], timeout: float
+    ) -> Any:
+        loop = asyncio.get_event_loop()
+        deadline_seconds = timeout if timeout and timeout > 0 else BROWSER_REQUEST_DEADLINE_SECONDS
+        if timeout and timeout > 0:
+            deadline_seconds = max(timeout, 1.0)
+        else:
+            deadline_seconds = max(BROWSER_REQUEST_DEADLINE_SECONDS, 300.0)
+        deadline = loop.time() + deadline_seconds
+        request_id = f"ix:{interaction_id}"
+        future: asyncio.Future = loop.create_future()
+        self._pending[request_id] = future
+        wire = {
+            "type": "interaction",
+            "id": request_id,
+            "interaction_id": interaction_id,
+            "args": frame,
+        }
+        try:
+            while not future.done():
+                remaining = deadline - loop.time()
+                if remaining <= 0:
+                    raise asyncio.TimeoutError()
+                try:
+                    await asyncio.wait_for(self._connected.wait(), timeout=remaining)
+                except asyncio.TimeoutError as exc:
+                    raise asyncio.TimeoutError() from exc
+                ws = self._pick_target()
+                if ws is None:
+                    continue
+                try:
+                    await self._send_to(ws, wire)
+                except Exception:
+                    await asyncio.sleep(0.1)
+                    continue
+                drop = asyncio.create_task(self._disconnected.wait())
+                try:
+                    await asyncio.wait(
+                        {future, drop},
+                        timeout=max(0.0, deadline - loop.time()),
+                        return_when=asyncio.FIRST_COMPLETED,
+                    )
+                finally:
+                    drop.cancel()
+            return future.result()
+        finally:
+            self._pending.pop(request_id, None)
+
+    def resolve_interaction(self, interaction_id: str, payload: Any) -> None:
+        request_id = str(interaction_id or "")
+        if not request_id.startswith("ix:"):
+            request_id = f"ix:{request_id}"
+        future = self._pending.get(request_id)
+        if future is not None and not future.done():
+            future.set_result(payload)
+            self._pending.pop(request_id, None)
+            bare = request_id[3:]
+            if isinstance(payload, dict):
+                self.interaction_broker.resolve(
+                    bare,
+                    status=str(payload.get("status") or "submitted"),
+                    values=payload.get("values")
+                    if isinstance(payload.get("values"), dict)
+                    else {},
+                    error=payload.get("error"),
+                    meta=payload.get("meta")
+                    if isinstance(payload.get("meta"), dict)
+                    else None,
+                )
+            return
+        bare = request_id[3:] if request_id.startswith("ix:") else request_id
+        self.interaction_broker.resolve(
+            bare,
+            status=str((payload or {}).get("status") or "submitted")
+            if isinstance(payload, dict)
+            else "error",
+            values=(payload or {}).get("values")
+            if isinstance(payload, dict)
+            else {},
+            error=(payload or {}).get("error") if isinstance(payload, dict) else None,
+            meta=(payload or {}).get("meta") if isinstance(payload, dict) else None,
+        )
+
+    async def try_answer_interaction(self, text: str) -> bool:
+        open_id = self.interaction_broker.open_id()
+        if not open_id:
+            return False
+        pending = self._pending.get(f"ix:{open_id}")
+        if pending is not None and pending.done():
+            return False
+        outcome = self.interaction_broker.answer_text(text)
+        if not outcome or not outcome.get("handled"):
+            return False
+        if outcome.get("status") == "error":
+            result = {
+                "status": "cancelled",
+                "interaction_id": open_id,
+                "values": {},
+                "meta": {
+                    "adapter": "site-chat",
+                    "via": "text_non_answer",
+                    "degraded": True,
+                },
+                "error": None,
+            }
+            self.resolve_interaction(open_id, result)
+            await self._emit(
+                {
+                    "type": "interaction_closed",
+                    "id": f"ix:{open_id}",
+                    "interaction_id": open_id,
+                    "result": result,
+                },
+                buffer=False,
+            )
+            return False
+        result = outcome.get("result") or {
+            "status": outcome.get("status") or "submitted",
+            "interaction_id": open_id,
+            "values": {},
+            "meta": {"adapter": "site-chat", "via": "text", "degraded": True},
+        }
+        self.resolve_interaction(open_id, result)
+        await self._emit(
+            {
+                "type": "interaction_closed",
+                "id": f"ix:{open_id}",
+                "interaction_id": open_id,
+                "result": result,
+            },
+            buffer=False,
+        )
+        return True
+
     async def _avatar_request(self, action: str, args: dict[str, Any]) -> Any:
         return await self._browser_request("avatar", action, args)
 

devplacepy/services/devii/text.py

@@ -18,6 +18,25 @@ BLANK_LINES = re.compile(r"\n\s*\n\s*\n+")
 TRAILING_SPACE = re.compile(r"[ \t]+\n")
 SKIP_HREF_PREFIXES = ("#", "javascript:")
 
+def normalize_newlines(text: str) -> str:
+    if not text or not isinstance(text, str):
+        return "" if text is None else str(text)
+    value = text
+    while "\\\\n" in value:
+        value = value.replace("\\\\n", "\\n")
+    while "\\\\t" in value:
+        value = value.replace("\\\\t", "\\t")
+    escaped_n = value.count("\\n")
+    real_n = value.count("\n")
+    if escaped_n > real_n:
+        value = value.replace("\\n", "\n")
+    escaped_t = value.count("\\t")
+    real_t = value.count("\t")
+    if escaped_t > real_t:
+        value = value.replace("\\t", "\t")
+    return value
+
+
 HIDDEN = "[hidden]"
 REDACT_FIELD_KEYS = frozenset(
     {

devplacepy/services/devrant/tokens.py

@@ -85,3 +85,6 @@ def revoke_all(user_uid: str) -> None:
         tokens.update(
             {"id": token["id"], "deleted_at": stamp, "deleted_by": user_uid}, ["id"]
         )
+    from devplacepy.utils.authcache import clear_user_cache
+
+    clear_user_cache(user_uid)

devplacepy/services/gitea/enhance.py

@@ -88,7 +88,10 @@ async def enhance_ticket(
         "max_tokens": MAX_TOKENS,
         "temperature": 0.2,
     }
-    headers = {"Content-Type": "application/json"}
+    headers = {
+        "Content-Type": "application/json",
+        "X-App-Reference": "devplace-gitea-enhance-v-1-0-0",
+    }
     if config.ai_key:
         headers["Authorization"] = f"Bearer {config.ai_key}"
 

devplacepy/services/gitea/planning.py

@@ -226,7 +226,10 @@ async def generate_plan(
         "max_tokens": MAX_TOKENS,
         "temperature": 0.2,
     }
-    headers = {"Content-Type": "application/json"}
+    headers = {
+        "Content-Type": "application/json",
+        "X-App-Reference": "devplace-gitea-planning-v-1-0-0",
+    }
     if config.ai_key:
         headers["Authorization"] = f"Bearer {config.ai_key}"
 

devplacepy/services/jobs/award_service.py

@@ -169,7 +169,10 @@ class AwardService(JobService):
             "output_format": "png",
         }
         url = f"{INTERNAL_BASE_URL}/openai/v1/images/generations"
-        headers = {"Content-Type": "application/json"}
+        headers = {
+            "Content-Type": "application/json",
+            "X-App-Reference": "devplace-awards-v-1-0-0",
+        }
         if api_key:
             headers["Authorization"] = f"Bearer {api_key}"
         with stealth.stealth_sync_client(timeout=AWARD_GENERATION_TIMEOUT_SECONDS) as client:

devplacepy/services/jobs/base.py

@@ -242,43 +242,30 @@ class JobService(BaseService):
             return None
 
     def collect_metrics(self) -> dict:
-        rows = queue.list_jobs(kind=self.kind)
+        metrics = queue.job_metrics(self.kind)
-        by_status = {queue.PENDING: 0, queue.RUNNING: 0, queue.DONE: 0, queue.FAILED: 0}
+        by_status = metrics["by_status"]
-        bytes_in = bytes_out = items = 0
-        durations = []
-        for row in rows:
-            by_status[row.get("status", "")] = (
-                by_status.get(row.get("status", ""), 0) + 1
-            )
-            bytes_in += int(row.get("bytes_in") or 0)
-            bytes_out += int(row.get("bytes_out") or 0)
-            items += int(row.get("item_count") or 0)
-            if row.get("status") == queue.DONE and row.get("duration_ms"):
-                durations.append(int(row["duration_ms"]))
-        avg_ms = int(sum(durations) / len(durations)) if durations else 0
         stats = [
-            {"label": "Total jobs", "value": len(rows)},
+            {"label": "Total jobs", "value": metrics["total"]},
             {"label": "Pending", "value": by_status[queue.PENDING]},
             {"label": "Running", "value": by_status[queue.RUNNING]},
             {"label": "Done", "value": by_status[queue.DONE]},
             {"label": "Failed", "value": by_status[queue.FAILED]},
-            {"label": "Items zipped", "value": items},
+            {"label": "Items zipped", "value": metrics["items"]},
-            {"label": "Bytes in", "value": _human_bytes(bytes_in)},
+            {"label": "Bytes in", "value": _human_bytes(metrics["bytes_in"])},
-            {"label": "Bytes out", "value": _human_bytes(bytes_out)},
+            {"label": "Bytes out", "value": _human_bytes(metrics["bytes_out"])},
-            {"label": "Avg duration", "value": f"{avg_ms} ms"},
+            {"label": "Avg duration", "value": f"{metrics['avg_ms']} ms"},
         ]
-        recent = sorted(rows, key=lambda r: r.get("uid", ""), reverse=True)[:10]
         table = {
             "columns": ["Job", "Status", "Name", "Out", "Items"],
             "rows": [
                 [
-                    row.get("uid", "")[:8],
+                    (row.get("uid") or "")[:8],
                     row.get("status", ""),
                     (row.get("preferred_name") or "")[:32],
                     _human_bytes(int(row.get("bytes_out") or 0)),
                     int(row.get("item_count") or 0),
                 ]
-                for row in recent
+                for row in metrics["recent"]
             ],
         }
         return {"stats": stats, "table": table}

devplacepy/services/jobs/deepsearch/crawl.py

@@ -167,29 +167,35 @@ def _snippet_page(candidate: dict, depth: int) -> CrawledPage | None:
     )
 
 
-async def _render_with_playwright(url: str) -> tuple[str, str, int, list[tuple[str, str]]]:
+async def _render_with_playwright(
+    url: str, browser=None
+) -> tuple[str, str, int, list[tuple[str, str]]]:
     from playwright.async_api import async_playwright
 
-    async with async_playwright() as pw:
+    own_browser = browser is None
+    if own_browser:
+        pw = await async_playwright().__aenter__()
         browser = await pw.chromium.launch(
             headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
         )
-        try:
+    try:
-            context = await browser.new_context(user_agent=USER_AGENT)
+        context = await browser.new_context(user_agent=USER_AGENT)
-            page = await context.new_page()
+        page = await context.new_page()
-            response = await page.goto(url, wait_until="load", timeout=30000)
+        response = await page.goto(url, wait_until="load", timeout=30000)
-            status = response.status if response else 0
+        status = response.status if response else 0
-            for hop in [response.url] if response else []:
+        for hop in [response.url] if response else []:
-                await guard_public_url(hop)
+            await guard_public_url(hop)
-            content = (await page.content())[:MAX_FETCH_BYTES]
+        content = (await page.content())[:MAX_FETCH_BYTES]
-            await context.close()
+        await context.close()
-            extracted = extract_html(content, base_url=url)
+        extracted = extract_html(content, base_url=url)
-            return extracted.title, extracted.text, status, extracted.links
+        return extracted.title, extracted.text, status, extracted.links
-        finally:
+    finally:
+        if own_browser:
             await browser.close()
+            await pw.__aexit__(None, None, None)
 
 
-async def fetch_page(url: str, depth: int) -> CrawledPage | None:
+async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None:
     try:
         await guard_public_url(url)
     except BlockedAddressError:
@@ -237,7 +243,7 @@ async def fetch_page(url: str, depth: int) -> CrawledPage | None:
             logger.info("deepsearch decode failed for %s: %s", url, exc)
     if len(text) < MIN_PAGE_CHARS:
         try:
-            r_title, r_text, r_status, r_links = await _render_with_playwright(url)
+            r_title, r_text, r_status, r_links = await _render_with_playwright(url, browser)
             if len(r_text) > len(text):
                 title, text, status, source, links = (
                     r_title or title,
@@ -261,12 +267,12 @@ async def fetch_page(url: str, depth: int) -> CrawledPage | None:
     )
 
 
-async def _resolve_candidate(candidate: dict, depth: int) -> CrawledPage | None:
+async def _resolve_candidate(candidate: dict, depth: int, browser=None) -> CrawledPage | None:
     url = candidate["url"]
     snippet_page = _snippet_page(candidate, depth)
     if _is_hostile(url):
         return snippet_page
-    page = await fetch_page(url, depth)
+    page = await fetch_page(url, depth, browser)
     if page and snippet_page:
         return page if len(page.text) >= len(snippet_page.text) else snippet_page
     return page or snippet_page
@@ -281,92 +287,111 @@ async def crawl(
     query: str = "",
     depth: int = 1,
 ) -> CrawlOutcome:
+    from playwright.async_api import async_playwright
+
     outcome = CrawlOutcome()
     fetched = 0
     seen_urls = {candidate["url"] for candidate in candidates}
     level_candidates = list(candidates)
     total = min(len(level_candidates), max_pages)
     cancelled = False
-    for level in range(max(1, depth)):
+
-        if cancelled or fetched >= max_pages or not level_candidates:
+    pw = None
-            break
+    browser = None
-        next_candidates: list[dict] = []
+    try:
-        for start in range(0, len(level_candidates), CRAWL_CONCURRENCY):
+        pw = await async_playwright().__aenter__()
-            if fetched >= max_pages:
+        browser = await pw.chromium.launch(
+            headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
+        )
+    except Exception as exc:
+        logger.warning("deepsearch playwright launch failed, pages will use httpx only: %s", exc)
+
+    try:
+        for level in range(max(1, depth)):
+            if cancelled or fetched >= max_pages or not level_candidates:
                 break
-            if await should_stop():
+            next_candidates: list[dict] = []
-                emit({"type": "stage", "stage": "cancelled", "message": "Crawl cancelled"})
+            for start in range(0, len(level_candidates), CRAWL_CONCURRENCY):
-                cancelled = True
-                break
-            batch = level_candidates[start : start + CRAWL_CONCURRENCY][: max_pages - fetched]
-            for candidate in batch:
-                emit(
-                    {
-                        "type": "progress",
-                        "done": fetched,
-                        "total": total,
-                        "url": candidate["url"],
-                        "depth": level,
-                        "message": f"Reading {candidate['url']}",
-                    }
-                )
-                if is_cached(candidate["url"]):
-                    emit({"type": "page_cached", "url": candidate["url"], "reason": "seen in a prior run"})
-            fetch_start = time.perf_counter()
-            results = await asyncio.gather(
-                *(_resolve_candidate(candidate, level) for candidate in batch),
-                return_exceptions=True,
-            )
-            elapsed_ms = int((time.perf_counter() - fetch_start) * 1000)
-            for candidate, page in zip(batch, results):
-                url = candidate["url"]
-                if isinstance(page, BaseException):
-                    logger.info("deepsearch fetch crashed for %s: %s", url, page)
-                    page = None
-                if page is None:
-                    emit(
-                        {
-                            "type": "page_skipped",
-                            "url": url,
-                            "reason": "no readable content",
-                            "elapsed_ms": elapsed_ms,
-                        }
-                    )
-                    continue
                 if fetched >= max_pages:
                     break
-                digest = content_hash(page.text)
+                if await should_stop():
-                if digest in outcome.seen_hashes:
+                    emit({"type": "stage", "stage": "cancelled", "message": "Crawl cancelled"})
+                    cancelled = True
+                    break
+                batch = level_candidates[start : start + CRAWL_CONCURRENCY][: max_pages - fetched]
+                for candidate in batch:
                     emit(
                         {
-                            "type": "page_duplicate",
+                            "type": "progress",
-                            "url": url,
+                            "done": fetched,
-                            "reason": "duplicate content",
+                            "total": total,
-                            "elapsed_ms": elapsed_ms,
+                            "url": candidate["url"],
+                            "depth": level,
+                            "message": f"Reading {candidate['url']}",
                         }
                     )
-                    continue
+                    if is_cached(candidate["url"]):
-                outcome.seen_hashes.add(digest)
+                        emit({"type": "page_cached", "url": candidate["url"], "reason": "seen in a prior run"})
-                outcome.pages.append(page)
+                fetch_start = time.perf_counter()
-                fetched += 1
+                results = await asyncio.gather(
-                emit(
+                    *(_resolve_candidate(candidate, level, browser) for candidate in batch),
-                    {
+                    return_exceptions=True,
-                        "type": "page_loaded",
-                        "url": page.url,
-                        "title": page.title,
-                        "source": page.source,
-                        "depth": level,
-                        "render": page.source == "playwright",
-                        "elapsed_ms": elapsed_ms,
-                        "done": fetched,
-                        "total": total,
-                    }
                 )
-                if level + 1 < depth:
+                elapsed_ms = int((time.perf_counter() - fetch_start) * 1000)
-                    for link in relevant_links(page.links, query, LINKS_PER_PAGE):
+                for candidate, page in zip(batch, results):
-                        if link not in seen_urls:
+                    url = candidate["url"]
-                            seen_urls.add(link)
+                    if isinstance(page, BaseException):
-                            next_candidates.append({"url": link})
+                        logger.info("deepsearch fetch crashed for %s: %s", url, page)
-        level_candidates = next_candidates
+                        page = None
-        total = min(total + len(next_candidates), max_pages)
+                    if page is None:
+                        emit(
+                            {
+                                "type": "page_skipped",
+                                "url": url,
+                                "reason": "no readable content",
+                                "elapsed_ms": elapsed_ms,
+                            }
+                        )
+                        continue
+                    if fetched >= max_pages:
+                        break
+                    digest = content_hash(page.text)
+                    if digest in outcome.seen_hashes:
+                        emit(
+                            {
+                                "type": "page_duplicate",
+                                "url": url,
+                                "reason": "duplicate content",
+                                "elapsed_ms": elapsed_ms,
+                            }
+                        )
+                        continue
+                    outcome.seen_hashes.add(digest)
+                    outcome.pages.append(page)
+                    fetched += 1
+                    emit(
+                        {
+                            "type": "page_loaded",
+                            "url": page.url,
+                            "title": page.title,
+                            "source": page.source,
+                            "depth": level,
+                            "render": page.source == "playwright",
+                            "elapsed_ms": elapsed_ms,
+                            "done": fetched,
+                            "total": total,
+                        }
+                    )
+                    if level + 1 < depth:
+                        for link in relevant_links(page.links, query, LINKS_PER_PAGE):
+                            if link not in seen_urls:
+                                seen_urls.add(link)
+                                next_candidates.append({"url": link})
+            level_candidates = next_candidates
+            total = min(total + len(next_candidates), max_pages)
+    finally:
+        if browser is not None:
+            await browser.close()
+        if pw is not None:
+            await pw.__aexit__(None, None, None)
     return outcome

devplacepy/services/jobs/deepsearch/enhance.py

@@ -81,6 +81,7 @@ async def plan_queries(
     headers = {
         "Authorization": f"Bearer {api_key}",
         "Content-Type": "application/json",
+        "X-App-Reference": "devplace-deepsearch-v-1-0-0",
     }
     try:
         async with stealth.stealth_async_client(timeout=ENHANCE_TIMEOUT_SECONDS) as client:

devplacepy/services/jobs/deepsearch/orchestrate.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import asyncio
 import json
 import logging
 import re
@@ -79,10 +80,14 @@ async def _retrieve_chunks(question: str, queries: list[str], store, api_key: st
         vectors = local_embed(texts).vectors
         if not vectors or len(vectors[0]) != stored_dim:
             return []
-    per_query = [
+    per_query = list(
-        store.hybrid_search(text, vector, top_k=RETRIEVE_TOP_K)
+        await asyncio.gather(
-        for text, vector in zip(texts, vectors)
+            *(
-    ]
+                store.hybrid_search(text, vector, top_k=RETRIEVE_TOP_K)
+                for text, vector in zip(texts, vectors)
+            )
+        )
+    )
     merged: list = []
     seen: set[str] = set()
     for tier in zip_longest(*per_query):
@@ -354,7 +359,7 @@ async def orchestrate(
     chunks: list = []
     if store is not None:
         try:
-            if store.count():
+            if await store.count():
                 chunks = await _retrieve_chunks(question, queries or [], store, api_key)
         except Exception as exc:
             logger.warning("deepsearch retrieval failed, using page context: %s", exc)

devplacepy/services/jobs/deepsearch/worker.py

@@ -149,7 +149,7 @@ async def _index_chunks(
     if forced_local:
         collected = local_embed([chunk.text for chunk in chunks]).vectors
         backend = "local"
-    store.add(chunks, collected)
+    await store.add(chunks, collected)
     final_backend = "local" if forced_local else backend
     emit({"type": "embed_done", "backend": final_backend, "chunk_count": total})
     return total, final_backend

devplacepy/services/jobs/isslop/agent/llm.py

@@ -64,7 +64,11 @@ class LlmClient:
             "messages": messages,
             "temperature": LLM_TEMPERATURE,
         }
-        headers = {"authorization": f"Bearer {self._api_key}", "content-type": "application/json"}
+        headers = {
+            "authorization": f"Bearer {self._api_key}",
+            "content-type": "application/json",
+            "X-App-Reference": "devplace-isslop-v-1-0-0",
+        }
         request_timeout = httpx.Timeout(timeout) if timeout else None
         last_error = "unknown"
         for attempt in range(1, LLM_MAX_RETRIES + 1):

devplacepy/services/jobs/queue.py

@@ -97,6 +97,49 @@ def touch_job(uid: str, extend_seconds: int) -> None:
     )
 
 
+def job_metrics(kind: str) -> dict:
+    empty = {
+        "total": 0,
+        "by_status": {PENDING: 0, RUNNING: 0, DONE: 0, FAILED: 0},
+        "bytes_in": 0,
+        "bytes_out": 0,
+        "items": 0,
+        "avg_ms": 0,
+        "recent": [],
+    }
+    if "jobs" not in db.tables:
+        return empty
+    metrics = dict(empty, by_status=dict(empty["by_status"]))
+    for row in db.query(
+        "SELECT status, COUNT(*) AS n, "
+        "COALESCE(SUM(bytes_in), 0) AS bytes_in, "
+        "COALESCE(SUM(bytes_out), 0) AS bytes_out, "
+        "COALESCE(SUM(item_count), 0) AS items, "
+        "COALESCE(AVG(CASE WHEN status = :done AND duration_ms > 0 "
+        "THEN duration_ms END), 0) AS avg_ms "
+        "FROM jobs WHERE kind = :kind GROUP BY status",
+        kind=kind,
+        done=DONE,
+    ):
+        status = row["status"] or ""
+        count = int(row["n"] or 0)
+        metrics["total"] += count
+        metrics["by_status"][status] = metrics["by_status"].get(status, 0) + count
+        metrics["bytes_in"] += int(row["bytes_in"] or 0)
+        metrics["bytes_out"] += int(row["bytes_out"] or 0)
+        metrics["items"] += int(row["items"] or 0)
+        if status == DONE:
+            metrics["avg_ms"] = int(row["avg_ms"] or 0)
+    metrics["recent"] = list(
+        db.query(
+            "SELECT uid, status, preferred_name, bytes_out, item_count "
+            "FROM jobs WHERE kind = :kind ORDER BY id DESC LIMIT 10",
+            kind=kind,
+        )
+    )
+    return metrics
+
+
 def list_jobs(kind: str = None, status: str = None, owner: tuple = None) -> list[dict]:
     if "jobs" not in db.tables:
         return []

devplacepy/services/messaging/persist.py

@@ -37,7 +37,8 @@ def _slim_attachment(attachment: dict[str, Any]) -> dict[str, Any]:
 
 
 def message_frame(
-    message: dict[str, Any], sender_username: str, client_id: Optional[str] = None
+    message: dict[str, Any], sender_username: str, client_id: Optional[str] = None,
+    sender_role: Optional[str] = None,
 ) -> dict[str, Any]:
     attachments = get_attachments("message", message["uid"])
     return {
@@ -45,6 +46,7 @@ def message_frame(
         "uid": message["uid"],
         "sender_uid": message["sender_uid"],
         "sender_username": sender_username,
+        "sender_role": sender_role,
         "receiver_uid": message["receiver_uid"],
         "content": message["content"],
         "created_at": message["created_at"],

devplacepy/services/messaging/relay.py

@@ -72,7 +72,7 @@ class MessageRelay:
         senders = get_users_by_uids(list(sender_uids)) if sender_uids else {}
         for row in pending:
             sender = senders.get(row["sender_uid"]) or {}
-            frame = message_frame(dict(row), sender.get("username", ""))
+            frame = message_frame(dict(row), sender.get("username", ""), sender_role=sender.get("role"))
             message_hub.mark_delivered(row["uid"])
             await message_hub.send_to_users(
                 [row["sender_uid"], row["receiver_uid"]], frame

devplacepy/services/news/service.py

@@ -594,6 +594,7 @@ class NewsService(BaseService):
 
         headers = {
             "Content-Type": "application/json",
+            "X-App-Reference": "devplace-news-v-1-0-0",
         }
         ai_key = _get_ai_key()
         if ai_key:
@@ -646,7 +647,7 @@ class NewsService(BaseService):
             "temperature": 0.3,
         }
 
-        headers = {"Content-Type": "application/json"}
+        headers = {"Content-Type": "application/json", "X-App-Reference": "devplace-news-v-1-0-0"}
         ai_key = _get_ai_key()
         if ai_key:
             headers["Authorization"] = f"Bearer {ai_key}"

devplacepy/services/openai_gateway/CLAUDE.md

@@ -4,7 +4,7 @@ This file documents the AI gateway subsystem (devplacepy/services/openai_gateway
 
 `GatewayService` (`services/openai_gateway/`) is an OpenAI-compatible LLM gateway (the ported `openai5.py`), mounted at `/openai/v1/*` (`routers/openai_gateway.py`, prefix `/openai`). It is a **service that serves an HTTP endpoint** rather than a loop.
 
-- The router is thin: it calls `service_manager.get_service("openai").handle(request, subpath)`. `POST /v1/chat/completions` runs the full gateway (vision augment -> model override -> forward -> optional SSE re-emit via `_fake_stream`); `POST /v1/embeddings` forwards to the configured embeddings upstream (`handle_embeddings`, no vision/streaming); `/v1/{path:path}` is a transparent passthrough to the upstream base. Disabled -> 503, unauthorized -> 401, upstream connection failure -> 502.
+- The router is thin: it calls `service_manager.get_service("openai").handle(request, subpath)`. `POST /v1/chat/completions` runs the full gateway (vision augment -> model override -> forward -> optional SSE re-emit via `_fake_stream`); `POST /v1/embeddings` forwards to the configured embeddings upstream (`handle_embeddings`, no vision/streaming); `GET /v1/models` is answered locally from the `gateway_models` chat routes (`_models_response`, publishing the public `molodetz`/`molodetz-pro` names), NOT proxied upstream; `/v1/{path:path}` is a transparent passthrough to the upstream base. **The gateway always forwards NON-streaming upstream** (`payload["stream"] = False`), then re-emits SSE itself when the client asked for `stream`; because `stream_options` is only valid alongside `stream=true`, `handle_chat` strips `stream_options` from the upstream payload (otherwise DeepSeek rejects it with `stream_options should be set along with stream = true`) and, when the client requested `stream_options.include_usage`, `_fake_stream` appends a final `choices: []` usage chunk built from the upstream `usage` before `[DONE]`. Disabled -> 503, unauthorized -> 401, upstream connection failure -> 502.
 - `main.py` registers it and exempts `/openai` from the rate-limit middleware and the maintenance gate. No new dependency (`httpx` already required).
 - `GatewayService` is `default_enabled=True` so internal callers work out of the box.
 
@@ -12,6 +12,15 @@ This file documents the AI gateway subsystem (devplacepy/services/openai_gateway
 
 Every gateway response (chat, embeddings, passthrough; success and error) carries `X-Gateway-*` headers describing that single call: `Model`, `Backend`, `Prompt-Tokens`, `Completion-Tokens`, `Total-Tokens`, `Cache-Hit-Tokens`, `Cache-Miss-Tokens`, `Reasoning-Tokens`, `Cost-USD`/`Input-Cost-USD`/`Output-Cost-USD` (dollars), `Cost-Native` (1 if the upstream returned a native cost), `Tokens-Per-Second`, `Upstream-Latency-Ms`, and `Context-Window`/`Context-Utilization` when the model's window is known, plus the timing headers `X-Gateway-Upstream-Latency-Ms`/`X-Gateway-Total-Latency-Ms`. `GatewayUsageLedger.record(...)` RETURNS the computed ledger row (or `None` on failure); each handler's `finalize` closure maps it through `usage.usage_response_headers(row)` and attaches it to the returned `Response` (`resp_headers`). The single denied path with no upstream call (embeddings disabled) carries no headers. This is what lets any caller read its own spend - the AI correction worker reads `X-Gateway-Cost-USD`/token headers off its own correction call to accumulate per-user totals (see "AI content correction" in the root `CLAUDE.md`).
 
+## App-reference header (`X-App-Reference`)
+
+Callers may send an optional `X-App-Reference` header to tag gateway calls by application. The value is validated and stored in the `app_reference` column of `gateway_usage_ledger`, surfaced in analytics and admin reporting.
+
+- **Validation** (`service._validate_app_reference`): trimmed whitespace, then matched against `^[a-zA-Z0-9_.-]{1,30}$`. Any value failing validation (empty, >30 chars, contains spaces or `@`/`/` etc.) silently falls back to `"default"`.
+- **Header name:** `X-App-Reference`.
+- **All internal callers** (news, bots, Devii, correction, jobs, deepsearch, dbapi, gitea) should pass a `devplace-<component>-v-<major>-<minor>-<patch>` value, e.g. `devplace-devii-v-1-0-0`, `devplace-news-v-1-0-0`.
+- The column is indexed (`CREATE INDEX IF NOT EXISTS`) for fast per-app queries.
+
 ## Per-worker runtime
 
 **Serves in every worker.** Config/enabled come from `site_settings` (via the service's `get_config()`/`is_enabled()`), so any uvicorn worker answers - not just the supervisor worker. Per-worker runtime (`GatewayRuntime`: `httpx.AsyncClient` pool + `asyncio.Semaphore` sized to `gateway_instances`, plus counters and a `VisionCache`) is created lazily on first request and rebuilt when `instances`/`timeout`/cache size change. `gateway_instances` is the **scaling knob** (concurrency per worker); process scaling is uvicorn workers.
@@ -83,7 +92,7 @@ Layered ON TOP of the single-provider service config above, which stays THE impl
 
 **Resolution is a per-request overlay, not a fork.** At request time `routing.chat_overlay(requested, cfg)` / `routing.embed_overlay(requested, cfg)` / `routing.image_overlay(requested, cfg)` resolve an active route by the requested model name and return a per-request OVERLAY dict of `gateway_*` cfg keys (`gateway_force_model`+`gateway_model`=target, `gateway_upstream_url`/`gateway_api_key` from the provider, the price keys, an augmented `gateway_model_context_map`, and vision overlay keys); `handle_chat`/`handle_embeddings` merge it onto the base cfg (`cfg = {**cfg, **overlay}`) BEFORE everything else, so the existing model-selection / `pricing_from_cfg` / `parse_context_map` / vision / url+key paths transparently use the route's provider, target model, pricing, vision model and context window. `_ensure` (the httpx pool / semaphore / breaker / vision cache) reads only the non-overlaid pool keys, so the connection pool is never churned per request.
 
-**No matching route = `None` overlay = byte-identical legacy behavior** - this is the "nobody feels the transformation" guarantee: `molodetz`/`molodetz~embed`/`molodetz-img-small` and every existing caller are byte-identical when no route matches. A route with a blank provider overlays only model+pricing(+vision), keeping the default upstream url/key.
+**No matching route = None overlay = model fallback.** When no route matches, the overlay returns `None` and the handler's model selection logic falls back to the default configured model (`gateway_model` for chat, `gateway_embed_model` for embeddings, `gateway_image_model` for images). The unknown model name is discarded, not forwarded upstream. This means an unknown or misspelled model name never causes a 4xx from the upstream - it is gracefully downgraded to the default. The `force_model` guard and the built-in `molodetz`/`molodetz~embed`/`molodetz-img-small` aliases are still respected before the route check: `force_model` or empty/named-alias model -> default directly; known model -> route resolution; unknown model with no route -> default fallback with a log message.
 
 **CRUD.** Admin JSON at `/admin/gateway/{providers,models}` (`routers/admin/gateway_configs.py`, `require_admin`, Pydantic `ProviderIn`/`ModelRouteIn` validation, accepts both JSON from `static/js/GatewayAdmin.js` and form from Devii), audited under `gateway.provider.*`/`gateway.model.*` (category `ai`), included in the admin package with the page at `/admin/gateway` (`templates/admin_gateway.html`, sidebar link, `admin_section="gateway"`).
 
@@ -101,4 +110,4 @@ Real providers sometimes charge more than a flat per-1M rate for one component:
 - **Overlay + computation.** `chat_overlay`/`embed_overlay` propagate all of these into the per-request cfg dict under `gateway_*` keys exactly like the existing price fields (section above); `usage.pricing_from_cfg` reads them generically (defaulting to `None`/`0`/disabled when absent), so **Layer A (the single global flat Pricing config fields) never gains these dimensions** - only a `gateway_models` route can enable them, preserving the "no matching route = byte-identical legacy behavior" guarantee. `usage.compute_cost` selects tier1 vs tier2 per rate component (`_tiered_rate`) based on whether `norm["prompt"]` (billable input tokens) exceeds the threshold, then applies the off-peak discount (`_effective_rate`/`_off_peak_active`, UTC wraparound-aware) to whichever rate was selected. **The response header format and `compute_cost`'s return shape (`total, input_cost, output_cost, native`) are unchanged** - this is purely an internal rate-selection step before the existing input/output split math runs; a native upstream `cost` (OpenRouter) still overrides the modeled total exactly as before.
 - **Migration.** New `gateway_models` columns are added via `has_column`/`create_column_by_example` in `routing.ensure_tables()` (the `CREATE TABLE IF NOT EXISTS` DDL string alone would never reach a pre-existing table - see the `database/CLAUDE.md` column-ensure idiom).
 - **Admin UI.** `/admin/gateway`'s model-route form has a "Tiered / off-peak pricing (optional)" subsection; off-peak start/end render as `<input type="time">` (converted to/from UTC minutes-of-day by `GatewayAdmin.js`), and the routes table shows `tiered`/`off-peak` badges when a route has either dimension configured.
-- **DeepSeek's real pricing is already the tier-1 shape, not a new dimension.** DeepSeek's actual API (verified against `api-docs.deepseek.com/quick_start/pricing`) bills three flat per-1M rates - cache-hit input, cache-miss input, output - with no current context-length tier or off-peak window for the V4 models; that shape was already fully modeled by the pre-existing `chat_cache_hit_per_m`/`chat_cache_miss_per_m`/`chat_output_per_m` fields before this section's tier2/off-peak fields existed. `routing.seed_default_deepseek_routes()` (called once from `database.migrate_ai_gateway_settings()` at the end of `init_db()`) idempotently inserts two ready-made routes - `deepseek-v4-flash` (`$0.0028`/`$0.14`/`$0.28` per 1M, 1M context) and `deepseek-v4-pro` (`$0.003625`/`$0.435`/`$0.87` per 1M, 1M context) - only when that `source_model` row does not already exist, so a caller or Devii can request either name explicitly and get correctly-priced, decoupled from whatever the single global `gateway_model` default happens to be set to (switching that global setting between the two real models does NOT retroactively fix the flat Pricing config fields - the seeded routes are the model-agnostic, always-correct way to reference a specific priced model). Neither seeded route sets the tier2/off-peak fields (DeepSeek does not use them today); an admin can add them later on the same row if DeepSeek (or any other provider routed here) introduces such pricing.
+- **DeepSeek's real pricing is already the tier-1 shape, not a new dimension.** DeepSeek's actual API (verified against `api-docs.deepseek.com/quick_start/pricing`) bills three flat per-1M rates - cache-hit input, cache-miss input, output - with no current context-length tier or off-peak window for the V4 models; that shape was already fully modeled by the pre-existing `chat_cache_hit_per_m`/`chat_cache_miss_per_m`/`chat_output_per_m` fields before this section's tier2/off-peak fields existed. `routing.seed_default_deepseek_routes()` (called once from `database.migrate_ai_gateway_settings()` at the end of `init_db()`) idempotently inserts four ready-made routes - `deepseek-v4-flash` (`$0.0028`/`$0.14`/`$0.28` per 1M, 1M context), `deepseek-v4-pro` (`$0.003625`/`$0.435`/`$0.87` per 1M, 1M context), and the two public `molodetz` aliases `molodetz` -> `deepseek-v4-flash` (flash rates) and `molodetz-pro` -> `deepseek-v4-pro` (pro rates) - only when that `source_model` row does not already exist, so a caller or Devii can request any name explicitly and get correctly-priced, decoupled from whatever the single global `gateway_model` default happens to be set to (switching that global setting between the two real models does NOT retroactively fix the flat Pricing config fields - the seeded routes are the model-agnostic, always-correct way to reference a specific priced model). The `molodetz`/`molodetz-pro` aliases are the public model names; `GET /v1/models` is served locally from these `source_model` rows (not proxied upstream), so it publishes exactly the models the gateway accepts. Neither seeded route sets the tier2/off-peak fields (DeepSeek does not use them today); an admin can add them later on the same row if DeepSeek (or any other provider routed here) introduces such pricing.

devplacepy/services/openai_gateway/gateway.py

@@ -33,7 +33,7 @@ from devplacepy.services.openai_gateway.vision import VisionAugmenter, VisionCac
 logger = logging.getLogger(__name__)
 
 
-def _fake_stream(data: dict, model: str):
+def _fake_stream(data: dict, model: str, include_usage: bool = False):
     chunk_id = data.get("id", f"chatcmpl-{uuid.uuid4().hex[:12]}")
     created = data.get("created", int(time.time()))
     out_model = data.get("model", model)
@@ -72,6 +72,21 @@ def _fake_stream(data: dict, model: str):
             for i in range(0, len(content), 50):
                 yield _chunk({"content": content[i : i + 50]})
         yield _chunk({}, finish="tool_calls" if tool_calls else "stop")
+        if include_usage and data.get("usage"):
+            yield (
+                "data: "
+                + json.dumps(
+                    {
+                        "id": chunk_id,
+                        "object": "chat.completion.chunk",
+                        "created": created,
+                        "model": out_model,
+                        "choices": [],
+                        "usage": data["usage"],
+                    }
+                )
+                + "\n\n"
+            )
         yield "data: [DONE]\n\n"
 
     return gen()
@@ -217,7 +232,7 @@ class GatewayRuntime:
         return resp, None, timing
 
     async def handle_chat(
-        self, body: dict, cfg: dict, owner: tuple, user_agent: str, log=None
+        self, body: dict, cfg: dict, owner: tuple, user_agent: str, app_reference: str, log=None
     ):
         log = log or (lambda message: None)
         overlay = chat_overlay(body.get("model"), cfg)
@@ -242,6 +257,7 @@ class GatewayRuntime:
                 owner=owner,
                 pricing=pricing,
                 context_map=context_map,
+                app_reference=app_reference,
             )
             messages = await augmenter.augment_messages(client, messages)
             self.vision_calls += augmenter.calls
@@ -252,14 +268,19 @@ class GatewayRuntime:
         requested = body.get("model")
         if cfg["gateway_force_model"] or not requested or requested == "molodetz":
             model = cfg["gateway_model"]
-        else:
+        elif overlay is not None:
             model = requested
+        else:
+            model = cfg["gateway_model"]
+            log(f"requested model {requested!r} has no route, falling back to {model!r}")
 
         stream = bool(body.get("stream"))
+        include_usage = bool((body.get("stream_options") or {}).get("include_usage"))
         payload = dict(body)
         payload["model"] = model
         payload["messages"] = messages
         payload["stream"] = False
+        payload.pop("stream_options", None)
 
         headers = {"Content-Type": "application/json"}
         if cfg["gateway_api_key"]:
@@ -287,6 +308,7 @@ class GatewayRuntime:
             "endpoint": "chat/completions",
             "model": model,
             "user_agent": user_agent,
+            "app_reference": app_reference,
             **params,
             **timing,
         }
@@ -372,14 +394,18 @@ class GatewayRuntime:
         log(f"chat POST -> 200 ({timing['upstream_latency_ms']:.0f}ms)")
         if stream:
             return StreamingResponse(
-                _fake_stream(data, model),
+                _fake_stream(data, model, include_usage),
                 media_type="text/event-stream",
                 headers=resp_headers,
             )
-        return JSONResponse(content=data, headers=resp_headers)
+        return Response(
+            content=resp.content,
+            media_type="application/json",
+            headers=resp_headers,
+        )
 
     async def handle_embeddings(
-        self, body: dict, cfg: dict, owner: tuple, user_agent: str, log=None
+        self, body: dict, cfg: dict, owner: tuple, user_agent: str, app_reference: str, log=None
     ):
         log = log or (lambda message: None)
         vision_cost = 0.0
@@ -425,8 +451,11 @@ class GatewayRuntime:
         requested = body.get("model")
         if cfg["gateway_force_model"] or not requested or requested == "molodetz~embed":
             model = cfg["gateway_embed_model"]
-        else:
+        elif overlay is not None:
             model = requested
+        else:
+            model = cfg["gateway_embed_model"]
+            log(f"requested embed model {requested!r} has no route, falling back to {model!r}")
 
         payload = dict(body)
         payload["model"] = model
@@ -457,6 +486,7 @@ class GatewayRuntime:
             "endpoint": "embeddings",
             "model": model,
             "user_agent": user_agent,
+            "app_reference": app_reference,
             **params,
             **timing,
         }
@@ -544,7 +574,7 @@ class GatewayRuntime:
         return JSONResponse(content=data, headers=resp_headers)
 
     async def handle_images(
-        self, body: dict, cfg: dict, owner: tuple, user_agent: str, log=None
+        self, body: dict, cfg: dict, owner: tuple, user_agent: str, app_reference: str, log=None
     ):
         log = log or (lambda message: None)
         overlay = image_overlay(body.get("model"), cfg)
@@ -595,8 +625,11 @@ class GatewayRuntime:
             or requested in ("molodetz-img-small", "molodetz-img")
         ):
             model = cfg["gateway_image_model"]
-        else:
+        elif overlay is not None:
             model = requested
+        else:
+            model = cfg["gateway_image_model"]
+            log(f"requested image model {requested!r} has no route, falling back to {model!r}")
 
         payload = dict(body)
         payload["model"] = model
@@ -627,6 +660,7 @@ class GatewayRuntime:
             "endpoint": "images/generations",
             "model": model,
             "user_agent": user_agent,
+            "app_reference": app_reference,
             **params,
             **timing,
         }
@@ -717,6 +751,7 @@ class GatewayRuntime:
         cfg: dict,
         owner: tuple,
         user_agent: str,
+        app_reference: str,
         log=None,
     ):
         log = log or (lambda message: None)
@@ -746,6 +781,7 @@ class GatewayRuntime:
             "endpoint": subpath,
             "model": cfg["gateway_model"],
             "user_agent": user_agent,
+            "app_reference": app_reference,
             **timing,
         }
 

devplacepy/services/openai_gateway/routing.py

@@ -419,6 +419,22 @@ DEEPSEEK_DEFAULT_ROUTES = (
         "price_cache_miss_per_m": 0.435,
         "price_output_per_m": 0.87,
     },
+    {
+        "source_model": "molodetz",
+        "target_model": "deepseek-v4-flash",
+        "context_window": 1_048_576,
+        "price_cache_hit_per_m": 0.0028,
+        "price_cache_miss_per_m": 0.14,
+        "price_output_per_m": 0.28,
+    },
+    {
+        "source_model": "molodetz-pro",
+        "target_model": "deepseek-v4-pro",
+        "context_window": 1_048_576,
+        "price_cache_hit_per_m": 0.003625,
+        "price_cache_miss_per_m": 0.435,
+        "price_output_per_m": 0.87,
+    },
 )
 
 

devplacepy/services/openai_gateway/service.py

@@ -3,18 +3,32 @@
 import json
 import logging
 import os
+import re
+import time
 
 from fastapi import HTTPException, Request
+from fastapi.responses import JSONResponse
 
 from devplacepy.database import get_int_setting
 from devplacepy.services.base import BaseService, ConfigField
 from devplacepy.services.openai_gateway import config
 from devplacepy.services.openai_gateway.analytics import summary_metrics
 from devplacepy.services.openai_gateway.gateway import GatewayRuntime
+from devplacepy.services.openai_gateway.routing import model_store
 from devplacepy.utils import get_current_user
 
 logger = logging.getLogger(__name__)
 
+APP_REFERENCE_PATTERN = re.compile(r"^[a-zA-Z0-9_.-]{1,30}$")
+DEFAULT_APP_REFERENCE = "default"
+
+
+def _validate_app_reference(value: str) -> str:
+    stripped = (value or "").strip()
+    if not stripped or not APP_REFERENCE_PATTERN.match(stripped):
+        return DEFAULT_APP_REFERENCE
+    return stripped
+
 
 def _presented_key(request: Request) -> str:
     key = request.headers.get("X-API-KEY")
@@ -449,6 +463,29 @@ class GatewayService(BaseService):
             return (kind, user.get("uid") or "unknown")
         return ("anonymous", "anonymous")
 
+    def _models_response(self) -> JSONResponse:
+        created = int(time.time())
+        seen: set = set()
+        data = []
+        for row in model_store.list():
+            if str(row.get("kind") or "chat") != "chat":
+                continue
+            if not row.get("is_active", True):
+                continue
+            source = str(row.get("source_model") or "").strip()
+            if not source or source in seen:
+                continue
+            seen.add(source)
+            data.append(
+                {
+                    "id": source,
+                    "object": "model",
+                    "created": created,
+                    "owned_by": "molodetz",
+                }
+            )
+        return JSONResponse({"object": "list", "data": data})
+
     async def handle(self, request: Request, subpath: str):
         if not self.is_enabled():
             raise HTTPException(status_code=503, detail="Gateway is disabled")
@@ -459,6 +496,11 @@ class GatewayService(BaseService):
         runtime = self.runtime()
         owner = self.resolve_owner(request)
         user_agent = request.headers.get("user-agent", "")
+        app_reference = _validate_app_reference(
+            request.headers.get("X-App-Reference", DEFAULT_APP_REFERENCE)
+        )
+        if subpath == "models" and request.method == "GET":
+            return self._models_response()
         if subpath == "chat/completions" and request.method == "POST":
             try:
                 body = await request.json()
@@ -468,7 +510,7 @@ class GatewayService(BaseService):
             if not isinstance(body, dict):
                 self.log("Rejected chat request: JSON body was not an object")
                 raise HTTPException(status_code=400, detail="Invalid JSON body")
-            return await runtime.handle_chat(body, cfg, owner, user_agent, self.log)
+            return await runtime.handle_chat(body, cfg, owner, user_agent, app_reference, self.log)
         if subpath == "embeddings" and request.method == "POST":
             try:
                 body = await request.json()
@@ -479,7 +521,7 @@ class GatewayService(BaseService):
                 self.log("Rejected embeddings request: JSON body was not an object")
                 raise HTTPException(status_code=400, detail="Invalid JSON body")
             return await runtime.handle_embeddings(
-                body, cfg, owner, user_agent, self.log
+                body, cfg, owner, user_agent, app_reference, self.log
             )
         if subpath == "images/generations" and request.method == "POST":
             try:
@@ -491,7 +533,7 @@ class GatewayService(BaseService):
                 self.log("Rejected images request: JSON body was not an object")
                 raise HTTPException(status_code=400, detail="Invalid JSON body")
             return await runtime.handle_images(
-                body, cfg, owner, user_agent, self.log
+                body, cfg, owner, user_agent, app_reference, self.log
             )
         body = await request.body()
         content_type = request.headers.get("content-type", "")
@@ -503,6 +545,7 @@ class GatewayService(BaseService):
             cfg,
             owner,
             user_agent,
+            app_reference,
             self.log,
         )
 

devplacepy/services/openai_gateway/usage.py

@@ -404,6 +404,7 @@ def usage_response_headers(row: Optional[dict]) -> dict:
         headers["X-Gateway-Context-Window"] = str(int(row["context_window"]))
     if row.get("context_utilization") is not None:
         headers["X-Gateway-Context-Utilization"] = str(row["context_utilization"])
+    headers["X-App-Reference"] = str(row.get("app_reference") or "default")
     return headers
 
 
@@ -531,6 +532,7 @@ class GatewayUsageLedger:
                 "retry_succeeded": 1 if raw.get("retry_succeeded") else 0,
                 "circuit_open": 1 if raw.get("circuit_open") else 0,
                 "user_agent": (raw.get("user_agent") or "")[:300],
+                "app_reference": (raw.get("app_reference") or "default")[:30],
             }
             get_table(GATEWAY_LEDGER).insert(row)
             self._audit(raw, norm, cost_usd)
@@ -551,6 +553,7 @@ class GatewayUsageLedger:
         success: bool,
         status_code: int,
         latency_ms: float = 0.0,
+        app_reference: str = "default",
     ) -> Optional[dict]:
         try:
             row = {
@@ -591,6 +594,7 @@ class GatewayUsageLedger:
                 "retry_succeeded": 0,
                 "circuit_open": 0,
                 "user_agent": "",
+                "app_reference": app_reference or "default",
             }
             get_table(GATEWAY_LEDGER).insert(row)
             self._audit_external(row)
@@ -695,6 +699,7 @@ def record_rsearch_call(
             cost_usd=cost,
             success=success,
             status_code=status_code,
+            app_reference="devplace-devii-rsearch-v-1-0-0",
         )
     except Exception as exc:
         logger.warning("rsearch usage ledger failed: %s", exc)

devplacepy/services/openai_gateway/vision.py

@@ -94,6 +94,7 @@ class VisionAugmenter:
         owner: tuple = ("unknown", "unknown"),
         pricing=None,
         context_map=None,
+        app_reference: str = "default",
     ):
         self.vision_url = vision_url
         self.vision_model = vision_model
@@ -105,6 +106,7 @@ class VisionAugmenter:
         self.owner = owner
         self.pricing = pricing
         self.context_map = context_map or {}
+        self.app_reference = app_reference
         self.calls = 0
         self.cost_usd = 0.0
 
@@ -126,6 +128,7 @@ class VisionAugmenter:
                 "success": success,
                 "error_category": category,
                 "usage": usage,
+                "app_reference": self.app_reference,
             },
             self.pricing,
             self.context_map,