Add personal notes, DeepSearch history, backup offload, and gateway auth throttling

Also streamline the top navigation: drop the Tools dropdown, make Quizzes
and Battles icon-only entries, and remove the Workspace, Containers and
Editor entry points from the project detail page.

.coveragerc

@@ -1,7 +1,7 @@
 # retoor <retoor@molodetz.nl>
 [run]
 source = devplacepy
-parallel = true
+parallel = false
 sigterm = true
 omit =
     tests/*

.env.example

@@ -29,9 +29,16 @@ SECRET_KEY=change-me
 # from the request.
 DEVPLACE_SITE_URL=
 
-# Host port the nginx front door binds.
+# Host port the nginx front door binds (Docker only - the app container's own
+# internal port stays fixed).
 PORT=10500
 
+# Port the uvicorn process itself binds to for `make dev`/`make prod` (bare
+# metal, no Docker/nginx in front). Also what the app calls itself on
+# internally (DEVII_BASE_URL default, INTERNAL_GATEWAY_URL). Unrelated to
+# PORT above - leave unset unless running bare metal on a non-default port.
+# DEVPLACE_PORT=10500
+
 # nginx upload ceiling. Must be >= the admin-configurable max_upload_size_mb.
 NGINX_MAX_BODY_SIZE=50m
 
@@ -42,3 +49,13 @@ NGINX_CACHE_MAX_SIZE=1g
 # Run the app container as this host user so shared files keep dev ownership.
 DEVPLACE_UID=1000
 DEVPLACE_GID=1000
+
+# OpenCode Zen client identity (devplacepy/services/openai_gateway/opencode_zen.py).
+# Optional: both already default to these exact values, which match the real
+# opencode CLI's own headers - only override if opencode ships a new version
+# and Zen starts rejecting the old one. The provider's base URL, API key, and
+# which model(s) route to it are NOT set here - those live in the
+# gateway_providers/gateway_models tables, configured at /admin/gateway
+# (provider client profile "OpenCode Zen").
+# OPENCODE_CLIENT_NAME=cli
+# OPENCODE_CLIENT_USER_AGENT=opencode/1.18.29 ai-sdk/provider-utils/4.0.23 runtime/bun/1.3.15

.githooks/pre-commit

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+# retoor <retoor@molodetz.nl>
+
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(
+    subprocess.run(
+        ["git", "rev-parse", "--show-toplevel"],
+        capture_output=True,
+        text=True,
+        check=True,
+    ).stdout.strip()
+)
+PYPROJECT = REPO_ROOT / "pyproject.toml"
+VERSION_LINE = re.compile(r'^version = "(\d+)\.(\d+)\.(\d+)"$', re.MULTILINE)
+STAGED_VERSION_CHANGE = re.compile(r'^[+-]version = "\d+\.\d+\.\d+"$', re.MULTILINE)
+
+
+def staged_diff(path: Path) -> str:
+    result = subprocess.run(
+        ["git", "diff", "--cached", "--unified=0", "--", str(path)],
+        capture_output=True,
+        text=True,
+        cwd=REPO_ROOT,
+    )
+    return result.stdout
+
+
+def main() -> int:
+    if (REPO_ROOT / ".git" / "MERGE_HEAD").exists():
+        return 0
+    if not PYPROJECT.is_file():
+        return 0
+    if STAGED_VERSION_CHANGE.search(staged_diff(PYPROJECT)):
+        return 0
+    text = PYPROJECT.read_text()
+    match = VERSION_LINE.search(text)
+    if not match:
+        return 0
+    major, minor, patch = (int(part) for part in match.groups())
+    bumped = f'version = "{major}.{minor}.{patch + 1}"'
+    PYPROJECT.write_text(VERSION_LINE.sub(bumped, text, count=1))
+    subprocess.run(["git", "add", str(PYPROJECT)], cwd=REPO_ROOT, check=True)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.gitignore

@@ -1,3 +1,6 @@
+.dpc
+.devplace
+dpc.log
 .cache
 .local
 .devplace_bots/
@@ -13,6 +16,10 @@ devplace-init.lock
 notification-private.pem
 notification-private.pkcs8.pem
 notification-public.pem
+# HOME=/app in the Docker app container, so an rclone.conf created via
+# `rclone config` or DEVPLACE_RCLONE_CONFIG's default lands here on the
+# bind-mounted host tree - it holds live remote-storage credentials.
+/.config/
 .pytest_cache/
 .ruff_cache/
 .opencode

CLAUDE.md

@@ -19,7 +19,7 @@ DevPlace is a server-rendered social network for developers. FastAPI backend ser
 ## Commands
 
 ```bash
-make install        # pip install -e . + playwright install chromium
+make install        # create .venv if needed, pip install -e ".[dev]", playwright chromium
 make ppy            # build the single shared container image (ppy:latest); run once before launching instances
 make dev            # uvicorn --reload on port 10500, backlog 4096
 make prod           # uvicorn --workers $(WEB_WORKERS) (defaults to nproc), port 10500 (backlog 8192)
@@ -33,6 +33,8 @@ make locust         # Locust load test, interactive web UI
 make locust-headless  # Locust CLI mode for CI
 ```
 
+Every Python make target (`install`, `dev`, `prod`, `test*`, `coverage*`, `locust*`) uses `.venv/bin/python`. If `.venv` is missing, make creates it from `python3`, installs `-e ".[dev]"`, and installs Playwright Chromium before running the target. `make install` refreshes that environment. `make clean` removes `.venv` as well as stray bytecode.
+
 The Makefile exports `PYTHONDONTWRITEBYTECODE=1` for every recipe, so no `.pyc` files or `__pycache__` directories are written by any `make` target. Keep it that way - do not add a target that re-enables bytecode writing. `make clean` removes any stray bytecode left from running Python outside make.
 
 Preliminary validation: confirm `python -c "from devplacepy.main import app"` imports clean and check each touched language manually (Python compiles/imports, JS parses, CSS braces and HTML tags balance). These checks are gates on the way to the real validation, never a substitute for it: **every change ends with the full test suite (`make test` - all three tiers, every test) and it must pass.**
@@ -56,6 +58,7 @@ devplace token prune                # soft-delete all expired access tokens
 devplace news clear         # delete all news rows
 devplace news sanitize      # strip HTML from news descriptions/content
 devplace attachments prune  # remove orphan attachment records/files
+devplace system prune [--dry-run]  # safe but aggressive: purges soft-deleted attachment/project-file blobs, sweeps blob files with zero DB reference at all (e.g. left by an interrupted/racing sync), and GCs orphaned container workspace dirs
 devplace devii reset-quota <username>   # reset one user's rolling 24h AI quota
 devplace devii reset-quota --guests     # reset every guest quota
 devplace devii reset-quota --all        # reset every quota (users and guests)
@@ -114,6 +117,7 @@ Tests run on port 10501 with a tempfile SQLite DB and a dedicated `DEVPLACE_DATA
 | Var | Default | Purpose |
 |-----|---------|---------|
 | `DEVPLACE_DATABASE_URL` | `sqlite:///<repo>/data/devplace.db` | Override DB path (tests use this) |
+| `DEVPLACE_PORT` | `10500` | Port the app itself binds to bare-metal (`make dev`/`make prod`, wired to uvicorn's `--port`; tests use their own `DEVPLACE_TEST_PORT`, default `10501`). Also `config.PORT`'s source, so `DEVPLACE_INTERNAL_BASE_URL`'s default and Devii's self-dial URL (`DEFAULT_BASE_URL`/`INSTANCE_ORIGIN_DEFAULT`) follow it automatically. The Docker app container pins it to `10500` in `docker-compose.yml` regardless of `.env` - Docker's externally reachable port is the unrelated `PORT` var (nginx's host mapping), never this one. |
 | `SECRET_KEY` | hardcoded fallback | Session signing |
 | `DEVPLACE_DISABLE_SERVICES` | unset | When `1`, NewsService and other background services skip start (set by test conftest) |
 | `PLAYWRIGHT_HEADLESS` | `1` in tests | Toggle headed mode |
@@ -143,7 +147,7 @@ Nested `CLAUDE.md` files (loaded automatically by Claude Code only when a file i
 | `devplacepy/services/moderation/CLAUDE.md` | Trust and safety: the reportable-target registry, the content filter and its five choke points, the report queue and its atomic resolution, enforcement, consent, maturity, account deletion |
 | `devplacepy/services/acceptance/CLAUDE.md` | Acceptance convergence: the opt-in service that grants every policy agreement to every account that has not declined it, its invisibility contract and the ledger-as-decline-register rule |
 | `devplacepy/services/audit/CLAUDE.md` | Audit log: recorders, categories, retention |
-| `devplacepy/services/backup/CLAUDE.md` | Backup service: targets, worker, schedules, primary-admin-only download |
+| `devplacepy/services/backup/CLAUDE.md` | Backup service: targets, worker, schedules, primary-admin-only download, remote offload to Hetzner Storage Box |
 | `devplacepy/services/telegram/CLAUDE.md` | Telegram bot bridge |
 | `devplacepy/services/email/CLAUDE.md` | Devii IMAP/SMTP email tools |
 | `devplacepy/services/gitea/CLAUDE.md` | Issue tracker (Gitea-backed, no local issue store) |
@@ -183,6 +187,7 @@ Routers in `devplacepy/routers/` are organised as a **directory tree that mirror
 |--------|--------|
 | `/auth` | auth/ package |
 | `/feed`, `/posts`, `/comments` | flat files |
+| `/topics` | topics.py - crawlable per-topic category index pages (`/topics` hub, `/topics/{topic}` listing) |
 | `/projects`, `/projects/{slug}/files`, `/projects/{slug}/containers` | projects/ package - see `routers/projects/CLAUDE.md` |
 | `/profile` | profile/ package (customization, notifications, ai-correction, ai-modifier, interactions, telegram, usage) |
 | `/messages` | messages.py - see `services/messaging/CLAUDE.md` |
@@ -258,7 +263,7 @@ Users and guests inject their own CSS and JS, scoped to a page type or globally,
 
 ### Container manager, Devii assistant, AI gateway, async jobs, audit log
 
-DevPlace ships a Docker-backed **Container manager** (admin-only, one shared `ppy:latest` image, security-hardened rootless workflow via a `sudo` superclone and `aptroot`), the **Devii** in-platform agentic assistant (WebSocket terminal, per-owner-channel sessions, scheduler/reminders, self-learning lessons, user-defined virtual tools, self-configured behavior, browser-automation client tools), the **AI gateway** (`/openai/v1/*`, single point of truth for every AI call, per-user cost attribution, provider/model routing overlay), **async job services** (`JobService` base pattern: zip, fork, SEO diagnostics, SEO metadata generation, DeepSearch, AI Usage Analyzer), and an admin-only append-only **audit log** (`record`/`record_system` entrypoints, `events.md` catalogue of 288 keys, never raises into the caller). These are among the largest subsystems in the codebase - read their dedicated nested `CLAUDE.md` files (see Subsystem map) before working in any of them; do not assume prior knowledge from this summary.
+DevPlace ships a Docker-backed **Container manager** (admin-only, one shared `ppy:latest` image, security-hardened rootless workflow via a `sudo` superclone and `aptroot`), the **Devii** in-platform agentic assistant (WebSocket terminal, per-owner-channel sessions, scheduler/reminders, self-learning lessons, user-defined virtual tools, self-configured behavior, browser-automation client tools), the **AI gateway** (`/openai/v1/*`, single point of truth for every AI call, per-user cost attribution, provider/model routing overlay), **async job services** (`JobService` base pattern: zip, fork, SEO diagnostics, SEO metadata generation, DeepSearch, AI Usage Analyzer), and an admin-only append-only **audit log** (`record`/`record_system` entrypoints, `events.md` catalogue of every recorded event key, never raises into the caller). These are among the largest subsystems in the codebase - read their dedicated nested `CLAUDE.md` files (see Subsystem map) before working in any of them; do not assume prior knowledge from this summary.
 
 ### Telegram bot, email, devRant compatibility API, issue tracker
 
@@ -282,7 +287,7 @@ The escape hatch is deliberately two-factor and must never be self-served: after
 
 - **No comments, no docstrings in source.** Code is self-documenting.
 - **Forbidden variable name patterns:** `_new`, `_old`, `_temp`, `_v2`, `better_`, `my_`, `the_`.
-- **Outbound HTTP goes through the stealth client only.** Never instantiate a bare `httpx.AsyncClient(...)`/`httpx.Client(...)` for outbound traffic - build it via `devplacepy.stealth.stealth_async_client(...)`/`stealth_sync_client(...)`. The async factory routes bytes through **`curl_cffi` (curl-impersonate, BoringSSL)** behind an `httpx` transport adapter, so TLS JA3/JA4 and the HTTP/2 fingerprint match Chrome 146 byte-for-byte; falls back to a pure-httpx Chrome-aligned transport if `curl_cffi` is missing. Applies to news, gateway upstream, Devii/DeepSearch/SEO fetch, gitea, push, attachments. SSRF-guarded fetches via `net_guard.guarded_async_client(...)` get the fingerprint automatically. Sole exception: the reverse-proxy forwarding core `services/containers/forward.py` (and the three routers that call it - `routers/proxy.py`, `routers/tunnel.py`, the workspace editor route), which relays the user's own headers verbatim and therefore keeps a plain `httpx.AsyncClient`; bolting the Chrome identity onto it would overwrite the very headers it exists to forward. **Load-bearing gotcha:** cleartext `http://` is forced to HTTP/1.1 (`curl_transport.http_version_for`) - the Chrome-impersonation profile defaults to HTTP/2, but a plaintext origin has no TLS/ALPN to negotiate h2 and uvicorn (HTTP/1.1 only) returns `400 "Invalid HTTP request received."` once a cleartext HTTP/2 body crosses ~128KB. Every internal AI consumer calls `INTERNAL_GATEWAY_URL` = `http://localhost:<port>/openai/v1/...`, so without this override a large Devii turn (big system prompt + tool list) fails on every request. `https://` keeps the HTTP/2 default. `stealth_sync_client` stays pure-httpx Chrome-aligned (no curl_cffi sync adapter). **Never bolt a hand-written browser-header dict (UA/`sec-ch-ua`/`Sec-Fetch-*`) onto a `stealth_async_client`/`stealth_sync_client` call** - curl_cffi's impersonation already emits an internally-consistent Chrome 146 identity across TLS, UA, and Client Hints; a caller-supplied header set claiming a different Chrome version or platform creates a UA-vs-TLS-vs-Client-Hints mismatch that is a textbook anti-bot tripwire (this was a real bug in Devii's `fetch_url`/`http_request` tool, fixed by deleting its hardcoded Chrome-131 header block). `stealth.detect_consent_gate(html)` recognizes the common JS-redirect cookie-consent wall pattern (DPG Media's `nu.nl` and sibling sites, and generically any page gated behind a `decodeURIComponent(...)` callback referencing a "privacy gate"/`consent.js`) - reuse it wherever a fetched page's real content sits behind that gate rather than special-casing a domain. `stealth_transport`/`stealth_async_client`/`stealth_sync_client` all accept a `proxy=` override, defaulting to `stealth.configured_proxy_url()` (the admin-editable `outbound_proxy_url` site setting, falling back to `DEVPLACE_OUTBOUND_PROXY_URL` when the setting is unset) - this is the only real fix for a destination that blocks by IP/ASN reputation (e.g. reddit.com returns an identical "blocked by network security" page across every TLS/header fingerprint tried, confirmed by testing curl_cffi's full impersonation list - it is blocking the datacenter ASN outright, not fingerprinting the client).
+- **Outbound HTTP goes through the stealth client only.** Never instantiate a bare `httpx.AsyncClient(...)`/`httpx.Client(...)` for outbound traffic - build it via `devplacepy.stealth.stealth_async_client(...)`/`stealth_sync_client(...)`. The async factory routes bytes through **`curl_cffi` (curl-impersonate, BoringSSL)** behind an `httpx` transport adapter, so TLS JA3/JA4 and the HTTP/2 fingerprint match Chrome 146 byte-for-byte; falls back to a pure-httpx Chrome-aligned transport if `curl_cffi` is missing. Applies to news, gateway upstream, Devii/DeepSearch/SEO fetch, gitea, web push, attachments. SSRF-guarded fetches via `net_guard.guarded_async_client(...)` get the fingerprint automatically. Two exceptions keep a plain `httpx.AsyncClient`: (1) the reverse-proxy forwarding core `services/containers/forward.py` (and the three routers that call it - `routers/proxy.py`, `routers/tunnel.py`, the workspace editor route), which relays the user's own headers verbatim - bolting the Chrome identity onto it would overwrite the very headers it exists to forward; (2) the APNs provider (`push/providers/apns.py` `gateway_client` / `delivery_client`) which talks to Apple's HTTP/2 provider API with `httpx.AsyncClient(http2=True, trust_env=False)` - Chrome impersonation, HTTP/2 PRIORITY frames, HPACK indexing of `:path`/`authorization`, extra browser headers, and the outbound proxy all break or starve that API. Web Push stays on stealth. **Load-bearing gotcha:** cleartext `http://` is forced to HTTP/1.1 (`curl_transport.http_version_for`) - the Chrome-impersonation profile defaults to HTTP/2, but a plaintext origin has no TLS/ALPN to negotiate h2 and uvicorn (HTTP/1.1 only) returns `400 "Invalid HTTP request received."` once a cleartext HTTP/2 body crosses ~128KB. Every internal AI consumer calls `INTERNAL_GATEWAY_URL` = `http://localhost:<port>/openai/v1/...`, so without this override a large Devii turn (big system prompt + tool list) fails on every request. `https://` keeps the HTTP/2 default. `stealth_sync_client` stays pure-httpx Chrome-aligned (no curl_cffi sync adapter). **Never bolt a hand-written browser-header dict (UA/`sec-ch-ua`/`Sec-Fetch-*`) onto a `stealth_async_client`/`stealth_sync_client` call** - curl_cffi's impersonation already emits an internally-consistent Chrome 146 identity across TLS, UA, and Client Hints; a caller-supplied header set claiming a different Chrome version or platform creates a UA-vs-TLS-vs-Client-Hints mismatch that is a textbook anti-bot tripwire (this was a real bug in Devii's `fetch_url`/`http_request` tool, fixed by deleting its hardcoded Chrome-131 header block). `stealth.detect_consent_gate(html)` recognizes the common JS-redirect cookie-consent wall pattern (DPG Media's `nu.nl` and sibling sites, and generically any page gated behind a `decodeURIComponent(...)` callback referencing a "privacy gate"/`consent.js`) - reuse it wherever a fetched page's real content sits behind that gate rather than special-casing a domain. `stealth_transport`/`stealth_async_client`/`stealth_sync_client` all accept a `proxy=` override, defaulting to `stealth.configured_proxy_url()` (the admin-editable `outbound_proxy_url` site setting, falling back to `DEVPLACE_OUTBOUND_PROXY_URL` when the setting is unset) - this is the only real fix for a destination that blocks by IP/ASN reputation (e.g. reddit.com returns an identical "blocked by network security" page across every TLS/header fingerprint tried, confirmed by testing curl_cffi's full impersonation list - it is blocking the datacenter ASN outright, not fingerprinting the client).
 - **Form validation is Pydantic-native.** Routers declare `data: Annotated[SomeForm, Form()]` (models in `models.py`); the global `RequestValidationError` handler re-renders auth pages with messages (400) or redirects other routes. Read raw `await request.form()` only when also reading an uploaded file alongside a model.
 - **`RedirectResponse(url=..., status_code=302)`** - always pass `status_code` as an integer literal, never `status.HTTP_302_FOUND`.
 - **Ownership checks:** compare `resource["user_uid"] == user["uid"]` before edit/delete (use `content.is_owner`). **Deletes additionally allow any admin:** `is_owner(...) or is_admin(user)` on every content delete endpoint, so an administrator may delete any member's content while members are limited to their own. Edits stay owner-only. Deletes are **soft** and cascade under one shared stamp - see `devplacepy/database/CLAUDE.md`.
@@ -348,7 +353,7 @@ Every feature in DevPlace is **one data source fanning out into several consumer
 
 1. **HTML** - `respond()` returns a rendered template for browsers.
 2. **JSON** - the SAME `respond(..., model=XOut)` returns JSON when `Accept: application/json`. The `*Out` schema is the gate: a context key not declared on `*Out` is silently dropped from JSON even though the template still sees it.
-3. **Agent tool** - `services/devii/actions/catalog.py` exposes the route as an `Action` so Devii can call it on the user's behalf. `requires_auth` mirrors the route guard.
+3. **Agent tool** - `services/devii/actions/catalog/` (the relevant module inside the package, e.g. `posts.py`, `admin.py`) exposes the route as an `Action` so Devii can call it on the user's behalf. `requires_auth` mirrors the route guard.
 4. **Documented endpoint** - `docs_api.py` `endpoint()` describes it (params, auth, `sample_response`) in the right group, rendered at `/docs/{group}.html`.
 
 A new public read almost always needs all four. The cardinal failure mode is changing one face and forgetting a connected one - a real worked example (followers/following listing) touched nine files across data helpers, output schema, route, view, agent tool, API docs, and the docs trio below for one conceptual feature; that count is the norm, not the exception. If a change touches only one file, confirm none of the four faces were missed. Checklist, ordered by data flow:
@@ -358,7 +363,7 @@ A new public read almost always needs all four. The cardinal failure mode is cha
 3. **Server layer.** Handler with the right auth guard (`get_current_user` public read, `require_user` member POST, `require_admin` admin). Specific paths (`/{username}/followers`) declared before catch-alls (`/{username}`). Return HTML+JSON via `respond(request, template, ctx, model=XOut)` or pure JSON via `JSONResponse`. Register any NEW router in `main.py`.
 4. **View layer.** Extend `base.html`; page CSS in `{% block extra_head %}`, page JS in `{% block extra_js %}`. Reuse partials and template globals. JS is ES6 modules, one class per file, instantiated on `app`.
 5. **Safety layer (skip only when the feature stores nothing and shows nothing).** If the feature adds a surface that carries user-generated content, register it in `database/moderation.py` `REPORTABLE_TARGETS` (or in `UNREPORTABLE_TABLES` with a reason), resolve it in `resolve_object_url`, and include `_report_button.html` in its action bar - the registry test and the e2e coverage test both fail otherwise. If it collects or transmits a new *category* of personal data, or sends anything new to a third party, update `/docs/privacy.html` and the app-store privacy declarations in the same change; that disclosure is part of the feature, not a follow-up.
-6. **Agent + docs layer (the most-forgotten step - do not skip).** `services/devii/actions/catalog.py` - add an `Action` tool if a user could ask Devii to do this (`requires_auth=False` for public reads). `docs_api.py` - every public/auth endpoint gets an `endpoint()` entry with params and a `sample_response`. `routers/docs/pages.py` `DOCS_PAGES` for prose pages. SEO (`seo.py`) - `base_seo_context` and any JSON-LD for a new public page; add to `routers/seo.py` sitemap if indexable.
+6. **Agent + docs layer (the most-forgotten step - do not skip).** `services/devii/actions/catalog/` (the relevant module inside the package) - add an `Action` tool if a user could ask Devii to do this (`requires_auth=False` for public reads). `docs_api.py` - every public/auth endpoint gets an `endpoint()` entry with params and a `sample_response`. `routers/docs/pages.py` `DOCS_PAGES` for prose pages. SEO (`seo.py`) - `base_seo_context` and any JSON-LD for a new public page; add to `routers/seo.py` sitemap if indexable.
 7. **Document.** `README.md` (product-facing, routes/config/dependencies/user-visible features), the relevant nested `CLAUDE.md` (any new mechanic, helper, pitfall, or cross-layer wiring), this root `CLAUDE.md` only when a NEW architectural rule or workflow step is introduced (not per feature).
 8. **Validate.** Confirm a clean import (`python -c "from devplacepy.main import app"`), check each touched language manually, grep for em-dash in touched files (neither the character nor its HTML entity - use a hyphen), and run `pyflakes`/`ruff check` on every touched file. **If the feature touches a spendable resource, a bounded state machine, or a read-then-write mutation reachable from more than one request path, also run the four-layer procedure in "Rigorous correctness verification" above** - it is the default for that shape of feature, not an optional extra. **Finish by running the full persisted test suite (`make test` - every tier, every test) and fixing any failure before the change is done.** Write new tests in the matching tier/path, following the required patterns above.
 
@@ -368,6 +373,12 @@ Failures at any implementation step block the workflow - never skip a failed ste
 
 Gitea Actions workflow at `.gitea/workflows/test.yaml` runs on every push/PR to `master`: installs dependencies + Playwright Chromium, runs the full suite serially under coverage, publishes coverage HTML as an artifact, uploads failure screenshots. CI must be green before merging. Changes move through DTAP: Development (`make dev`) -> Test (CI suite + coverage on `master`) -> Acceptance (`master` to `production` promotion via `make deploy`) -> Production (Docker Compose stack). Only CI-green `master` commits are promoted to `production`.
 
+## Version bumping
+
+`pyproject.toml` `version` is bumped automatically, by a real git `pre-commit` hook, not by an agent remembering to edit it. The hook lives at `.githooks/pre-commit` (tracked in the repo, plain stdlib Python) and `make install` points git at it with `git config core.hooksPath .githooks` - run `make install` once per clone (or that one `git config` line by hand) to activate it; a clone that has never run `make install` simply gets no auto-bump, which is a safe, backwards-compatible no-op, never a broken commit.
+
+On every commit the hook increments the patch component (`1.0.0` -> `1.0.1`) and stages the change, so the bump rides in the same commit with no extra step. It defers to a deliberate version edit already staged in that same commit (a hand-set major/minor bump in `pyproject.toml` is left exactly as written, never incremented further) and does nothing during a merge (`.git/MERGE_HEAD` present) or when `pyproject.toml` does not exist. It never blocks a commit - a missing or unparsable version line is a silent no-op, not a failure.
+
 ## Diagnosing a production failure (the order that finds it fastest)
 
 This procedure exists because a single "the editor is down" report turned out to be **three unrelated faults stacked on each other** (a stale URL, a firewalled network leg, and a corrupt database), and the investigation wasted hours by guessing before measuring. Work the layers outward from the browser; each step is cheap and each one eliminates a whole class of cause. **Never skip to a hypothesis, and never repair anything before the layer above it is proven healthy.**

Dockerfile

@@ -3,7 +3,7 @@ FROM python:3.13-slim
 WORKDIR /app
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    curl ca-certificates \
+    curl ca-certificates rclone \
     libglib2.0-0 libpango-1.0-0 libpangoft2-1.0-0 libharfbuzz0b libfontconfig1 \
     fonts-dejavu-core \
     && rm -rf /var/lib/apt/lists/*

Makefile

@@ -8,21 +8,42 @@ LOCUST_RUN_TIME ?= 120s
 LOCUST_WEB_WORKERS ?= 4
 WEB_WORKERS ?= $(shell nproc 2>/dev/null || echo 2)
 DEVPLACE_RATE_LIMIT ?= 1000000
+DEVPLACE_PORT ?= 10500
+
+VENV ?= $(CURDIR)/.venv
+PYTHON := $(VENV)/bin/python
+VENV_STAMP := $(VENV)/.installed
+BOOTSTRAP_PYTHON := $(shell command -v python3 2>/dev/null || command -v python 2>/dev/null)
 
 PYTHONDONTWRITEBYTECODE := 1
 export PYTHONDONTWRITEBYTECODE
 
-.PHONY: install dev clean tree tree-loc zip test test-headed test-unit test-api test-e2e test-fast test-failed test-first-failure test-slowest test-cache-clean coverage coverage-headed coverage-html locust locust-headless
+.PHONY: venv install dev prod clean tree tree-loc zip test test-headed test-unit test-api test-e2e test-fast test-failed test-first-failure test-slowest test-cache-clean coverage coverage-headed coverage-html locust locust-headless prune prune-dry-run
 
-install:
-	pip install -e .
-	python -m playwright install chromium
+$(PYTHON):
+	@test -n "$(BOOTSTRAP_PYTHON)" || { echo "python3 is required to create $(VENV)"; exit 1; }
+	$(BOOTSTRAP_PYTHON) -m venv $(VENV)
 
-dev:
-	uvicorn devplacepy.main:app --reload --reload-dir devplacepy --host 0.0.0.0 --port 10500 --backlog 4096
+$(VENV_STAMP): $(PYTHON) pyproject.toml
+	$(PYTHON) -m pip install -U pip
+	$(PYTHON) -m pip install -e ".[dev]"
+	$(PYTHON) -m playwright install chromium
+	touch $(VENV_STAMP)
 
-prod:
-	DEVPLACE_STATIC_VERSION=$$(date +%s) DEVPLACE_TEMPLATE_AUTO_RELOAD=0 DEVPLACE_WEB_WORKERS=$(WEB_WORKERS) uvicorn devplacepy.main:app --host 0.0.0.0 --port 10500 --workers $(WEB_WORKERS) --backlog 8192 --proxy-headers --forwarded-allow-ips '*'
+venv: $(VENV_STAMP)
+
+install: $(PYTHON)
+	$(PYTHON) -m pip install -U pip
+	$(PYTHON) -m pip install -e ".[dev]"
+	$(PYTHON) -m playwright install chromium
+	git config core.hooksPath .githooks
+	touch $(VENV_STAMP)
+
+dev: $(VENV_STAMP)
+	DEVPLACE_PORT=$(DEVPLACE_PORT) $(PYTHON) -m uvicorn devplacepy.main:app --reload --reload-dir devplacepy --host 0.0.0.0 --port $(DEVPLACE_PORT) --backlog 4096
+
+prod: $(VENV_STAMP)
+	DEVPLACE_STATIC_VERSION=$$(date +%s) DEVPLACE_TEMPLATE_AUTO_RELOAD=0 DEVPLACE_WEB_WORKERS=$(WEB_WORKERS) DEVPLACE_PORT=$(DEVPLACE_PORT) $(PYTHON) -m uvicorn devplacepy.main:app --host 0.0.0.0 --port $(DEVPLACE_PORT) --workers $(WEB_WORKERS) --backlog 8192 --proxy-headers --forwarded-allow-ips '*'
 
 delete-pyc:
 	find . -name "__pycache__" -type d -prune -exec rm -rf {} + 2>/dev/null || true
@@ -42,78 +63,78 @@ zip:
 	@git ls-files -z | xargs -0 zip -q $(notdir $(CURDIR)).zip
 	@printf 'Wrote %s (%s files)\n' "$(notdir $(CURDIR)).zip" "$$(git ls-files | wc -l)"
 
-test:
-	PLAYWRIGHT_HEADLESS=1 python -m pytest tests/
+test: $(VENV_STAMP)
+	PLAYWRIGHT_HEADLESS=1 $(PYTHON) -m pytest tests/
 
-test-headed:
-	PLAYWRIGHT_HEADLESS=0 python -m pytest tests/
+test-headed: $(VENV_STAMP)
+	PLAYWRIGHT_HEADLESS=0 $(PYTHON) -m pytest tests/
 
-test-unit:
-	python -m pytest tests/unit
+test-unit: $(VENV_STAMP)
+	$(PYTHON) -m pytest tests/unit
 
-test-api:
-	python -m pytest tests/api
+test-api: $(VENV_STAMP)
+	$(PYTHON) -m pytest tests/api
 
-test-e2e:
-	PLAYWRIGHT_HEADLESS=1 python -m pytest tests/e2e
+test-e2e: $(VENV_STAMP)
+	PLAYWRIGHT_HEADLESS=1 $(PYTHON) -m pytest tests/e2e
 
-test-fast:
-	python -m pytest tests/unit tests/api
+test-fast: $(VENV_STAMP)
+	$(PYTHON) -m pytest tests/unit tests/api
 
-test-failed:
-	PLAYWRIGHT_HEADLESS=1 python -m pytest tests/ --last-failed --last-failed-no-failures none
+test-failed: $(VENV_STAMP)
+	PLAYWRIGHT_HEADLESS=1 $(PYTHON) -m pytest tests/ --last-failed --last-failed-no-failures none
 
-test-first-failure:
-	PLAYWRIGHT_HEADLESS=1 python -m pytest tests/ -x
+test-first-failure: $(VENV_STAMP)
+	PLAYWRIGHT_HEADLESS=1 $(PYTHON) -m pytest tests/ -x
 
-test-slowest:
-	PLAYWRIGHT_HEADLESS=1 python -m pytest tests/ --durations=40
+test-slowest: $(VENV_STAMP)
+	PLAYWRIGHT_HEADLESS=1 $(PYTHON) -m pytest tests/ --durations=40
 
-coverage:
+coverage: $(VENV_STAMP)
 	rm -f .coverage .coverage.*
 	COVERAGE_PROCESS_START=$(CURDIR)/.coveragerc PLAYWRIGHT_HEADLESS=1 \
-		python -m coverage run -m pytest tests/
-	python -m coverage combine
-	python -m coverage report
+		$(PYTHON) -m coverage run -m pytest tests/
+	$(PYTHON) -m coverage combine
+	$(PYTHON) -m coverage report
 
-coverage-headed:
+coverage-headed: $(VENV_STAMP)
 	rm -f .coverage .coverage.*
 	COVERAGE_PROCESS_START=$(CURDIR)/.coveragerc PLAYWRIGHT_HEADLESS=0 \
-		python -m coverage run -m pytest tests/
-	python -m coverage combine
-	python -m coverage report
+		$(PYTHON) -m coverage run -m pytest tests/
+	$(PYTHON) -m coverage combine
+	$(PYTHON) -m coverage report
 
 coverage-html: coverage
-	python -m coverage html
+	$(PYTHON) -m coverage html
 	@echo "Report written to htmlcov/index.html"
 
-locust:
+locust: $(VENV_STAMP)
 	export DEVPLACE_DATABASE_URL="sqlite:///$(LOCUST_DB)"; \
 	export DEVPLACE_RATE_LIMIT=$(DEVPLACE_RATE_LIMIT); \
 	fuser -k $(LOCUST_PORT)/tcp 2>/dev/null || true; \
 	sleep 1; \
 	mkdir -p $(LOCUST_DB_DIR); \
 	rm -f $(LOCUST_DB); \
-	DEVPLACE_TEMPLATE_AUTO_RELOAD=0 DEVPLACE_WEB_WORKERS=$(LOCUST_WEB_WORKERS) uvicorn devplacepy.main:app --host 127.0.0.1 --port $(LOCUST_PORT) --workers $(LOCUST_WEB_WORKERS) --backlog 8192 > /tmp/devplace_locust_server.log 2>&1 & \
+	DEVPLACE_TEMPLATE_AUTO_RELOAD=0 DEVPLACE_WEB_WORKERS=$(LOCUST_WEB_WORKERS) $(PYTHON) -m uvicorn devplacepy.main:app --host 127.0.0.1 --port $(LOCUST_PORT) --workers $(LOCUST_WEB_WORKERS) --backlog 8192 > /tmp/devplace_locust_server.log 2>&1 & \
 	PID=$$!; \
 	while kill -0 $$PID 2>/dev/null && ! curl -s http://127.0.0.1:$(LOCUST_PORT)/ > /dev/null 2>&1; do sleep 0.5; done; \
 	if ! kill -0 $$PID 2>/dev/null; then echo "Server failed to start (port $(LOCUST_PORT) busy?). See /tmp/devplace_locust_server.log"; exit 1; fi; \
-	locust --host http://127.0.0.1:$(LOCUST_PORT) --web-port $(LOCUST_WEB_PORT); \
+	$(PYTHON) -m locust --host http://127.0.0.1:$(LOCUST_PORT) --web-port $(LOCUST_WEB_PORT); \
 	kill $$PID 2>/dev/null || true; \
 	rm -rf $(LOCUST_DB_DIR)
 
-locust-headless:
+locust-headless: $(VENV_STAMP)
 	export DEVPLACE_DATABASE_URL="sqlite:///$(LOCUST_DB)"; \
 	export DEVPLACE_RATE_LIMIT=$(DEVPLACE_RATE_LIMIT); \
 	fuser -k $(LOCUST_PORT)/tcp 2>/dev/null || true; \
 	sleep 1; \
 	mkdir -p $(LOCUST_DB_DIR); \
 	rm -f $(LOCUST_DB); \
-	DEVPLACE_TEMPLATE_AUTO_RELOAD=0 DEVPLACE_WEB_WORKERS=$(LOCUST_WEB_WORKERS) uvicorn devplacepy.main:app --host 127.0.0.1 --port $(LOCUST_PORT) --workers $(LOCUST_WEB_WORKERS) --backlog 8192 > /tmp/devplace_locust_server.log 2>&1 & \
+	DEVPLACE_TEMPLATE_AUTO_RELOAD=0 DEVPLACE_WEB_WORKERS=$(LOCUST_WEB_WORKERS) $(PYTHON) -m uvicorn devplacepy.main:app --host 127.0.0.1 --port $(LOCUST_PORT) --workers $(LOCUST_WEB_WORKERS) --backlog 8192 > /tmp/devplace_locust_server.log 2>&1 & \
 	PID=$$!; \
 	while kill -0 $$PID 2>/dev/null && ! curl -s http://127.0.0.1:$(LOCUST_PORT)/ > /dev/null 2>&1; do sleep 0.5; done; \
 	if ! kill -0 $$PID 2>/dev/null; then echo "Server failed to start (port $(LOCUST_PORT) busy?). See /tmp/devplace_locust_server.log"; exit 1; fi; \
-	locust --host http://127.0.0.1:$(LOCUST_PORT) --headless -u $(LOCUST_USERS) -r $(LOCUST_SPAWN_RATE) --run-time $(LOCUST_RUN_TIME) --html=$(LOCUST_DB_DIR)/report.html; \
+	$(PYTHON) -m locust --host http://127.0.0.1:$(LOCUST_PORT) --headless -u $(LOCUST_USERS) -r $(LOCUST_SPAWN_RATE) --run-time $(LOCUST_RUN_TIME) --html=$(LOCUST_DB_DIR)/report.html; \
 	kill $$PID 2>/dev/null || true; \
 	rm -rf $(LOCUST_DB_DIR)
 
@@ -127,6 +148,15 @@ clean:
 test-cache-clean:
 	rm -rf .pytest_cache
 
+# Safe but aggressive disk-space cleanup, acting on the REAL data/devplace.db
+# and data/ tree (never the test database) - see CLAUDE.md "devplace system
+# prune". prune-dry-run reports what would be removed without deleting.
+prune: $(VENV_STAMP)
+	$(PYTHON) -m devplacepy.cli system prune
+
+prune-dry-run: $(VENV_STAMP)
+	$(PYTHON) -m devplacepy.cli system prune --dry-run
+
 # Container Manager works out of the box: the overlay installs the docker CLI in
 # the image and mounts the host socket. DOCKER_GID is read straight from the
 # socket so the UID-1000 app can use it; the data dir is the project's own data/

README.md

@@ -7,7 +7,7 @@ Server-rendered social network for developers. FastAPI backend serving Jinja2 te
 ## Quick start
 
 ```bash
-make install        # pip install -e .
+make install        # create .venv if needed, pip install -e ".[dev]", playwright chromium
 make dev            # uvicorn --reload on port 10500
 make test           # Playwright integration + unit tests, headless, fail-fast
 make test-headed    # same tests in visible browser
@@ -63,17 +63,18 @@ devplacepy/
 | `/` | Home page: marketing splash for guests, personalized home (welcome, feed shortcut, latest posts, news) for signed-in users. Does not redirect. Latest-posts section interleaves authors so no two consecutive posts share an author. |
 | `/auth` | Signup, login, logout, forgot/reset password |
 | `/feed` | Post feed with topic/tab filtering and free-text `search` (title, content, and author username) in the left panel (public). Each page interleaves authors so no two consecutive posts share an author. |
+| `/topics` | Crawlable per-topic category pages (public): `/topics` hub links every topic with a live post count, `/topics/{topic}` lists that topic's posts with its own canonical URL, title, and breadcrumbs |
 | `/news` | Developer news listing, detail page with comments |
 | `/posts` | Post detail, creation |
-| `/gists` | Code gist listing, detail, creation, and editing; left panel offers language filtering and free-text `search` (title, description, and author username), public read |
-| `/comments` | Comment creation, owner editing (`POST /comments/edit/{comment_uid}`), deletion |
+| `/gists` | Code gist listing, detail, creation, and editing; left panel offers language filtering and free-text `search` (title, description, and author username), public read. A plain Markdown gist has a **View rendered / View raw** toggle beside its Copy button, switching between the raw source and the same rendered view a `Markdown Rendered` gist always shows |
+| `/comments` | Comment creation, owner editing (`POST /comments/edit/{comment_uid}`), deletion. Every comment has a **Copy link** button that copies its permalink (the parent post/gist/project/news URL plus `#comment-{uid}`) to the clipboard |
 | `/projects` | Project listing (left panel offers type filtering and free-text `search` over title, description, and author username), creation, owner editing (`POST /projects/edit/{slug}`), and per-project visibility toggles: `POST /projects/{slug}/private` (owner-only visibility) and `POST /projects/{slug}/readonly` (immutable files). A project hidden by a member stays visible to administrators, but a project hidden by an administrator is visible only to that owner administrator - other administrators cannot see it, its files, or its containers (web UI and REST API alike). The primary administrator (the first Admin account) is the single exception and retains full visibility |
 | `/projects/{slug}` | Dedicated project page: one encompassing card with a cover banner and project logo (owner-uploaded through the standard attachment uploader), the title overlaid on the banner, status/type/platform chips, owner-set Website and Repository links, section tabs (Overview, Devlog, Screenshots, Comments, Files), an About section, the Devlog timeline of every post linked to the project (owners post updates straight from the page via the shared composer preset to the `devlog` topic), a Screenshots gallery built from image attachments (owners add more from the More menu), and a sidebar with links, stats and the author card |
 | `/projects/{slug}/files` | Per-project filesystem: directory and file CRUD, upload, inline editing, and line-range operations (`lines` read, `replace-lines`, `insert-lines`, `delete-lines`, `append`) for surgical edits to large text files (public read, owner write; all writes refused while the project is read-only) |
 | `/zips` | Zip job status (`/zips/{uid}`) and archive download (`/zips/{uid}/download`); archives are queued via `/projects/{slug}/zip` and `/projects/{slug}/files/zip` |
 | `/forks` | Fork job status (`/forks/{uid}`); forks are queued via `/projects/{slug}/fork`. Any signed-in user can fork a project they can view into a new project they own; once the job finishes the response carries the new project URL |
-| `/tools` | Public developer tools. `/tools/seo` is **SEO Diagnostics**: audit any URL or sitemap and stream live progress over a websocket. Queue with `POST /tools/seo/run`, poll `GET /tools/seo/{uid}`, read the full report at `GET /tools/seo/{uid}/report`. `/tools/deepsearch` is **DeepSearch**: a multi-agent deep web researcher that crawls and indexes sources, synthesises a cited report with confidence scoring, and lets you chat over the gathered evidence. Queue with `POST /tools/deepsearch/run`, poll `GET /tools/deepsearch/{uid}`, read the report at `GET /tools/deepsearch/{uid}/session`, export at `/export.{md,json,pdf}`. `/tools/isslop` is the **AI Usage Analyzer**: classify a git repository or website as AI slop, sophisticated AI-assisted work or genuine human work. Queue with `POST /tools/isslop/run`, poll `GET /tools/isslop/{uid}` or the event trail at `GET /tools/isslop/{uid}/events`, read the report at `GET /tools/isslop/{uid}/report` (`.md` to download) and embed the SVG authenticity badge from `GET /tools/isslop/{uid}/badge.svg` |
-| `/projects/{slug}/containers` | Admin per-project container manager: create and control container instances, all running the shared prebuilt `ppy` image (there is no in-app image building). Reachable from the project page via the admin-only **Containers** button |
+| `/tools` | Public developer tools. `/tools/seo` is **SEO Diagnostics**: audit any URL or sitemap and stream live progress over a websocket. Queue with `POST /tools/seo/run`, poll `GET /tools/seo/{uid}`, read the full report at `GET /tools/seo/{uid}/report`. `/tools/deepsearch` is **DeepSearch**: a multi-agent deep web researcher that crawls and indexes sources, synthesises a cited report with confidence scoring, and lets you chat over the gathered evidence. Queue with `POST /tools/deepsearch/run`, poll `GET /tools/deepsearch/{uid}`, read the report at `GET /tools/deepsearch/{uid}/session`, export at `/export.{md,json,pdf}`. Every past run is kept at `GET /tools/deepsearch/history`, which lists your own research sessions with a link back to reopen the report and continue its grounded chat. `/tools/isslop` is the **AI Usage Analyzer**: classify a git repository or website as AI slop, sophisticated AI-assisted work or genuine human work. Queue with `POST /tools/isslop/run`, poll `GET /tools/isslop/{uid}` or the event trail at `GET /tools/isslop/{uid}/events`, read the report at `GET /tools/isslop/{uid}/report` (`.md` to download) and embed the SVG authenticity badge from `GET /tools/isslop/{uid}/badge.svg` |
+| `/projects/{slug}/containers` | Admin per-project container manager: create and control container instances, all running the shared prebuilt `ppy` image (there is no in-app image building). Reachable by direct URL and from the admin index |
 | `/admin/containers` | Admin **Containers** manager: list, create, edit, and control container instances across projects, under strict per-user isolation: the primary administrator sees and manages every instance; every other administrator sees instances on public projects plus their own (instances attached to another user's private project are excluded entirely) and manages only the instances they own (created by them or attached to their own project) - all other rows are view-only. The list (`/admin/containers`) has inline start/stop/restart/terminal/edit/delete on each row and a create form (pick a project, optionally a run-as user, a boot language with a source editor, restart policy, start-on-boot, plus env/ports/limits/ingress). Each instance has a detail page (`/admin/containers/{uid}`) with lifecycle controls, live logs and metrics, an interactive terminal, schedules, ingress, workspace sync, and a status history, and an edit page (`/admin/containers/{uid}/edit`) |
 | `/p/{slug}` | Public ingress proxy (HTTP + WebSocket) to a running container instance's published port, opt-in per instance via `ingress_slug` |
 | `/profile` | Profile view, editing, a public **Media** tab (`?tab=media`) showing every attachment a user uploaded newest first, and a live **online / last-seen** presence indicator |
@@ -82,10 +83,11 @@ devplacepy/
 | `/admin/trash` | Admin **Trash**: review, restore, and permanently purge soft-deleted content (posts, comments, gists, projects, news, project files, attachments) across the platform |
 | `/admin/devii-tasks` | Admin **Devii tasks**: every scheduled task across all owners with its schedule, run count, expiry and failure streak, plus per-task disable and delete |
 | `/notifications` | Notification list, mark read, live unread counts (`/notifications/counts`) |
-| `/messages` | Real-time direct messaging over WebSocket (`/messages/ws`): live bidirectional delivery, optimistic send, typing indicators, read receipts, and online/last-seen presence. Messages render through the shared content pipeline (emoji shortcodes, image, video and audio embeds, autolink, sanitization). AI content correction and the AI modifier apply to direct messages, so typing an inline `@ai <instruction>` in a message executes it and the resolved result appears live in the chat for both participants (the `message` WS frame carries an additive `ai_processed` flag when a pending correction/modification was applied). An opened conversation loads its 500 most recent messages; older history is retained in the database. `GET /messages/conversations` returns the conversation list as JSON for a live client refresh; the `POST /messages/send` form remains as a no-JavaScript fallback and now accepts an attachment-only, empty-content message. `POST /messages/ws-ticket` exchanges a session/API-key auth to a short-lived (30s), single-use ticket a browser WebSocket can carry as `?ticket=...` on the handshake, since a native `WebSocket` cannot set an `Authorization`/`X-API-KEY` header - this is what makes off-session embedding of the chat possible. CLI: `devplace messaging prune-tickets` removes expired tickets |
+| `/messages` | Real-time direct messaging over WebSocket (`/messages/ws`): live bidirectional delivery, optimistic send, typing indicators, read receipts, reconnect catch-up, and online/last-seen presence. Messages render through the shared content pipeline (emoji shortcodes, image, video and audio embeds, autolink, sanitization). AI content correction and the AI modifier apply to direct messages: the original is delivered immediately, then a second `message` frame with `ai_processed` replaces the bubble for both participants when the rewrite lands (including across workers). An opened conversation loads its 500 most recent messages and can page older rows with `?before=`; switching threads does not drop the socket. `GET /messages/conversations` returns the conversation list as JSON for a live client refresh; the `POST /messages/send` form remains as a no-JavaScript fallback and now accepts an attachment-only, empty-content message. `POST /messages/ws-ticket` exchanges a session/API-key auth to a short-lived (30s), single-use ticket a browser WebSocket can carry as `?ticket=...` on the handshake, since a native `WebSocket` cannot set an `Authorization`/`X-API-KEY` header - this is what makes off-session embedding of the chat possible. CLI: `devplace messaging prune-tickets` removes expired tickets |
 | `/votes` | Upvote/downvote on posts, comments, projects |
 | `/reactions` | Emoji reactions on posts, comments, gists, projects |
 | `/bookmarks` | Save/unsave content; `/bookmarks/saved` personal list |
+| `/notes` | Private per-user notes attached to a post, gist, project, or news article; `POST /notes/{target_type}/{target_uid}` adds or replaces the note, `POST /notes/{target_type}/{target_uid}/delete` removes it, `/notes/saved` is your personal notes list |
 | `/polls` | Vote on post-attached polls |
 | `/follow` | Follow/unfollow users |
 | `/reports` | Content reporting: `POST /reports/{target_type}/{target_uid}` files a report against any user-generated surface, `GET /reports/mine` lists the reports you filed and their outcome, `GET /reports/reasons` serves the reason registry so every client renders the same dialog |
@@ -107,7 +109,7 @@ devplacepy/
 | `/openai` | OpenAI-compatible LLM gateway service (`/openai/v1/chat/completions`, `/openai/v1/*`) |
 | `/devii` | Devii agentic assistant: WebSocket terminal (`/devii/ws`), standalone page, usage (`/devii/usage`), session bootstrap |
 | `(none)` | `/robots.txt`, `/sitemap.xml` (SEO) |
-| `(none)` | `/push.json` (VAPID key + subscribe), `/service-worker.js`, `/manifest.json` (push + PWA) |
+| `(none)` | `/push.json` (VAPID key + subscribe/unsubscribe), `/service-worker.js`, `/manifest.json` (push + PWA) |
 
 ## Gamification
 
@@ -224,6 +226,7 @@ The farm refreshes live over the pub/sub bus (a watered build appears on the own
 - **Polls** - a post can carry a poll (question plus up to six options); results appear as live bars once the viewer votes, one vote per member. A poll can be attached when the post is created or added later by editing a post that has none.
 - **Paste an image to attach it** - pressing Ctrl+V (Cmd+V) with a screenshot or copied image on the clipboard while writing a post, a comment, a direct message, an issue, a gist, or a project attaches it immediately, with no trip through the file picker. The upload, its limits, and the resulting attachment are identical to picking the file by hand.
 - **Bookmarks** - save posts, gists, projects, and news to a personal list at `/bookmarks/saved`.
+- **Personal notes** - attach a private note to a post, gist, project, or news article; nobody else, not even the content's author, can ever see it. Managed from an "Add note"/"Edit note" button on the item and listed on a personal page at `/notes/saved`.
 - **Private projects** - an owner can mark a project private so it is visible only to them (and administrators) and excluded from listings, profiles, search, the sitemap, and zip access. Set at creation or toggled later from the project page.
 - **Read-only projects** - an owner can mark a project read-only, making its entire virtual filesystem immutable: every write, edit, line-edit, move, delete, and upload is refused from all paths (the web UI, the HTTP API, the Devii agent, and container workspace sync) until read-only is turned off. Devii may toggle read-only only after the user explicitly confirms.
 
@@ -259,12 +262,14 @@ The log is **administrator-only**. `/admin/audit-log` is a paginated, filterable
 |---------|---------|---------|
 | `DEVPLACE_DATABASE_URL` | `sqlite:///<repo>/data/devplace.db` | Database connection string |
 | `DEVPLACE_DATA_DIR` | `<repo>/data` | Single root for every runtime/user-generated artifact (DB, uploads, VAPID keys, locks, bot state, job staging, container workspaces), outside the package and not served via `/static`. Point at a volume in production. Defined once in `config.py` (`DATA_PATHS` registry, created by `ensure_data_dirs()`) |
+| `DEVPLACE_PORT` | `10500` | Port the app itself binds to (`make dev`/`make prod`, and what `Makefile`'s `dev`/`prod` targets pass to uvicorn's `--port`). Also feeds `config.PORT`, so `DEVPLACE_INTERNAL_BASE_URL`'s default and Devii's own self-dial URL follow it automatically. Bare metal only - the Docker app container always binds its fixed internal port regardless of this var; for Docker, use `PORT` (below) to change the externally reachable port |
+| `PORT` | `10500` | Docker only: the host port `docker-compose.yml` publishes nginx on (`127.0.0.1:${PORT}:80`). Unrelated to `DEVPLACE_PORT` above - it never reaches the app container |
 | `SECRET_KEY` | hardcoded fallback | Session signing key |
 | `DEVPLACE_VAPID_SUB` | `mailto:retoor@molodetz.nl` | Contact address in the VAPID JWT `sub` claim |
 | `DEVPLACE_INTERNAL_BASE_URL` | `http://localhost:10500` | Base URL the platform's own services dial for the AI gateway |
 | `DEVPLACE_XMLRPC_PORT` | `10550` | Loopback port the forking XML-RPC bridge binds; the app and nginx reverse-proxy `/xmlrpc` to it |
 | `DEVPLACE_XMLRPC_BIND` | `127.0.0.1` | Bind address for the XML-RPC bridge (loopback; the app and nginx are the intended front doors) |
-| `DEVPLACE_STATIC_VERSION` | server boot unix timestamp | Cache-busting version stamped into every static asset URL (`/static/v<version>/...`). Set it at launch so multiple workers agree (the `prod` target and Docker image do this); leave unset in dev to refresh on each reload. See [Static asset caching](#static-asset-caching) |
+| `DEVPLACE_STATIC_VERSION` | server boot unix timestamp | The boot-id half of the cache-busting version stamped into every static asset URL (`/static/v<app-version>-<boot-id>/...`, e.g. `/static/v1.0.1-1718040000/...`). Set it at launch so multiple workers agree (the `prod` target and Docker image do this); leave unset in dev to refresh on each reload. See [Static asset caching](#static-asset-caching) |
 | `DEEPSEEK_API_KEY` / `OPENROUTER_API_KEY` | unset | Upstream provider keys; migrated into the gateway settings on first boot |
 | `DEVPLACE_PRESENCE_TIMEOUT_SECONDS` | `60` | Online-presence window: a user counts as online for this many seconds after their last activity. `last_seen` is refreshed by a throttled in-place update at most once per half this interval per worker (no per-load inserts, no data growth) |
 | `DEVPLACE_PRESENCE_ONLINE_LIMIT` | `30` | Maximum avatars shown in the feed's live "Online now" panel (ordered alphabetically by username) |
@@ -285,6 +290,7 @@ Operational behavior is tunable live from `/admin/settings` (stored in `site_set
 | `registration_open` | `1` | When `0`, new sign-ups are rejected |
 | `maintenance_mode` | `0` | When `1`, non-admins see the maintenance page; admins retain access |
 | `maintenance_message` | scheduled-maintenance text | Message shown during maintenance |
+| `happy_404_enabled` | `1` | When `1`, an HTML page that would otherwise 404 renders a random existing post instead ("Happy 404"); JSON/API requests always get a real 404 regardless. See [Happy 404](#happy-404) |
 | `customization_enabled` | `1` | When `0`, no user CSS/JS customization is injected on any page |
 | `customization_js_enabled` | `1` | When `0`, user custom CSS is still served but custom JavaScript is suppressed |
 | `audit_log_retention_days` | `90` | Audit rows older than this are pruned daily by the Audit retention service; `0` disables pruning |
@@ -339,6 +345,27 @@ curl -H "Accept: application/json" https://your-host/feed
 curl -H "Accept: application/json" -X POST -d "content=hi&title=T&topic=devlog" https://your-host/posts/create
 ```
 
+## Happy 404
+
+Instead of a bare error page, an HTML request that would 404 (an unmatched route, or an app route
+raising `not_found()` for a missing resource) instead renders a random existing post at that URL,
+using the exact same template and context as the real `/posts/{slug}` page. JSON/API requests are
+unaffected and still get a normal `404` - the substitution only ever applies to a browser HTML
+navigation.
+
+- **Toggle:** `happy_404_enabled` site setting (`/admin/settings`, on by default).
+- **Scope:** any HTML `GET` 404, app-wide - not just under `/posts`.
+- **Performance:** a small pool of random post slugs is cached in-process for a few minutes and
+  refreshed with a fresh random sample on expiry, so every request only does an in-memory pick plus
+  one indexed lookup - no per-request full-table scan - while the pool composition still cycles
+  through the whole `posts` table over time.
+- **SEO safety:** the substituted page is always marked `noindex,nofollow` so the decoy URL is never
+  indexed under the wrong address.
+- **Fail-closed:** any error while building the substitute page falls straight through to the normal
+  404 page - this feature can never turn a real error into a worse one.
+
+Implementation: `devplacepy/happy404.py`, wired into the `404` exception handler in `main.py`.
+
 ## XML-RPC bridge
 
 The full REST API is also reachable over XML-RPC at `/xmlrpc`. A standalone forking XML-RPC
@@ -452,16 +479,16 @@ and its full configuration are documented automatically - including future servi
 - **`ConfigField`** - declarative parameter spec (type, default, validation, secret) a service uses to declare its editable settings
 - **`BaseService`** - abstract class with a reconciling run loop that honors the persisted `enabled`/command/interval state, plus `config_fields`, `get_config()`, `describe()`, and a log buffer
 - **`ServiceManager`** - singleton: `register`, `describe_all`, `set_enabled`, `send_command`, `save_config`, `supervise`, `shutdown_all`
-- **`NewsService`** - a fully automatic, zero-maintenance news pipeline: fetches news from `news.app.molodetz.nl/api`, cleans each article, fetches and perceptually compares the images to reject placeholders and detect uniqueness, grades each one deterministically, reformats every valid article into clean Markdown (paragraphs, headings, lists) with the AI so the source wall of text reads as a proper article, and auto-rotates the best articles to Featured and the landing page. Its AI spend is metered from the gateway response headers and reported on the admin Services page (calls, tokens, total cost, and per-call averages)
+- **`NewsService`** - a fully automatic, zero-maintenance news pipeline: fetches news from `news.app.molodetz.nl/api`, cleans each article, fetches and perceptually compares the images to reject placeholders and detect uniqueness, grades each one against a free, local, deterministic quality model (aquality - no billed LLM call, no API key), optionally reformats articles that clear the publish grade threshold into clean Markdown with a generative AI model (off by default, since the default grading model cannot generate text), and auto-rotates the best articles to Featured and the landing page. Any AI spend it does incur is metered from the gateway response headers and reported on the admin Services page (calls, tokens, total cost, and per-call averages)
 - **`BotsService`** - Playwright fleet of AI personas that browse and interact with a DevPlace instance, with live cost/usage metrics and a live screenshot monitor at `/admin/bots` (opt-in; install the `bots` extra)
-- **`GatewayService`** - OpenAI-compatible LLM gateway at `/openai/v1/*`, forwarding to DeepSeek (default) with optional vision augmentation; the single point of truth for AI that every other service routes through (enabled by default). Every chat request is made date-aware by injecting the current date in EU `DD/MM/YYYY` format into the system message when it contains no date already (date only, never time, to keep upstream prompt caching effective), and an admin-configurable system preamble (`gateway_system_preamble`) can be prepended ahead of the client's system message on every call. **Provider and model routing** (admin **Gateway** page, `/admin/gateway`) maps any number of requested model names onto named upstream providers and target models, each with its own pricing economy (input, output and cache-hit/cache-miss rates per million tokens) and an optional vision model that describes image content before forwarding, so text and vision models are merged transparently. Unmapped requests fall through to the default upstream unchanged, so existing clients are unaffected
+- **`GatewayService`** - OpenAI-compatible LLM gateway at `/openai/v1/*`, forwarding to DeepSeek (default) with optional vision augmentation; the single point of truth for AI that every other service routes through (enabled by default). Every chat request is made date-aware by injecting the current date in EU `DD/MM/YYYY` format into the system message when it contains no date already (date only, never time, to keep upstream prompt caching effective), and an admin-configurable system preamble (`gateway_system_preamble`) can be prepended ahead of the client's system message on every call. **Thinking is disabled by default** on every chat and vision call (`thinking.type=disabled` on DeepSeek, `reasoning.effort=none` on OpenRouter, `think=false` on Ollama) so the fast path is the default; a client may re-enable it per request, and an administrator may flip `gateway_thinking` on. **Provider and model routing** (admin **Gateway** page, `/admin/gateway`) maps any number of requested model names onto named upstream providers and target models, each with its own pricing economy (input, output and cache-hit/cache-miss rates per million tokens) and an optional vision model that describes image content before forwarding, so text and vision models are merged transparently. Unmapped requests fall through to the default upstream unchanged, so existing clients are unaffected. Each route may also name a **fallback model** - another already-configured model of the same kind, picked from a select box of the public model names (never a free-text provider model id) - that the gateway retries once, automatically, whenever the primary model fails after its own retries are exhausted, so a struggling model degrades to a working one instead of failing the caller
 - **`JobService` / `ZipService` / `ForkService`** - generic async job framework (`services/jobs/`) for heavy, blocking work run off the request path; `ZipService` builds project zip archives in a subprocess, `ForkService` copies a project into a new project owned by the forking user
 - **`ContainerService`** - the admin container manager (`services/containers/`): a reconciling supervisor for container instances, all running one shared prebuilt image
 - **`AcceptanceService`** - grants every policy agreement (Terms of Service, Privacy Policy, third-party AI processing, activity recording, container credentials) to every account that has not declined it, so an instance kept production-identical for extended manual testing never interrupts with an acceptance dialog. Administrator-only, **off by default**, with a separate switch per agreement type and a dry run that reports what it would do without writing. An account that withdrew a consent is never granted it again, with no further action: the consent ledger itself is the decline register. It is not appropriate on a real production host
 
 ### Container manager (admin only)
 
-`services/containers/` runs supervised container instances from the web UI, the HTTP API, and Devii. It drives the `docker` CLI via async subprocesses behind a pluggable `Backend` interface (a `DockerCliBackend` plus a `FakeBackend` for tests). **There is no in-app image building.** Every instance runs ONE shared prebuilt image, `ppy:latest` (override `DEVPLACE_CONTAINER_IMAGE`), built once with **`make ppy`** from `ppy.Dockerfile`: a Python + Playwright base with a broad set of common libraries preinstalled, the `pravda` (uid 1000) user, the sudo superclone, `pagent` at `/usr/bin/pagent.py`, and the `code-server` browser IDE that backs dev workspaces all baked in. Creating an instance is then an instant `docker run` (it fails fast with a clear error if the `ppy` image has not been built yet). `ContainerService` reconciles desired instance state against `docker ps` each tick (containers are labeled `devplace.instance=<uid>`, so orphans are reaped and no state is lost), applies restart policies, fires cron/interval/one-time schedules, and samples metrics. The container's `/app` is bind-mounted to a persistent project workspace (materialized from the project files) and **stays in sync automatically**: the manager runs a bidirectional, newer-wins sync between the project files and the workspace on every start and roughly once a minute while running, so edits made inside the container and edits made in the project file editor converge without manual intervention (the sync only ever creates or overwrites the older copy of a file, never deletes one; a read-only project is export-only). Projects that need extra packages use runtime `pip install` (pravda owns the site-packages, no sudo needed) or `apt install` directly (the `pravda` user runs `apt`/`dpkg` through a fakeroot wrapper, so system packages install without root) or add the library to `ppy.Dockerfile` and rerun `make ppy`. Each instance can run a **boot script** in Python or Bash (written into the workspace and run on launch) or a plain boot command, can be set to **start automatically** whenever the container service starts, and can be configured to **run as** a chosen DevPlace user - which only selects whose identity and API key are injected into the container (the container always runs as the unprivileged `pravda` user). Every status change is recorded and shown as a status history on the instance page. A running instance can be **published** with an `ingress_slug`, making its service reachable (HTTP and WebSocket) at `/p/<slug>` through DevPlace. The manager is reached two ways: the admin **Containers** sidebar entry (`/admin/containers`) lists, creates, edits, and controls every instance across all projects and opens a dedicated detail page per instance, and each project page carries an admin-only **Containers** button to its own instance manager.
+`services/containers/` runs supervised container instances from the web UI, the HTTP API, and Devii. It drives the `docker` CLI via async subprocesses behind a pluggable `Backend` interface (a `DockerCliBackend` plus a `FakeBackend` for tests). **There is no in-app image building.** Every instance runs ONE shared prebuilt image, `ppy:latest` (override `DEVPLACE_CONTAINER_IMAGE`), built once with **`make ppy`** from `ppy.Dockerfile`: a Python + Playwright base with a broad set of common libraries preinstalled, the `pravda` (uid 1000) user, the sudo superclone, `pagent` at `/usr/bin/pagent.py`, and the `code-server` browser IDE that backs dev workspaces all baked in. Creating an instance is then an instant `docker run` (it fails fast with a clear error if the `ppy` image has not been built yet). `ContainerService` reconciles desired instance state against `docker ps` each tick (containers are labeled `devplace.instance=<uid>`, so orphans are reaped and no state is lost), applies restart policies, fires cron/interval/one-time schedules, and samples metrics. The container's `/app` is bind-mounted to a persistent project workspace (materialized from the project files) and **stays in sync automatically**: the manager runs a bidirectional sync between the project files and the workspace on every start and roughly once a minute while running, so edits made inside the container and edits made in the project file editor converge without manual intervention. Deletions propagate too, in both directions: deleting a file inside the container removes it from the project, and deleting it in the project's file editor removes it from the container, tracked against a per-file sync baseline so a genuine deletion is never confused with a file that simply has not been materialized to that workspace yet; an edit made after a conflicting deletion always wins and restores the file (a read-only project is export-only and always mirrors the project verbatim, including removing files the project no longer has). Projects that need extra packages use runtime `pip install` (pravda owns the site-packages, no sudo needed) or `apt install` directly (the `pravda` user runs `apt`/`dpkg` through a fakeroot wrapper, so system packages install without root) or add the library to `ppy.Dockerfile` and rerun `make ppy`. Each instance can run a **boot script** in Python or Bash (written into the workspace and run on launch) or a plain boot command, can be set to **start automatically** whenever the container service starts, and can be configured to **run as** a chosen DevPlace user - which only selects whose identity and API key are injected into the container (the container always runs as the unprivileged `pravda` user). Every status change is recorded and shown as a status history on the instance page. A running instance can be **published** with an `ingress_slug`, making its service reachable (HTTP and WebSocket) at `/p/<slug>` through DevPlace. The manager is reached two ways: the admin **Containers** sidebar entry (`/admin/containers`) lists, creates, edits, and controls every instance across all projects and opens a dedicated detail page per instance, and each project page carries an admin-only **Containers** button to its own instance manager.
 
 **Security:** this requires mounting the Docker socket, which grants host root. Every run, exec, lifecycle, and schedule operation is administrator-only; `--privileged` is never used and all docker calls are argument-list subprocesses. Containers are additionally isolated per user: an instance is managed only by its owner (the administrator who created it, or the owner of its project) and by the primary administrator, who alone sees and manages every instance including those on private projects; other administrators get a read-only view of instances on public projects and none of another user's private-project instances (exec, terminals, schedules, edits, and lifecycle actions are all refused and audited as denied). The service is disabled by default; an admin enables **Containers** on `/admin/services`. CLI: `devplace containers list | reconcile | prune | prune-builds | gc-workspaces` (`prune-builds` is a one-time cleanup that removes legacy per-project images and the old dockerfiles/builds tables).
 
@@ -470,10 +497,19 @@ and its full configuration are documented automatically - including future servi
 ### Dev Workspaces and the browser editor
 
 A **workspace** is a member-facing container running the DevPlace browser editor, layered on the
-container runtime above. It is opened from a project's **Workspace** page and reached at
-`/projects/{slug}/workspace`; the editor itself is proxied at
-`/projects/{slug}/containers/instances/{uid}/code/`, and an **Editor** button appears on the project
-page whenever the workspace is running.
+container runtime above. It is reached at `/projects/{slug}/workspace`; the editor itself is proxied
+at `/projects/{slug}/containers/instances/{uid}/code/`, and the workspace page shows an **Open
+editor** link whenever the editor is actually reachable.
+
+**The workspace page reports the editor's real state, live.** A workspace has a *phase* derived on
+the server from its desired state, its container status and a TCP probe of the editor port:
+`stopped`, `starting`, `ready`, `stopping`, `crashed` or `suspended`. The page renders the control
+that matches the phase, so pressing **Start** turns the button into a spinner reading *Starting the
+editor* and the **Open editor** link appears only once code-server answers, never while the
+container is still booting. While a workspace is in transition the page polls every two seconds
+(twenty seconds otherwise) and also receives pushed updates on the owner's private pub/sub topic, so
+the label changes on its own without a reload; the project page's **Editor** button follows the same
+readiness rule. The phase, its label and `editor_ready` are part of the workspace JSON.
 
 The editor is `code-server`, rebranded as DevPlace end to end: the application name, the browser tab
 icon and PWA icons, the login page styling, and `product.json` all carry DevPlace, and a bundled
@@ -486,10 +522,21 @@ coding agent baked into the image, and a plain login shell beside it with the Py
 Swift toolchains on `PATH`. Both are configurable, and `bash` stays the default profile for
 terminals the member opens later.
 
-The workspace opens straight onto the member's files rather than a welcome page, and the editor's
-own built-in chat assistant is suppressed so `dpc` is the only agent on offer and every token it
-spends is ledgered against the member's DevPlace account. `dpc`'s own working files (`.dpc/`,
-`dpc.log`) are in `SYNC_SKIP_NAMES`, so running an agent on every boot never pollutes the project.
+**The first boot of a workspace opens the DevPlace welcome page** beside the terminals: a webview
+introducing the workspace and DevPlace Code (its 900k token context, vision, parallel sub-agents,
+deep research, safety gates and the daily credits it runs on), with example prompts and buttons that
+focus the agent terminal, start the walkthrough, show the public tunnels and open the workspace
+settings. It is shown once per workspace and can be reopened with **DevPlace: Show the welcome
+page**. The editor's own built-in chat assistant and VS Code's own welcome page stay suppressed so
+`dpc` is the only agent on offer and every token it spends is ledgered against the member's DevPlace
+account. `dpc`'s own working files (`.dpc/`, `dpc.log`) are in `SYNC_SKIP_NAMES`, so running an
+agent on every boot never pollutes the project.
+
+**The terminal panel gets about a third of the window by default.** Every preset is a fixed number
+of steps up from the panel's minimum height (`normal`, the default, lands at roughly a third of a
+typical window; `short` at a fifth, `tall` at about half) and `maximized` fills the editor area.
+A preset is applied on the first boot of a workspace and again whenever it changes; a height the
+member drags themselves is kept across restarts.
 
 **Every workspace is trusted.** VS Code Restricted Mode is disabled at the command line and in the
 seeded settings, so nothing prompts and automatic tasks run. This is a deliberate default with a
@@ -520,15 +567,15 @@ restart.
 
 `ForkService` copies a project into a new project owned by the forking user. The **Fork** button on the project page (any signed-in user) prompts for a name; the job creates the destination project, duplicates the entire virtual filesystem, and records a directional `project_forks` relation so each fork shows a "Forked from X" link. The frontend `app.projectForker` enqueues, polls `/forks/{uid}`, and redirects to the new project once it is done; on failure the partially created project is rolled back. The forked project is permanent, so retention removes only the job tracking row. CLI: `devplace forks prune` / `devplace forks clear` (job rows only).
 
-`SeoService` powers the public **Tools -> SEO Diagnostics** auditor. It runs a headless-browser (Playwright) crawl of a single URL or a sitemap (capped pages) in a subprocess and runs a broad battery of checks across eleven categories: crawlability and indexing (status, redirects, HTTPS/HSTS, canonical, robots/meta-robots, sitemap, URL hygiene, mixed content), on-page meta and content (title, description, headings, language, charset, viewport, favicon, content depth), links, structured data and rich results (JSON-LD validity and required properties, microdata/RDFa), social cards (Open Graph, Twitter), Core Web Vitals and performance (LCP, CLS, FCP, TTFB, page weight, requests, DOM size, compression, caching, image optimisation, console errors), mobile and accessibility (responsive layout, tap targets, image alt, form labels), security headers, and AI/LLM-search readiness (server-rendered-vs-JS content parity, `llms.txt`, semantic HTML). It produces a weighted score and grade with per-category subscores and a recommendation for every finding. Progress streams live over `WS /tools/seo/{uid}/ws`; the full report is available at `/tools/seo/{uid}/report` (HTML or JSON). CLI: `devplace seo prune` / `devplace seo clear`. Playwright is a core dependency; `make install` fetches the Chromium browser.
+`SeoService` powers the public **SEO Diagnostics** auditor at `/tools/seo`. It runs a headless-browser (Playwright) crawl of a single URL or a sitemap (capped pages) in a subprocess and runs a broad battery of checks across eleven categories: crawlability and indexing (status, redirects, HTTPS/HSTS, canonical, robots/meta-robots, sitemap, URL hygiene, mixed content), on-page meta and content (title, description, headings, language, charset, viewport, favicon, content depth), links, structured data and rich results (JSON-LD validity and required properties, microdata/RDFa), social cards (Open Graph, Twitter), Core Web Vitals and performance (LCP, CLS, FCP, TTFB, page weight, requests, DOM size, compression, caching, image optimisation, console errors), mobile and accessibility (responsive layout, tap targets, image alt, form labels), security headers, and AI/LLM-search readiness (server-rendered-vs-JS content parity, `llms.txt`, semantic HTML). It produces a weighted score and grade with per-category subscores and a recommendation for every finding. Progress streams live over `WS /tools/seo/{uid}/ws`; the full report is available at `/tools/seo/{uid}/report` (HTML or JSON). CLI: `devplace seo prune` / `devplace seo clear`. Playwright is a core dependency; `make install` fetches the Chromium browser.
 
 `SeoMetaService` is a separate AI subservice that generates a clean, search-optimised title, description and short keyword list for every published post, project, gist, news article and issue, entirely off the request path so it never slows the web server. The work is queued whenever content is created, edited or published; until the AI value is ready a plain-content default (built from the markdown-stripped text and clamped to safe lengths) fills the fields, so a page's metadata is **always populated, never empty**. The service uses the built-in internal AI gateway and meters its own AI cost and statistics in a dedicated usage table, surfaced together with its live task pipeline on the **Admin -> Services** page. This release also fixes the on-page metadata: the `<meta name="description">` is now stripped of markdown markup (it previously leaked `#`, `**` and `[](...)` from the raw body), a `<meta name="keywords">` tag is emitted (a short honest list, not stuffed), and social-card image dimensions and alt text are added. CLI: `devplace seo-meta prune` / `devplace seo-meta clear` (job rows only; the generated metadata persists).
 
-`DeepsearchService` powers the public **Tools -> DeepSearch** researcher. Given a single research question it plans a set of diverse web search queries, interleaves their results so every angle contributes sources, and crawls the most relevant pages in a subprocess (concurrent fetches, plain HTTP first with a headless-browser fallback for JavaScript-heavy pages, PDF documents streamed and text-extracted, every URL SSRF-guarded). Social sites that block bots (X, YouTube, Reddit and similar) contribute the readable text supplied by the search engine, so their content is not lost to a login wall. A readability-grade extractor isolates the main article content of each page (navigation, cookie banners and footers are discarded), and the configured depth follows the most relevant in-page links for deeper coverage. Content is de-duplicated and indexed into a per-session ChromaDB vector collection (embeddings via the AI gateway with a local fallback). A chain of agents (report writer, findings extractor, linker) then synthesises a thorough cited markdown report grounded on the passages retrieved from that index, with key findings, a confidence score and source diversity; if synthesis fails the report is clearly marked as degraded instead of silently shipping raw source material. Progress streams live over `WS /tools/deepsearch/{uid}/ws`; the report is at `/tools/deepsearch/{uid}/session` (HTML or JSON) and can be exported as Markdown, JSON or PDF. A grounded chat over the session's evidence runs at `WS /tools/deepsearch/{uid}/chat` using hybrid retrieval (vector + keyword/BM25). Runs can be paused, resumed or cancelled. A cross-session URL cache avoids re-fetching pages seen by earlier runs. CLI: `devplace deepsearch prune` / `devplace deepsearch clear`. ChromaDB, weasyprint, pypdf and Playwright are core dependencies.
+`DeepsearchService` powers the public **DeepSearch** researcher at `/tools/deepsearch`, an iterative agent modelled on the way Perplexity and OpenAI's own deep-research agents work: search, read, decide what is still missing, search again. Given a single research question it plans a set of diverse web search queries, interleaves their results so every angle contributes sources, and crawls the most relevant pages in a subprocess (concurrent fetches, plain HTTP first with a headless-browser fallback for JavaScript-heavy pages, PDF documents streamed and text-extracted, every URL SSRF-guarded). After the first pass it runs up to two automatic gap-filling refinement rounds - a lightweight planner looks at what has been gathered so far and proposes a few more targeted queries only if real coverage gaps remain, bounded by the page budget, and stops the moment nothing more is needed. Social sites that block bots (X, YouTube, Reddit and similar) contribute the readable text supplied by the search engine, so their content is not lost to a login wall. A readability-grade extractor isolates the main article content of each page (navigation, cookie banners and footers are discarded), and the configured depth follows the most relevant in-page links for deeper coverage. Content is de-duplicated and indexed into a per-session ChromaDB vector collection (embeddings via the AI gateway with a local fallback); retrieval reranks candidate passages with Maximal Marginal Relevance so the report is grounded on genuinely diverse evidence rather than several near-duplicate hits on the same page. A chain of agents (report writer, findings extractor, linker) then synthesises a thorough cited markdown report grounded on the passages retrieved from that index, with key findings, a confidence score, source diversity and a short list of suggested follow-up questions you can ask straight into the chat; if synthesis fails the report is clearly marked as degraded instead of silently shipping raw source material. Progress streams live over `WS /tools/deepsearch/{uid}/ws`; the report is at `/tools/deepsearch/{uid}/session` (HTML or JSON) and can be exported as Markdown, JSON or PDF. A grounded chat over the session's evidence runs at `WS /tools/deepsearch/{uid}/chat` using hybrid retrieval (vector + keyword/BM25). Runs can be paused, resumed or cancelled. Every run you have started is listed at `GET /tools/deepsearch/history`, newest first with its query, status and score, so you can reopen a finished report and pick the chat back up later. A cross-session URL cache avoids re-fetching pages seen by earlier runs. CLI: `devplace deepsearch prune` / `devplace deepsearch clear`. ChromaDB, weasyprint, pypdf and Playwright are core dependencies.
 
-`IsslopService` powers the public **Tools -> AI Usage Analyzer**, which classifies a git repository or website as AI slop, sophisticated AI-assisted work or genuine human work. Sources are acquired in an isolated subprocess (git URLs are probed with `git ls-remote` and shallow-cloned with a 3 GB guard; websites render in a stealth headless browser with an HTTP fallback, bounded by depth, file and byte caps), inventoried with exclusion rules, and scored by a multi-signal static engine (twenty-one detector families, 126 signal types across an origin axis and a quality-deficit axis). For a live website, the analyzer also opens the home page in a headless browser and inspects what actually renders: it fingerprints AI website-builder platforms directly (Lovable, Bolt.new, Framer and others) and checks the page's real computed styles, layout and build artifacts, not just its file contents. Representative files receive an AI review pass and images an AI-generation review through the internal gateway (model `molodetz`, internal key); the static engine stays authoritative when the gateway is unreachable. Every pipeline step is persisted as an ordered event trail and streamed live over the pub/sub topic `public.isslop.{uid}`. The verdict is an A-F authenticity grade with a human/AI split and one of five categories (`ai-slop`, `sophisticated-ai`, `human-clean`, `human-messy`, `uncertain`), published as a persistent report with an embeddable SVG badge. Members keep their analysis history on their account; guest history is session-bound and claimed by the account on the first signed-in visit. Admin settings (private-host allowance, AI/image review toggles, image cap, retention, concurrency) live on `/admin/services`. CLI: `devplace isslop analyze <url>` / `devplace isslop prune` / `devplace isslop clear`. Playwright plus playwright-stealth back the website crawler.
+`IsslopService` powers the public **AI Usage Analyzer** at `/tools/isslop`, which classifies a git repository or website as AI slop, sophisticated AI-assisted work or genuine human work. Sources are acquired in an isolated subprocess (git URLs are probed with `git ls-remote` and shallow-cloned with a 3 GB guard; websites render in a stealth headless browser with an HTTP fallback, bounded by depth, file and byte caps), inventoried with exclusion rules, and scored by a multi-signal static engine (twenty-one detector families, 126 signal types across an origin axis and a quality-deficit axis). For a live website, the analyzer also opens the home page in a headless browser and inspects what actually renders: it fingerprints AI website-builder platforms directly (Lovable, Bolt.new, Framer and others) and checks the page's real computed styles, layout and build artifacts, not just its file contents. Representative files receive an AI review pass and images an AI-generation review through the internal gateway (model `molodetz`, internal key); the static engine stays authoritative when the gateway is unreachable. Every pipeline step is persisted as an ordered event trail and streamed live over the pub/sub topic `public.isslop.{uid}`. The verdict is an A-F authenticity grade with a human/AI split and one of five categories (`ai-slop`, `sophisticated-ai`, `human-clean`, `human-messy`, `uncertain`), published as a persistent report with an embeddable SVG badge. Members keep their analysis history on their account; guest history is session-bound and claimed by the account on the first signed-in visit. Admin settings (private-host allowance, AI/image review toggles, image cap, retention, concurrency) live on `/admin/services`. CLI: `devplace isslop analyze <url>` / `devplace isslop prune` / `devplace isslop clear`. Playwright plus playwright-stealth back the website crawler.
 
-`BackupService` powers the admin **Admin -> Backups** dashboard, an enterprise-grade backup system that runs entirely as asynchronous jobs so it never impacts the running server. An administrator can back up one of four targets: the **database** (a consistent SQLite snapshot of the main database and the Devii task/lesson databases, taken with SQLite's online backup API so it is consistent under WAL), **uploads** (every attachment and project file), **keys and config** (VAPID keys), or the **full data directory** (database snapshot, uploads, and keys in one archive, excluding regenerable staging, locks, caches, and container workspaces). Each backup is compressed to a `tar.gz` in a stdlib subprocess off the request path and recorded with its size, file count, and a SHA-256 checksum. Archives live under `data/backups/` (sharded on the random uuid tail) and are served only through `/admin/backups/{uid}/download`, which is restricted to the **primary administrator** - the first user created with the Admin role. Every other administrator receives a 403 from the endpoint and sees the Download button disabled with the tooltip `Not available`; creating, running, deleting, and scheduling backups remain available to all administrators. The dashboard reports detailed storage usage - the size and file count of every major data area, the total data-directory footprint, the total size and count of stored backups, and disk usage (total, used, free, percent), computed in a worker thread and cached briefly so the page never blocks. Backups can be **scheduled** (CRUD) on an interval or 5-field cron expression with a `keep_last` rotation count that prunes older backups of the same schedule; the service evaluates schedules only on the lock-owning worker so each fires exactly once. Backup archives are permanent operational artifacts: job retention only removes the tracking row, never the archive, which is deleted only by an administrator, by schedule rotation, or via the CLI. CLI: `devplace backups list` / `devplace backups run <target>` / `devplace backups prune` / `devplace backups clear`. Devii tools: `backups_overview`, `backup_run`, `backup_status`, `backup_delete`, `backup_schedule_create`, `backup_schedule_delete` (all admin-only). The service creates and stores backups but does not restore them into a live server; restore is a documented manual procedure (stop the server, unpack the archive over the data directory, verify the checksum, restart).
+`BackupService` powers the admin **Admin -> Backups** dashboard, an enterprise-grade backup system that runs entirely as asynchronous jobs so it never impacts the running server. An administrator can back up one of four targets: the **database** (a consistent SQLite snapshot of the main database and the Devii task/lesson databases, taken with SQLite's online backup API so it is consistent under WAL), **uploads** (every attachment and project file), **keys and config** (VAPID keys), or the **full data directory** (database snapshot, uploads, and keys in one archive, excluding regenerable staging, locks, caches, and container workspaces). Each backup is compressed to a `tar.gz` in a stdlib subprocess off the request path and recorded with its size, file count, and a SHA-256 checksum. Archives live under `data/backups/` (sharded on the random uuid tail) and are served only through `/admin/backups/{uid}/download`, which is restricted to the **primary administrator** - the first user created with the Admin role. Every other administrator receives a 403 from the endpoint and sees the Download button disabled with the tooltip `Not available`; creating, running, deleting, and scheduling backups remain available to all administrators. The dashboard reports detailed storage usage - the size and file count of every major data area, the total data-directory footprint, the total size and count of stored backups, and disk usage (total, used, free, percent). Disk percent is an O(1) volume stat used by the backup service tick; per-directory file counts are a single walk of the data directory, run in a worker thread and cached for minutes, so neither the request path nor the event loop ever walks the tree. Backups can be **scheduled** (CRUD) on an interval or 5-field cron expression with a `keep_last` rotation count that prunes older backups of the same schedule; the service evaluates schedules only on the lock-owning worker so each fires exactly once. Backup archives are permanent operational artifacts: job retention only removes the tracking row, never the archive, which is deleted only by an administrator, by schedule rotation, or via the CLI. CLI: `devplace backups list` / `devplace backups run <target>` / `devplace backups prune` / `devplace backups clear`. Devii tools: `backups_overview`, `backup_run`, `backup_status`, `backup_delete`, `backup_schedule_create`, `backup_schedule_delete` (all admin-only). The service creates and stores backups but does not restore them into a live server; restore is a documented manual procedure (stop the server, unpack the archive over the data directory, verify the checksum, restart).
 
 ### Adding a service
 
@@ -557,9 +604,10 @@ Configuration on the Services tab (`/admin/services`):
 |-----------|---------|---------|
 | `news_grade_threshold` | `7` | Minimum AI grade for auto-publish |
 | `news_api_url` | `https://news.app.molodetz.nl/api` | News source |
-| `news_ai_url` | `http://localhost:10500/openai/v1/chat/completions` | AI grading endpoint (the internal gateway) |
-| `news_ai_model` | `molodetz` | Generic model name; the gateway maps it to the real model |
-| `news_ai_key` | internal key | AI API key (`NEWS_AI_KEY` env, then the auto-generated gateway internal key) |
+| `news_ai_url` | `https://aquality.cloud.pravda.education/v1/chat/completions` | AI grading endpoint - a free, local, deterministic quality model by default |
+| `news_ai_model` | `aquality` | Generic model name; echoed back, not used for routing |
+| `news_ai_key` | internal key | AI API key (`NEWS_AI_KEY` env, then the auto-generated gateway internal key); unused by the default aquality endpoint |
+| `news_format_enabled` | `false` | Reformat article bodies into Markdown with AI after grading; off by default since aquality only scores and cannot generate text - enable when `news_ai_url` points at a generative chat model |
 | `news_service_interval` | `3600` | Seconds between fetch cycles (min 60) |
 
 News articles have detail pages at `/news/{slug}` with full comment support (same component as posts/projects). The landing page can display curated articles toggled from admin.
@@ -656,6 +704,7 @@ Configuration on the Services tab:
 | `gateway_upstream_url` | `https://api.deepseek.com/chat/completions` | Where requests are forwarded |
 | `gateway_model` | `deepseek-v4-flash` | Real model sent upstream (what `molodetz` maps to); 1M-token context, 384K max output |
 | `gateway_force_model` | on | Override the client-requested model (and the `molodetz` alias) |
+| `gateway_thinking` | off | When off, chat completions disable model thinking unless the client explicitly enables it (`think` / `thinking` / `reasoning`). Fastest default. |
 | `gateway_api_key` | migrated from env | Upstream key, shown and editable (auto-migrated from `DEEPSEEK_API_KEY`/`OPENROUTER_API_KEY`) |
 | `gateway_instances` | `4` | Max concurrent upstream forwards per worker (pool + semaphore) |
 | `gateway_timeout` | `300` | Upstream timeout (seconds); minimum five minutes; also bounds the vision describe-image call |
@@ -665,6 +714,8 @@ Configuration on the Services tab:
 | `gateway_allow_admins` / `gateway_allow_users` | on / off | Which DevPlace users may call it (any auth scheme) |
 | `gateway_access_key` | empty | A standalone key (sent as `X-API-KEY`/Bearer) that always grants access |
 | `gateway_internal_key` | auto (uuid4) | Auto-generated on boot; DevPlace's own services authenticate with this. Clear and restart to rotate |
+| `gateway_auth_throttle_enabled` | on | Track failed authentication attempts per IP and block further unauthenticated attempts once an IP crosses the failure threshold. Never blocks a request presenting valid credentials |
+| `gateway_auth_throttle_max_failures` / `_window_seconds` | 10 / 60 | Failed-auth attempts allowed per IP within the sliding window before further unauthenticated attempts get `429` |
 | `gateway_price_cache_hit_per_m` / `_cache_miss_per_m` / `_output_per_m` | 0.0028 / 0.14 / 0.28 | Chat cost per 1M tokens, used when the upstream returns no native cost (DeepSeek) |
 | `gateway_vision_price_input_per_m` / `_output_per_m` | 0 / 0 | Vision cost per 1M tokens, used only when the vision upstream returns no native cost |
 | `gateway_embed_price_input_per_m` | 0.01 | Embeddings cost per 1M input tokens, used only when the embeddings upstream returns no native cost |
@@ -919,13 +970,25 @@ receives a notification through every provider they hold a live subscription for
 | Provider | Registration | Transport |
 |----------|--------------|-----------|
 | `webpush` | `PushSubscription` from the browser `PushManager` (endpoint + `p256dh`/`auth` keys) | Web Push Protocol, VAPID signed, `aesgcm` encrypted payload |
-| `apns` | Hexadecimal device token | `POST https://api.push.apple.com/3/device/{token}` over HTTP/2, ES256 provider token |
+| `apns` | Hexadecimal device token, optional stable `client_id` | Dedicated HTTP/2 client to `api.push.apple.com` or `api.sandbox.push.apple.com` (`POST /3/device/{token}`), ES256 provider token. Not the stealth/Chrome client. |
 
 `POST /push.json` accepts a registration for any active provider; a body without a
-`provider` field is a `webpush` body, so browsers need no change. `GET /push.json` returns
-the VAPID public key plus the providers currently accepting registrations. A provider that
-is disabled or not fully configured accepts no registrations and is skipped during
-delivery, so an unconfigured provider is inert rather than an error.
+`provider` field is a `webpush` body, so browsers need no change. An APNs body is
+`{"provider": "apns", "token": "...", "client_id": "..."}` - `client_id` is optional and
+identifies the device across Apple token rotations, so a new token updates that row
+instead of inserting another. A token-only body still works and revives a previously
+dead token. `GET /push.json` returns the VAPID public key plus the providers currently
+accepting registrations; when `apns` is active it includes `environment` (`production` or
+`sandbox`). A provider that is disabled or not fully configured accepts no registrations
+and is skipped during delivery, so an unconfigured provider is inert rather than an error.
+
+`DELETE /push.json` unregisters exactly one registration, identified the same way it was
+created (`endpoint` for webpush, `token` or `client_id` for apns). It is idempotent - an
+unknown or already-removed identity still returns `200 {"unregistered": false}`. The web
+frontend calls it automatically before navigating to `/auth/logout` (`PushManager.js`), so a
+browser subscription stops receiving notifications the moment the user signs out; a native
+app integrating `apns` must call it itself at logout, since the server has no way to detect
+a native client closing on its own.
 
 Every provider setting is edited at **`/admin/services/push`**: per provider an `Enabled`
 toggle, the VAPID subject for `webpush`, and team id, key id, `.p8` auth key (stored as a
@@ -951,6 +1014,7 @@ in real time, bridged onto the in-process pub/sub bus by a lock-owner relay:
 | Direct message received | receiver |
 | Comment on your post | post author |
 | Reply to your comment | comment author |
+| Any comment on a post you've also commented on | every other commenter on that post |
 | `@mention` in any content | mentioned user |
 | Upvote on your content | content owner |
 | New follower | followed user |
@@ -960,13 +1024,15 @@ in real time, bridged onto the in-process pub/sub bus by a lock-owner relay:
 `create_notification` schedules delivery as a fire-and-forget async task, so a dead
 subscription or push-service error never blocks the triggering request. Delivery
 (`push.notify_user`) reads a user's subscriptions once, groups them by provider, builds
-each provider's payload once, and sends over a single shared HTTP client. A subscription
-the push service reports as gone (`404`/`410` for Web Push, `410` or an `Unregistered`
-class reason for APNs) is soft-deleted; any other failure is logged and the subscription is
-kept.
+each provider's payload once, and sends over that provider's own HTTP client (stealth for
+Web Push, a dedicated HTTP/2 client for APNs). A subscription the push service reports as
+gone (`404`/`410` for Web Push, `410` or an `Unregistered` class reason for APNs) is
+soft-deleted and the provider reason is logged; any other failure is logged and the
+subscription is kept. APNs environment is stored per registration so a sandbox debug
+token and a production token can coexist.
 
 A notification is also **marked read automatically when you open the page that shows its
-content** - viewing a post clears its comment, reply, upvote and mention notifications;
+content** - viewing a post clears its comment, reply, thread, upvote and mention notifications;
 opening a conversation clears its direct-message notifications; visiting a profile clears
 the matching follow, badge and level notifications; and the issue, reminder and farm-raid
 notifications clear on their respective pages. You no longer have to dismiss each one by
@@ -1022,7 +1088,7 @@ offline. Installation requires a secure origin (HTTPS, or `localhost` for develo
 |------|------|
 | `devplacepy/push/providers/` | Provider protocol, Web Push (VAPID keys, payload encryption), APNs |
 | `devplacepy/push/store.py` | `push_registration` reads and writes |
-| `devplacepy/push/delivery.py` | `notify_user` - group by provider, deliver, reap dead subscriptions |
+| `devplacepy/push/delivery.py` | `notify_user` / `notify_registration` - group by provider, per-provider client, reap dead subscriptions |
 | `devplacepy/services/push/service.py` | Provider configuration at `/admin/services/push`, retention sweep, metrics |
 | `devplacepy/routers/push.py` | `/push.json`, `/service-worker.js`, `/manifest.json` |
 | `static/js/PushManager.js` | Service-worker registration + subscribe + opt-in UI |
@@ -1169,13 +1235,13 @@ reverse_proxy localhost:10500 {
 
 ### Static asset caching
 
-Static assets (CSS, JS, vendored libraries) are served with a **one-year immutable cache** for the best Lighthouse "efficient cache policy" score, while deploys still take effect immediately. Every app-owned static URL carries a boot-time version path segment, `/static/v<timestamp>/...`, where `<timestamp>` is the unix time the server process started (`config.STATIC_VERSION`). A restart changes the segment, so every asset URL changes and returning browsers refetch on their next page load - no cache purge, no hashing build step.
+Static assets (CSS, JS, vendored libraries) are served with a **one-year immutable cache** for the best Lighthouse "efficient cache policy" score, while deploys still take effect immediately. Every app-owned static URL carries a version path segment, `/static/v<app-version>-<boot-id>/...` (`config.STATIC_VERSION`) - `<app-version>` is `pyproject.toml`'s `version` (auto-bumped on every commit, see "Version bumping" in `CLAUDE.md`) and `<boot-id>` is the unix time the server process started. A restart changes the segment, so every asset URL changes and returning browsers refetch on their next page load - no cache purge, no hashing build step.
 
 The version sits in the **path**, not a query string, because the frontend is unbundled ES6 modules wired with relative imports: a path segment is inherited automatically by every transitively imported module and relative CSS `url()`, so the whole graph busts on deploy. Templates emit URLs through the `static_url` Jinja global and runtime JavaScript through the `assetUrl` helper (`static/js/assetVersion.js`, reading `<meta name="asset-version">`). User uploads under `/static/uploads/` and the `service-worker.js` route are excluded. Set `DEVPLACE_STATIC_VERSION` at launch so multiple workers share one value (the `prod` target and Docker image do this). Full detail: `/docs/static-caching.html`.
 
 ### Bare-metal alternative
 
-`make prod` runs the same app without containers (`uvicorn ... --workers $(WEB_WORKERS) --proxy-headers`, where `WEB_WORKERS` defaults to `nproc`) from the project root, sharing the identical database and files. Note it binds port 10500, so it conflicts with the Docker front door on the same port - run one, or set a different `PORT`.
+`make prod` runs the same app without containers (`uvicorn ... --workers $(WEB_WORKERS) --proxy-headers`, where `WEB_WORKERS` defaults to `nproc`) from the project root, sharing the identical database and files. It binds port 10500 by default, so it conflicts with the Docker front door on the same port - run one, or set `DEVPLACE_PORT=<other-port> make prod` (also honoured by `make dev`).
 
 ### Multi-worker safety
 

devplacepy/attachments.py

@@ -3,6 +3,7 @@
 import asyncio
 import ipaddress
 import logging
+import os
 import socket
 from datetime import datetime, timezone
 from pathlib import Path
@@ -10,7 +11,7 @@ from urllib.parse import urlparse
 from PIL import Image
 from io import BytesIO
 import httpx
-from devplacepy import stealth
+from devplacepy.net_guard import BlockedAddressError, guarded_async_client
 from devplacepy.database import get_table, db, get_setting
 from devplacepy.config import UPLOADS_DIR, ATTACHMENTS_DIR
 from devplacepy.utils import generate_uid
@@ -389,7 +390,7 @@ async def fetch_remote_file(url, filename=None):
     await _guard_public_url(url)
     max_bytes = _get_max_upload_bytes()
     try:
-        async with stealth.stealth_async_client(
+        async with guarded_async_client(
             follow_redirects=True,
             timeout=REMOTE_FETCH_TIMEOUT,
             headers={"User-Agent": REMOTE_FETCH_USER_AGENT},
@@ -412,6 +413,8 @@ async def fetch_remote_file(url, filename=None):
                             413,
                         )
                 data = b"".join(chunks)
+    except BlockedAddressError as exc:
+        raise RemoteFetchError(str(exc), 400) from exc
     except httpx.HTTPError as exc:
         raise RemoteFetchError(f"Could not fetch {url}: {exc}", 400) from exc
 
@@ -453,6 +456,33 @@ def link_attachments(uids, target_type, target_uid):
         )
 
 
+def split_attachment_uids(raw):
+    return [
+        uid.strip() for item in raw or [] for uid in str(item).split(",") if uid.strip()
+    ]
+
+
+def get_orphan_attachments_batch(uids, user, admin=False):
+    if not uids:
+        return []
+    placeholders = ",".join(f":p{i}" for i in range(len(uids)))
+    params = {f"p{i}": uid for i, uid in enumerate(uids)}
+    rows = db.query(
+        f"SELECT * FROM attachments WHERE uid IN ({placeholders}) AND deleted_at IS NULL",
+        **params,
+    )
+    by_uid = {row["uid"]: row for row in rows}
+    owned = []
+    for uid in uids:
+        row = by_uid.get(uid)
+        if not row or row.get("target_uid"):
+            continue
+        if row.get("user_uid") and row["user_uid"] != user["uid"] and not admin:
+            continue
+        owned.append(uid)
+    return owned
+
+
 def set_gitea_asset_id(uid, asset_id):
     get_table("attachments").update(
         {"uid": uid, "gitea_asset_id": int(asset_id)}, ["uid"]
@@ -472,7 +502,7 @@ async def mirror_attachment_to_gitea(uid):
         return None
     path = ATTACHMENTS_DIR / row.get("directory", "") / row.get("stored_name", "")
     try:
-        data = path.read_bytes()
+        data = await asyncio.to_thread(path.read_bytes)
     except OSError as exc:
         logger.warning("Cannot read attachment %s for Gitea mirror: %s", uid, exc)
         return None
@@ -516,6 +546,28 @@ async def remove_gitea_asset(row):
         logger.warning("Gitea asset delete failed for %s: %s", row.get("uid"), exc)
 
 
+_pending_gitea_tasks: set[asyncio.Task] = set()
+
+
+def _fire_and_forget(coro) -> None:
+    try:
+        loop = asyncio.get_running_loop()
+    except RuntimeError:
+        coro.close()
+        return
+    task = loop.create_task(coro)
+    _pending_gitea_tasks.add(task)
+    task.add_done_callback(_pending_gitea_tasks.discard)
+
+
+def schedule_gitea_mirror(uid: str) -> None:
+    _fire_and_forget(mirror_attachment_to_gitea(uid))
+
+
+def schedule_gitea_removal(row: dict) -> None:
+    _fire_and_forget(remove_gitea_asset(row))
+
+
 def _unlink_attachment_files(row):
     stored_name = row.get("stored_name", "")
     directory = row.get("directory", "")
@@ -586,6 +638,73 @@ def restore_attachment(uid):
     return True
 
 
+def purge_soft_deleted_attachments(*, dry_run: bool = False) -> tuple[int, int]:
+    if "attachments" not in db.tables:
+        return 0, 0
+    table = get_table("attachments")
+    rows = list(table.find(table.table.columns.deleted_at.isnot(None)))
+    removed = 0
+    freed = 0
+    for row in rows:
+        directory = row.get("directory")
+        stored_name = row.get("stored_name")
+        if directory and stored_name:
+            path = ATTACHMENTS_DIR / directory / stored_name
+            try:
+                freed += path.stat().st_size
+            except OSError:
+                pass
+            for thumb in (ATTACHMENTS_DIR / directory).glob(
+                f"{Path(stored_name).stem}_thumb.*"
+            ):
+                try:
+                    freed += thumb.stat().st_size
+                except OSError:
+                    pass
+            if not dry_run:
+                _unlink_attachment_files(row)
+        if not dry_run:
+            table.delete(id=row["id"])
+        removed += 1
+    return removed, freed
+
+
+def sweep_orphan_attachment_blobs(*, dry_run: bool = False) -> tuple[int, int]:
+    if not ATTACHMENTS_DIR.exists():
+        return 0, 0
+    referenced_stems = set()
+    if "attachments" in db.tables:
+        for row in get_table("attachments").find():
+            directory = row.get("directory")
+            stored_name = row.get("stored_name")
+            if directory and stored_name:
+                referenced_stems.add((directory, Path(stored_name).stem))
+    removed = 0
+    freed = 0
+    base = str(ATTACHMENTS_DIR)
+    for root, _dirs, files in os.walk(base):
+        directory = os.path.relpath(root, base)
+        for name in files:
+            stem = Path(name).stem
+            if stem.endswith("_thumb"):
+                stem = stem[: -len("_thumb")]
+            if (directory, stem) in referenced_stems:
+                continue
+            file_path = os.path.join(root, name)
+            try:
+                size = os.path.getsize(file_path)
+            except OSError:
+                continue
+            if not dry_run:
+                try:
+                    os.unlink(file_path)
+                except OSError:
+                    continue
+            removed += 1
+            freed += size
+    return removed, freed
+
+
 def soft_delete_target_attachments(target_type, target_uid, deleted_by):
     stamp = datetime.now(timezone.utc).isoformat()
     for row in get_table("attachments").find(

devplacepy/cli/__init__.py

@@ -45,6 +45,7 @@ from devplacepy.cli.containers import (
 )
 from devplacepy.cli.quiz import cmd_quiz_prune
 from devplacepy.cli.migrate import cmd_emoji_sync, cmd_migrate_data
+from devplacepy.cli.system import cmd_system_prune
 
 __all__ = [
     "main",
@@ -91,4 +92,5 @@ __all__ = [
     "cmd_quiz_prune",
     "cmd_emoji_sync",
     "cmd_migrate_data",
+    "cmd_system_prune",
 ]

devplacepy/cli/containers.py

@@ -66,23 +66,27 @@ def cmd_containers_prune_builds(args):
     )
 
 
+def _human_bytes(n):
+    n = float(n)
+    for unit in ("B", "KB", "MB", "GB", "TB"):
+        if n < 1024:
+            return f"{n:.1f}{unit}"
+        n /= 1024
+    return f"{n:.1f}PB"
+
+
 def cmd_containers_gc_workspaces(args):
-    import shutil
-    from pathlib import Path
-    from devplacepy import config
     from devplacepy.services.containers import store
 
-    active = {inst["project_uid"] for inst in store.all_instances()}
-    base = Path(config.CONTAINER_WORKSPACES_DIR)
-    removed = 0
-    if base.is_dir():
-        for child in base.iterdir():
-            if child.is_dir() and child.name not in active:
-                shutil.rmtree(child, ignore_errors=True)
-                removed += 1
-    _audit_cli("cli.containers.gc_workspaces", f"CLI removed {removed} unused workspace dirs", metadata={"count": removed})
+    removed, freed = store.gc_workspaces()
+    _audit_cli(
+        "cli.containers.gc_workspaces",
+        f"CLI removed {removed} unused workspace dirs, freed {_human_bytes(freed)}",
+        metadata={"count": removed, "bytes_freed": freed},
+    )
     print(
         f"Removed {removed} unused workspace director{'y' if removed == 1 else 'ies'}"
+        f" ({_human_bytes(freed)})"
     )
 
 

devplacepy/cli/main.py

@@ -17,6 +17,7 @@ from devplacepy.cli.game import register_game
 from devplacepy.cli.quiz import register_quiz
 from devplacepy.cli.gateway import register_gateway
 from devplacepy.cli.messaging import register_messaging
+from devplacepy.cli.system import register_system
 
 
 def build_parser():
@@ -38,6 +39,7 @@ def build_parser():
     register_gateway(sub)
     register_messaging(sub)
     register_accounts(sub)
+    register_system(sub)
 
     return parser
 

devplacepy/cli/system.py

@@ -0,0 +1,202 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.cli._shared import _audit_cli
+
+RULE_WIDTH = 72
+
+
+def _rule(char: str = "=") -> str:
+    return char * RULE_WIDTH
+
+
+def _human(n) -> str:
+    n = float(n)
+    for unit in ("B", "KB", "MB", "GB", "TB"):
+        if n < 1024:
+            return f"{n:.1f}{unit}"
+        n /= 1024
+    return f"{n:.1f}PB"
+
+
+def _dir_stats(path) -> tuple[int, int]:
+    import os
+
+    count = 0
+    total = 0
+    for root, _dirs, files in os.walk(str(path)):
+        for name in files:
+            count += 1
+            try:
+                total += os.path.getsize(os.path.join(root, name))
+            except OSError:
+                continue
+    return count, total
+
+
+def _section(title: str) -> None:
+    print()
+    print(_rule())
+    print(title)
+    print(_rule())
+
+
+def _build_areas():
+    from pathlib import Path
+
+    from devplacepy import config
+    from devplacepy.attachments import (
+        ATTACHMENTS_DIR,
+        purge_soft_deleted_attachments,
+        sweep_orphan_attachment_blobs,
+    )
+    from devplacepy.project_files import (
+        PROJECT_FILES_DIR,
+        purge_soft_deleted_project_files,
+        sweep_orphan_project_file_blobs,
+    )
+    from devplacepy.services.containers import store as container_store
+
+    return [
+        {
+            "label": "Attachments",
+            "path": ATTACHMENTS_DIR,
+            "checks": [
+                ("soft-deleted attachments", purge_soft_deleted_attachments),
+                (
+                    "orphan attachment blobs (zero DB reference)",
+                    sweep_orphan_attachment_blobs,
+                ),
+            ],
+        },
+        {
+            "label": "Project files",
+            "path": PROJECT_FILES_DIR,
+            "checks": [
+                ("soft-deleted project files", purge_soft_deleted_project_files),
+                (
+                    "orphan project-file blobs (zero DB reference)",
+                    sweep_orphan_project_file_blobs,
+                ),
+            ],
+        },
+        {
+            "label": "Container workspaces",
+            "path": Path(config.CONTAINER_WORKSPACES_DIR),
+            "checks": [
+                (
+                    "orphaned workspace directories (no live instance)",
+                    container_store.gc_workspaces,
+                ),
+            ],
+        },
+    ]
+
+
+def cmd_system_prune(args):
+    import time
+
+    dry_run = bool(args.dry_run)
+    areas = _build_areas()
+
+    print("System prune - safe but aggressive disk-space cleanup")
+    print(
+        "Removes only content with zero live reference: soft-deleted attachment/"
+        "\nproject-file blobs, blob files with no matching database row at all"
+        "\n(orphans, e.g. left by an interrupted or racing sync), and orphaned"
+        "\ncontainer workspace directories. Never touches live content."
+    )
+    if dry_run:
+        print("DRY RUN: this pass only reports; nothing will be deleted.")
+
+    started = time.monotonic()
+
+    _section("Where the data is, and what can be reclaimed (estimate)")
+    estimate_items = 0
+    estimate_bytes = 0
+    for area in areas:
+        count, size = _dir_stats(area["path"])
+        print(f"\n{area['label']}")
+        print(f"  location: {area['path']}")
+        print(f"  currently on disk: {count} file(s), {_human(size)}")
+        for check_label, fn in area["checks"]:
+            c, f = fn(dry_run=True)
+            estimate_items += c
+            estimate_bytes += f
+            print(f"  expected to reclaim - {check_label}: {c} item(s), {_human(f)}")
+
+    print()
+    print(_rule("-"))
+    print(
+        f"Estimated total: {estimate_items} item(s), {_human(estimate_bytes)} reclaimable"
+    )
+    print(_rule("-"))
+
+    if dry_run:
+        print()
+        print("DRY RUN complete: nothing was deleted. Re-run without --dry-run to apply.")
+        return
+
+    _section("Executing")
+    actual_items = 0
+    actual_bytes = 0
+    for area in areas:
+        print(f"\n{area['label']} - {area['path']}")
+        for check_label, fn in area["checks"]:
+            c, f = fn(dry_run=False)
+            actual_items += c
+            actual_bytes += f
+            print(f"  removed - {check_label}: {c} item(s), {_human(f)}")
+
+    _section("After")
+    for area in areas:
+        count, size = _dir_stats(area["path"])
+        print(f"  {area['label']} ({area['path']}): {count} file(s), {_human(size)}")
+
+    elapsed = time.monotonic() - started
+
+    print()
+    print(_rule())
+    print(
+        f"Removed {actual_items} item(s) total, freed {_human(actual_bytes)} "
+        f"in {elapsed:.1f}s"
+    )
+    if estimate_bytes != actual_bytes or estimate_items != actual_items:
+        print(
+            f"(Estimate was {estimate_items} item(s), {_human(estimate_bytes)} - "
+            "state changed between the estimate and execution passes, e.g. the "
+            "live server wrote or soft-deleted something in between.)"
+        )
+    print(_rule())
+
+    _audit_cli(
+        "cli.system.prune",
+        f"CLI system prune removed {actual_items} item(s), freed {_human(actual_bytes)}",
+        metadata={
+            "items": actual_items,
+            "bytes_freed": actual_bytes,
+            "estimated_items": estimate_items,
+            "estimated_bytes": estimate_bytes,
+            "elapsed_seconds": round(elapsed, 3),
+        },
+    )
+
+
+def register_system(subparsers):
+    system = subparsers.add_parser("system", help="Cross-cutting system maintenance")
+    system_sub = system.add_subparsers(title="action", dest="action")
+    prune = system_sub.add_parser(
+        "prune",
+        help=(
+            "Safe but aggressive disk-space cleanup: purges soft-deleted "
+            "attachment/project-file blobs and any blob with zero database "
+            "reference at all, plus orphaned container workspace directories. "
+            "Reports location, current size, and estimated reclaim per area "
+            "before acting, then confirms what was actually freed."
+        ),
+    )
+    prune.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Report location, size, and estimated reclaim without deleting anything",
+    )
+    prune.set_defaults(func=cmd_system_prune)

devplacepy/config.py

@@ -1,6 +1,7 @@
 # retoor <retoor@molodetz.nl>
 
 import time
+import tomllib
 from pathlib import Path
 from dotenv import load_dotenv
 from os import environ
@@ -11,6 +12,8 @@ BASE_DIR = Path(__file__).resolve().parent.parent
 STATIC_DIR = BASE_DIR / "devplacepy" / "static"
 TEMPLATES_DIR = BASE_DIR / "devplacepy" / "templates"
 
+LOG_LEVEL = environ.get("DEVPLACE_LOG_LEVEL", "INFO").upper()
+
 DATA_DIR = Path(environ.get("DEVPLACE_DATA_DIR", str(BASE_DIR / "data")))
 UPLOADS_DIR = DATA_DIR / "uploads"
 ATTACHMENTS_DIR = UPLOADS_DIR / "attachments"
@@ -22,6 +25,15 @@ ZIP_STAGING_DIR = DATA_DIR / "zip_staging"
 FORK_STAGING_DIR = DATA_DIR / "fork_staging"
 BACKUPS_DIR = DATA_DIR / "backups"
 BACKUP_STAGING_DIR = DATA_DIR / "backup_staging"
+
+RCLONE_BIN = environ.get("DEVPLACE_RCLONE_BIN", "rclone")
+RCLONE_CONFIG_FILE = environ.get(
+    "DEVPLACE_RCLONE_CONFIG",
+    str(Path(environ.get("HOME", "/root")) / ".config" / "rclone" / "rclone.conf"),
+)
+BACKUP_OFFLOAD_REMOTE = environ.get(
+    "DEVPLACE_BACKUP_OFFLOAD_REMOTE", "storagebox:devplacepy-backups"
+)
 SEO_REPORTS_DIR = DATA_DIR / "seo_reports"
 PLANNING_REPORTS_DIR = DATA_DIR / "planning_reports"
 DBAPI_DIR = DATA_DIR / "dbapi"
@@ -45,7 +57,7 @@ SECRET_KEY = environ.get("SECRET_KEY", "devplace-secret-key-change-in-production
 SECONDS_PER_DAY = 86400
 SESSION_MAX_AGE = SECONDS_PER_DAY * 7
 SESSION_MAX_AGE_REMEMBER = SECONDS_PER_DAY * 30
-PORT = 10500
+PORT = int(environ.get("DEVPLACE_PORT", "10500"))
 SITE_URL = environ.get("DEVPLACE_SITE_URL", "").rstrip("/")
 
 PRESENCE_TIMEOUT_SECONDS = int(environ.get("DEVPLACE_PRESENCE_TIMEOUT_SECONDS", "60"))
@@ -59,7 +71,9 @@ PRESENCE_ONLINE_MARGIN_SECONDS = int(
 XMLRPC_BIND = environ.get("DEVPLACE_XMLRPC_BIND", "127.0.0.1")
 XMLRPC_PORT = int(environ.get("DEVPLACE_XMLRPC_PORT", "10550"))
 
-STATIC_VERSION = environ.get("DEVPLACE_STATIC_VERSION") or str(int(time.time()))
+APP_VERSION = tomllib.loads((BASE_DIR / "pyproject.toml").read_text())["project"]["version"]
+BOOT_ID = environ.get("DEVPLACE_STATIC_VERSION") or str(int(time.time()))
+STATIC_VERSION = f"{APP_VERSION}-{BOOT_ID}"
 
 TEMPLATE_AUTO_RELOAD = environ.get("DEVPLACE_TEMPLATE_AUTO_RELOAD", "1") != "0"
 
@@ -72,6 +86,9 @@ INTERNAL_MODEL = "molodetz"
 INTERNAL_EMBED_MODEL = "molodetz~embed"
 INTERNAL_IMAGE_MODEL = "molodetz-img-small"
 
+AQUALITY_NEWS_GRADING_URL = "https://aquality.cloud.pravda.education/v1/chat/completions"
+AQUALITY_NEWS_GRADING_MODEL = "aquality"
+
 AWARD_GIVE_COOLDOWN_HOURS_DEFAULT = 24
 AWARD_RECEIVE_COOLDOWN_HOURS_DEFAULT = 24
 AWARD_DISPLAY_HOURS_DEFAULT = 24

devplacepy/constants.py

@@ -2,6 +2,16 @@
 
 TOPICS = ["devlog", "showcase", "question", "rant", "fun", "random", "politics"]
 
+TOPIC_LABELS = {
+    "devlog": "Devlog",
+    "showcase": "Showcase",
+    "question": "Question",
+    "rant": "Rant",
+    "fun": "Fun",
+    "random": "Random",
+    "politics": "Politics",
+}
+
 REACTION_EMOJI = [
     "\U0001f44d",
     "❤️",

devplacepy/content.py

@@ -19,6 +19,7 @@ from devplacepy.database import (
     get_user_votes,
     get_reactions_by_targets,
     get_user_bookmarks,
+    get_user_notes,
     get_blocked_uids,
     get_poll_for_post,
     update_target_stars,
@@ -38,6 +39,8 @@ from devplacepy.database import (
     get_int_setting,
     _now_iso,
     db,
+    get_user_recent_items,
+    invalidate_user_recent_cache,
 )
 from devplacepy.utils import (
     time_ago,
@@ -46,6 +49,7 @@ from devplacepy.utils import (
     award_rewards,
     track_action,
     create_notification,
+    create_thread_notifications,
     create_mention_notifications,
     is_admin,
     is_primary_admin,
@@ -67,6 +71,7 @@ CREATE_METADATA_KEYS = ("project_type", "is_private", "language", "topic", "stat
 
 BOOKMARKABLE_TYPES = {"post", "gist", "project", "news", "quiz"}
 REACTABLE_TYPES = {"post", "comment", "gist", "project", "quiz"}
+NOTABLE_TYPES = {"post", "gist", "project", "news"}
 
 logger = logging.getLogger(__name__)
 
@@ -132,6 +137,18 @@ def can_view_project(project: dict | None, user: dict | None) -> bool:
     return not _owner_is_admin(project)
 
 
+def get_user_sidebar_gists(user_uid: str, limit: int = 5) -> list[dict]:
+    return get_user_recent_items("gists", user_uid)[:limit]
+
+
+def get_user_sidebar_projects(
+    user_uid: str, viewer: dict | None, limit: int = 5
+) -> list[dict]:
+    rows = get_user_recent_items("projects", user_uid)
+    visible = [p for p in rows if can_view_project(p, viewer)]
+    return visible[:limit]
+
+
 def owns_instance(
     instance: dict | None, project: dict | None, user: dict | None
 ) -> bool:
@@ -263,6 +280,7 @@ def create_content_item(
         from devplacepy.templating import clear_user_projects_cache
 
         clear_user_projects_cache(user["uid"])
+    invalidate_user_recent_cache(table_name, user["uid"])
     award_rewards(user["uid"], xp, badge)
     if attachment_uids:
         link_attachments(attachment_uids, target_type, uid)
@@ -309,10 +327,16 @@ def apply_vote(request, user: dict, target_type: str, target_uid: str, value: in
     existing = votes.find_one(
         user_uid=user["uid"], target_uid=target_uid, target_type=target_type
     )
-    old_value = int(existing["value"]) if existing else 0
+    old_value = int(existing["value"]) if existing and not existing.get("deleted_at") else 0
     did_upvote = False
     new_value = value
-    if existing:
+    if value == 0:
+        if existing and not existing.get("deleted_at"):
+            votes.update(
+                {"id": existing["id"], "deleted_at": _now_iso(), "deleted_by": user["uid"]},
+                ["id"],
+            )
+    elif existing:
         if existing.get("deleted_at"):
             votes.update(
                 {
@@ -438,6 +462,7 @@ def create_comment_record(
     comment_url = f"{redirect_url}#comment-{comment_uid}"
 
     if target_type == "post":
+        already_notified = {user["uid"]}
         if parent_uid:
             parent = get_table("comments").find_one(uid=parent_uid, deleted_at=None)
             if parent and parent["user_uid"] != user["uid"]:
@@ -448,6 +473,7 @@ def create_comment_record(
                     user["uid"],
                     comment_url,
                 )
+                already_notified.add(parent["user_uid"])
         else:
             posts = get_table("posts")
             post = posts.find_one(uid=target_uid)
@@ -461,6 +487,9 @@ def create_comment_record(
                     user["uid"],
                     comment_url,
                 )
+                already_notified.add(post["user_uid"])
+
+        create_thread_notifications(target_uid, user["uid"], comment_url, already_notified)
 
     create_mention_notifications(content, user["uid"], comment_url)
     record_screening(
@@ -627,6 +656,7 @@ def detail_context(
         "attachments": detail["attachments"],
         "reactions": detail.get("reactions", {"counts": {}, "mine": []}),
         "bookmarked": detail.get("bookmarked", False),
+        "note_content": detail.get("note_content"),
         "poll": detail.get("poll"),
         "war": detail.get("war"),
         "project_link": detail.get("project_link"),
@@ -672,6 +702,7 @@ def edit_content_item(
         "updated_at": datetime.now(timezone.utc).isoformat(),
     }
     table.update({"uid": item["uid"], **update_fields}, ["uid"])
+    invalidate_user_recent_cache(table_name, item["user_uid"])
     record_screening(
         screening,
         target_type=kind,
@@ -763,6 +794,7 @@ def delete_content_item(
         soft_delete_all_project_files(item["uid"], actor)
         soft_delete_fork_relations(item["uid"], actor)
         clear_user_projects_cache(item["user_uid"])
+    invalidate_user_recent_cache(table_name, item["user_uid"])
     soft_delete(table_name, actor, stamp=stamp, uid=item["uid"])
     logger.info(f"{table_name} {item['uid']} soft-deleted by {user['username']}")
     audit.record(
@@ -805,6 +837,11 @@ def load_detail(
         and target_type in BOOKMARKABLE_TYPES
         and item["uid"] in get_user_bookmarks(user["uid"], target_type, [item["uid"]])
     )
+    note_content = (
+        get_user_notes(user["uid"], target_type, [item["uid"]]).get(item["uid"])
+        if user and target_type in NOTABLE_TYPES
+        else None
+    )
     return {
         "item": item,
         "author": author,
@@ -818,6 +855,7 @@ def load_detail(
         "time_ago": time_ago(item["created_at"]),
         "reactions": reactions,
         "bookmarked": bookmarked,
+        "note_content": note_content,
         "poll": get_poll_for_post(item["uid"], user) if target_type == "post" else None,
         "war": war_store.get_war_serialized_for_post(item["uid"], user)
         if target_type == "post"

devplacepy/database/CLAUDE.md

@@ -124,7 +124,7 @@ Profile with the real database before and after any change here (`cProfile` arou
 
 Attachments (see "Profile media gallery and soft-deleted attachments" below) are one instance of a platform-wide model: **every removal is a soft delete; only garbage collection is a hard delete.** Every removable row carries two columns, `deleted_at` (ISO timestamp) and `deleted_by` (actor uid, or `system`). A live row has `deleted_at = NULL`, and every list/count read filters `deleted_at IS NULL`.
 
-- **Tables (`database.SOFT_DELETE_TABLES`):** posts, comments, gists, projects, news, news_images, project_files, attachments, votes, reactions, bookmarks, follows, poll_votes, polls, poll_options, sessions, instances, instance_schedules, devii_conversations, devii_tasks, devii_lessons, devii_virtual_tools, user_customizations, project_forks. `init_db` loops `ensure_soft_delete_columns(t)` for each (adds both columns + `idx_<t>_deleted` / the partial trash index described above). The Devii task/lesson/virtual-tool stores run on their own DB handles, so their `_ensure_indexes` adds the columns there too.
+- **Tables (`database.SOFT_DELETE_TABLES`):** posts, comments, gists, projects, news, news_images, project_files, attachments, votes, reactions, bookmarks, notes, follows, poll_votes, polls, poll_options, sessions, instances, instance_schedules, devii_conversations, devii_tasks, devii_lessons, devii_virtual_tools, user_customizations, project_forks. `init_db` loops `ensure_soft_delete_columns(t)` for each (adds both columns + `idx_<t>_deleted` / the partial trash index described above). The Devii task/lesson/virtual-tool stores run on their own DB handles, so their `_ensure_indexes` adds the columns there too.
 - **Every INSERT into a soft-deletable table MUST write `deleted_at: None, deleted_by: None`.** `dataset.find(deleted_at=None)` on a table that lacks the column matches NOTHING (a false predicate), silently hiding all rows - the born-live insert is what creates the column. Add the pair to any new insert.
 - **Central helpers (`database/`):** `soft_delete(table, deleted_by, *, stamp=None, **criteria)` (equality), `soft_delete_in(table, column, uids, deleted_by, *, stamp=None, **extra)` (IN-clause cascade), `restore(table, **criteria)`, `purge(table, **criteria)` (real delete), `list_deleted(table, page)` / `count_deleted(table)` (trash listings), and the event helpers `restore_event(stamp)` / `purge_event(stamp)` that act across ALL tables sharing one `deleted_at` stamp.
 - **Two generic chokepoints are conditionally filtered:** `resolve_by_slug(table, slug, include_deleted=False)` (detail-page lookups; restore passes `include_deleted=True`) and `paginate(table, ...)` (auto-appends `deleted_at IS NULL` when the table has the column and the caller did not pass `deleted_at`). `seo._collect` does the same for the sitemap. Read filters were threaded through every batch helper, analytics/activity/leaderboard UNION, feed/profile/listing route, and store; never re-introduce an unfiltered read of a soft-deletable table.
@@ -198,8 +198,8 @@ Site settings are seeded on startup (`site_settings` table):
 | `site_name` / `site_description` / `site_tagline` | DevPlace branding | General site metadata |
 | `news_grade_threshold` | `"7"` | Minimum AI grade for auto-publish |
 | `news_api_url` | `"https://news.app.molodetz.nl/api"` | News source API |
-| `news_ai_url` | `"https://openai.app.molodetz.nl/v1/chat/completions"` | AI grading endpoint |
-| `news_ai_model` | `"molodetz"` | AI model identifier |
+| `news_ai_url` | `"https://aquality.cloud.pravda.education/v1/chat/completions"` | AI grading endpoint - the free, local aquality quality model by default (see `devplacepy/services/news/CLAUDE.md`) |
+| `news_ai_model` | `"aquality"` | AI model identifier |
 | `max_upload_size_mb` / `allowed_file_types` / `max_attachments_per_resource` | `"10"` / `""` / `"10"` | Upload limits |
 | `rate_limit_per_minute` | `"60"` | Mutating requests per IP per window (`main.py` middleware); a `429` carries a `Retry-After: <window>` header |
 | `rate_limit_window_seconds` | `"60"` | Rolling window for the rate limit (also the `Retry-After` value) |
@@ -211,6 +211,7 @@ Site settings are seeded on startup (`site_settings` table):
 | `maintenance_message` | scheduled-maintenance text | Body shown on the maintenance 503 page |
 | `customization_enabled` | `"1"` | When `"0"`, `custom_css_tag`/`custom_js_tag` inject nothing (feature off) |
 | `customization_js_enabled` | `"1"` | When `"0"`, custom CSS still serves but custom JS is suppressed |
+| `happy_404_enabled` | `"1"` | When `"0"`, `happy404.render()` is a no-op and a real 404 page is shown; when on, an HTML 404 renders a random existing post instead (see below) |
 | `moderation_sla_hours` | `"24"` | The published moderation response window; the admin queue badge turns red past it |
 | `moderation_filter_mode` | `"review"` | `off`/`label`/`review`/`block` - how the content filter acts on a match |
 | `moderation_filter_review_score` | `"2"` | Rule score at which a match becomes a report rather than a label |
@@ -239,6 +240,7 @@ Operational settings - read sites and rules:
 | `service_<name>_enabled` / `service_<name>_command` / `service_<name>_log_size` | `BaseService` reconciling loop | Generic per-service controls written by the Services tab; the loop reconciles within ~1s |
 | `session_max_age_days` / `session_remember_days` | `auth.py` signup + login | Multiplied by `SECONDS_PER_DAY`; passed to `create_session(uid, max_age)` so the cookie and the DB session row expire together |
 | `registration_open` | `auth.py` `signup_page` (GET) and `signup` (POST) | POST returns before any DB write when closed |
+| `happy_404_enabled` | `happy404.render()`, called from the `404` handler in `main.py` | Gates the whole feature; checked per-request via the normal 60s `get_setting` TTL cache, so a toggle takes effect within a minute across all workers with no restart |
 
 **Booleans are `<select>`, never checkboxes.** The settings save handler (`admin.py`) skips empty form values so empty fields don't clobber existing rows. An unchecked checkbox submits nothing, so it could never be turned off - `registration_open` and `maintenance_mode` use `<option value="1">`/`<option value="0">` so a value is always submitted.
 

devplacepy/database/__init__.py

@@ -8,7 +8,7 @@ from .users import get_users_by_uids, _admins_cache, invalidate_admins_cache, ge
 from .relations import _relations_cache, get_user_relations, get_blocked_uids, get_muted_uids, get_silenced_uids, invalidate_user_relations
 from .pagination import PAGE_SIZE, paginate, interleave_by_author, paginate_diverse, get_user_post_count, clear_user_post_count, build_pagination
 from .soft_delete import SOFT_DELETE_TABLES, ensure_soft_delete_columns, soft_delete, soft_delete_in, restore, purge, list_deleted, count_deleted, restore_event, purge_event
-from .engagement import _comment_count_cache, get_comment_counts_by_post_uids, get_post_counts_by_user_uids, get_vote_counts, get_user_votes, get_reactions_by_targets, get_user_bookmarks, get_polls_by_post_uids, get_poll_for_post
+from .engagement import _comment_count_cache, get_comment_counts_by_post_uids, get_post_counts_by_user_uids, get_vote_counts, get_user_votes, get_reactions_by_targets, get_user_bookmarks, get_user_notes, get_polls_by_post_uids, get_poll_for_post
 from .usage import _add_usage, _get_usage, add_correction_usage, get_correction_usage, add_modifier_usage, get_modifier_usage, NEWS_USAGE_KEY, add_news_usage, get_news_usage, ISSUE_USAGE_KEY, add_issue_usage, get_issue_usage, SEO_USAGE_KEY, add_seo_usage, get_seo_usage, AWARD_USAGE_KEY, add_award_usage, get_award_usage
 from .awards import (
     AWARDS_PER_PAGE,
@@ -34,7 +34,7 @@ from .email import EMAIL_ACCOUNT_DEFAULTS, list_email_accounts, get_email_accoun
 from .notifications import NOTIFICATION_TYPES, NOTIFICATION_CHANNELS, _NOTIFICATION_CHANNEL_COLUMNS, _NOTIFICATION_CHANNEL_DEFAULTS, _NOTIFICATION_TYPE_KEYS, _notification_prefs_cache, _notification_default, get_notification_default, set_notification_default, _notification_overrides, notification_enabled, get_notification_prefs, set_notification_pref, reset_notification_prefs, mark_notifications_read_by_target
 from .forks import record_fork, get_fork_parent, count_forks, soft_delete_fork_relations, delete_fork_relations
 from .follows import get_follow_counts, get_follow_list, get_following_among
-from .deepsearch import _ds_now, create_deepsearch_session, update_deepsearch_session, get_deepsearch_session, add_deepsearch_message, get_deepsearch_messages, get_cached_deepsearch_url, upsert_deepsearch_url_cache
+from .deepsearch import _ds_now, create_deepsearch_session, update_deepsearch_session, get_deepsearch_session, list_deepsearch_sessions, add_deepsearch_message, get_deepsearch_messages, get_cached_deepsearch_url, upsert_deepsearch_url_cache
 from .ranking import VOTABLE_TARGETS, STAR_TARGETS, _authors_cache, _ranked_authors, _rank_map, get_top_authors, get_leaderboard, get_user_rank, get_user_stars, clear_user_stars, update_target_stars, soft_delete_engagement, delete_engagement, get_target_owner_uid
 from .moderation import (
     ACTIONS_TABLE,
@@ -74,7 +74,7 @@ from .moderation import (
     years_between,
 )
 from .comments import _drop_blocked, _build_comment_items, load_comments, get_recent_comments_by_target_uids, get_recent_comments_by_post_uids, load_comments_by_target_uids
-from .content import resolve_by_slug, resolve_object_url, get_uids_by_username_match, text_search_clause, get_daily_topic, get_featured_news, get_trending_topics
+from .content import resolve_by_slug, resolve_object_url, get_projects_by_uids, get_uids_by_username_match, text_search_clause, get_daily_topic, get_featured_news, get_featured_topics, get_trending_topics, get_user_recent_items, invalidate_user_recent_cache
 from .attachments_data import get_attachments, get_attachments_by_type, get_news_images_by_uids, delete_attachment_record, delete_attachments, _delete_attachment_file, get_user_media, get_user_attachments, get_user_attachment, get_deleted_media
 from .stats import _stats_cache, get_site_stats, _analytics_cache, get_platform_analytics, _gist_languages_cache, get_gist_languages
 from .schema import BUG_TABLE_RENAMES, migrate_bug_tables_to_issue_tables, init_db, _refresh_query_planner_stats, OLD_GATEWAY_URL, migrate_ai_gateway_settings, backfill_api_keys, _backfill_gamification
@@ -157,6 +157,7 @@ __all__ = [
     "get_user_votes",
     "get_reactions_by_targets",
     "get_user_bookmarks",
+    "get_user_notes",
     "get_polls_by_post_uids",
     "get_poll_for_post",
     "_add_usage",
@@ -236,6 +237,7 @@ __all__ = [
     "create_deepsearch_session",
     "update_deepsearch_session",
     "get_deepsearch_session",
+    "list_deepsearch_sessions",
     "add_deepsearch_message",
     "get_deepsearch_messages",
     "get_cached_deepsearch_url",
@@ -296,10 +298,14 @@ __all__ = [
     "load_comments_by_target_uids",
     "resolve_by_slug",
     "resolve_object_url",
+    "get_projects_by_uids",
     "get_uids_by_username_match",
     "text_search_clause",
     "get_daily_topic",
     "get_featured_news",
+    "get_featured_topics",
+    "get_user_recent_items",
+    "invalidate_user_recent_cache",
     "get_trending_topics",
     "get_attachments",
     "get_attachments_by_type",

devplacepy/database/content.py

@@ -1,5 +1,7 @@
 # retoor <retoor@molodetz.nl>
 
+import os
+import random
 from collections import Counter
 
 from devplacepy.cache import TTLCache
@@ -8,6 +10,34 @@ from .core import db, get_table, or_
 
 _daily_topic_cache = TTLCache(ttl=60, max_size=1)
 _trending_cache = TTLCache(ttl=15, max_size=1)
+FEATURED_TOPICS_POOL_TTL = int(os.environ.get("DEVPLACE_FEATURED_TOPICS_POOL_TTL", "300"))
+FEATURED_TOPICS_POOL_SIZE = 20
+_featured_topics_cache = TTLCache(ttl=FEATURED_TOPICS_POOL_TTL, max_size=1)
+
+USER_RECENT_ITEMS_TTL = int(os.environ.get("DEVPLACE_USER_RECENT_ITEMS_TTL", "15"))
+_user_recent_cache = TTLCache(ttl=USER_RECENT_ITEMS_TTL, max_size=2000)
+
+
+def _user_recent_key(table_name: str, user_uid: str) -> str:
+    return f"{table_name}:{user_uid}"
+
+
+def invalidate_user_recent_cache(table_name: str, user_uid: str) -> None:
+    _user_recent_cache.pop(_user_recent_key(table_name, user_uid))
+
+
+def get_user_recent_items(table_name: str, user_uid: str) -> list[dict]:
+    key = _user_recent_key(table_name, user_uid)
+    cached = _user_recent_cache.get(key)
+    if cached is not None:
+        return cached
+    items = []
+    if table_name in db.tables:
+        rows = list(get_table(table_name).find(user_uid=user_uid, deleted_at=None))
+        rows.sort(key=lambda row: row.get("updated_at") or row.get("created_at") or "", reverse=True)
+        items = rows
+    _user_recent_cache.set(key, items)
+    return items
 
 
 def resolve_by_slug(table, slug, include_deleted=False):
@@ -102,6 +132,17 @@ def resolve_object_url(target_type: str, target_uid: str) -> str:
     return "/feed"
 
 
+def get_projects_by_uids(uids):
+    if not uids or "projects" not in db.tables:
+        return {}
+    projects = get_table("projects")
+    if "uid" not in projects.columns:
+        return {}
+    seen = set()
+    unique = [u for u in uids if u not in seen and not seen.add(u)]
+    return {p["uid"]: p for p in projects.find(projects.table.columns.uid.in_(unique))}
+
+
 def get_uids_by_username_match(search, limit=200):
     term = (search or "").strip()
     if not term or "users" not in db.tables:
@@ -160,6 +201,49 @@ def _load_daily_topic():
     }
 
 
+def get_featured_topics(count: int = 3) -> list[dict]:
+    pool = _featured_topics_pool()
+    if not pool:
+        return []
+    return random.sample(pool, min(count, len(pool)))
+
+
+def _featured_topics_pool() -> list[dict]:
+    cached = _featured_topics_cache.get("pool")
+    if cached is not None:
+        return cached
+    pool = _load_featured_topics_pool()
+    _featured_topics_cache.set("pool", pool)
+    return pool
+
+
+def _load_featured_topics_pool(limit: int = FEATURED_TOPICS_POOL_SIZE) -> list[dict]:
+    if "news" not in db.tables:
+        return []
+    rows = db["news"].find(
+        featured=1,
+        status="published",
+        deleted_at=None,
+        order_by=["-synced_at"],
+        _limit=limit,
+    )
+    topics = []
+    for article in rows:
+        desc = (article.get("description") or "")[:160] or (
+            article.get("content") or ""
+        )[:160]
+        topics.append(
+            {
+                "title": article.get("title", ""),
+                "summary": desc,
+                "slug": article.get("slug", ""),
+                "url": article.get("url", ""),
+                "image_url": article.get("image_url", "") or "",
+            }
+        )
+    return topics
+
+
 def get_featured_news(limit=5):
     if "news" not in db.tables:
         return []

devplacepy/database/core.py

@@ -9,6 +9,8 @@ from collections import defaultdict
 from datetime import datetime, timedelta, timezone
 from devplacepy.cache import TTLCache
 from devplacepy.config import (
+    AQUALITY_NEWS_GRADING_MODEL,
+    AQUALITY_NEWS_GRADING_URL,
     DATABASE_URL,
     DEFAULT_CORRECTION_PROMPT,
     DEFAULT_MODIFIER_PROMPT,

devplacepy/database/deepsearch.py

@@ -54,6 +54,20 @@ def get_deepsearch_session(uid: str) -> dict | None:
     return get_table("deepsearch_sessions").find_one(uid=uid, deleted_at=None)
 
 
+def list_deepsearch_sessions(owner_kind: str, owner_id: str, limit: int = 50) -> list[dict]:
+    if "deepsearch_sessions" not in db.tables:
+        return []
+    return list(
+        get_table("deepsearch_sessions").find(
+            owner_kind=owner_kind,
+            owner_id=owner_id,
+            deleted_at=None,
+            order_by=["-created_at"],
+            _limit=limit,
+        )
+    )
+
+
 def add_deepsearch_message(
     uid: str, session_uid: str, role: str, content: str, citations: str = ""
 ) -> None:

devplacepy/database/engagement.py

@@ -116,6 +116,19 @@ def get_user_bookmarks(user_uid, target_type, target_uids):
     return {row["target_uid"] for row in rows}
 
 
+def get_user_notes(user_uid, target_type, target_uids):
+    if not user_uid or not target_uids or "notes" not in db.tables:
+        return {}
+    placeholders, params = _in_clause(target_uids)
+    params["u"] = user_uid
+    params["tt"] = target_type
+    rows = db.query(
+        f"SELECT target_uid, content FROM notes WHERE user_uid=:u AND target_type=:tt AND target_uid IN ({placeholders}) AND deleted_at IS NULL",
+        **params,
+    )
+    return {row["target_uid"]: row["content"] for row in rows}
+
+
 def get_polls_by_post_uids(post_uids, user=None):
     if not post_uids or "polls" not in db.tables:
         return {}

devplacepy/database/moderation.py

@@ -47,6 +47,7 @@ UNREPORTABLE_TABLES: dict[str, str] = {
     "votes": "engagement counters, carry no authored content",
     "reactions": "engagement counters, carry no authored content",
     "bookmarks": "private to the owner",
+    "notes": "private to the owner",
     "follows": "relationship rows, carry no authored content",
     "poll_votes": "private ballots",
     "opinion_war_fighters": "membership and damage counters, carry no authored content",

devplacepy/database/notifications.py

@@ -8,6 +8,7 @@ from .soft_delete import soft_delete
 NOTIFICATION_TYPES = [
     {"key": "comment", "label": "Comments", "description": "Someone comments on your post"},
     {"key": "reply", "label": "Replies", "description": "Someone replies to your comment"},
+    {"key": "thread", "label": "Thread activity", "description": "Someone else comments on a post you've commented on"},
     {"key": "mention", "label": "Mentions", "description": "Someone mentions you with @username"},
     {"key": "vote", "label": "Upvotes", "description": "Someone ++'d your content"},
     {"key": "follow", "label": "Followers", "description": "Someone starts following you"},
@@ -23,6 +24,7 @@ NOTIFICATION_TYPES = [
     {"key": "workspace", "label": "Dev workspaces", "description": "Idle, quota, retention and moderation notices for your dev workspaces"},
     {"key": "moderation", "label": "Moderation", "description": "Acknowledgements for reports you file and decisions taken on your content"},
     {"key": "system", "label": "System alerts", "description": "Platform infrastructure alerts (e.g. the AI gateway going down)"},
+    {"key": "ai_quota_warning", "label": "AI quota warnings", "description": "You're approaching your daily Devii AI usage limit"},
 ]
 
 

devplacepy/database/pagination.py

@@ -18,6 +18,7 @@ def paginate(
     order=None,
     cursor_field="created_at",
     viewer_uid=None,
+    limit=PAGE_SIZE,
     **filters,
 ):
     order = order or ["-" + cursor_field]
@@ -30,9 +31,9 @@ def paginate(
             clauses.append(table.table.columns.user_uid.notin_(blocked))
     if before:
         clauses.append(table.table.columns[cursor_field] < before)
-    rows = list(table.find(*clauses, **filters, order_by=order, _limit=PAGE_SIZE + 1))
-    has_more = len(rows) > PAGE_SIZE
-    rows = rows[:PAGE_SIZE]
+    rows = list(table.find(*clauses, **filters, order_by=order, _limit=limit + 1))
+    has_more = len(rows) > limit
+    rows = rows[:limit]
     next_cursor = rows[-1][cursor_field] if has_more and rows else None
     return rows, next_cursor
 
@@ -64,6 +65,7 @@ def paginate_diverse(
     cursor_field="created_at",
     uid_key="user_uid",
     viewer_uid=None,
+    limit=PAGE_SIZE,
     **filters,
 ):
     rows, next_cursor = paginate(
@@ -73,6 +75,7 @@ def paginate_diverse(
         order=order,
         cursor_field=cursor_field,
         viewer_uid=viewer_uid,
+        limit=limit,
         **filters,
     )
     return interleave_by_author(rows, uid_key=uid_key), next_cursor

devplacepy/database/ranking.py

@@ -4,7 +4,7 @@ import os
 
 from .core import TTLCache, _in_clause, _now_iso, db, get_table
 from .users import get_users_by_uids
-from .soft_delete import soft_delete, soft_delete_in
+from .soft_delete import soft_delete_in
 
 
 VOTABLE_TARGETS: dict[str, str] = {
@@ -133,6 +133,26 @@ def update_target_stars(target_type: str, target_uid: str, net_stars: int) -> No
         get_table(table_name).update({"uid": target_uid, "stars": net_stars}, ["uid"])
 
 
+def _child_uids(table_name, parent_column, parent_uids, live_only=False):
+    if table_name not in db.tables:
+        return []
+    table = db[table_name]
+    if parent_column not in table.columns:
+        return []
+    clause = table.table.columns[parent_column].in_(parent_uids)
+    if live_only:
+        return [row["uid"] for row in table.find(clause, deleted_at=None)]
+    return [row["uid"] for row in table.find(clause)]
+
+
+def _delete_in(table_name, column, uids):
+    placeholders, params = _in_clause(uids)
+    with db:
+        db.query(
+            f"DELETE FROM {table_name} WHERE {column} IN ({placeholders})", **params
+        )
+
+
 def soft_delete_engagement(target_type: str, target_uids: list, deleted_by: str) -> None:
     uids = [uid for uid in (target_uids or []) if uid]
     if not uids:
@@ -145,21 +165,15 @@ def soft_delete_engagement(target_type: str, target_uids: list, deleted_by: str)
         "bookmarks", "target_uid", uids, deleted_by, stamp=stamp, target_type=target_type
     )
     if target_type == "post" and "polls" in db.tables:
-        for uid in uids:
-            for poll in db["polls"].find(post_uid=uid, deleted_at=None):
-                soft_delete("poll_votes", deleted_by, stamp=stamp, poll_uid=poll["uid"])
-                soft_delete("poll_options", deleted_by, stamp=stamp, poll_uid=poll["uid"])
-            soft_delete("polls", deleted_by, stamp=stamp, post_uid=uid)
+        poll_uids = _child_uids("polls", "post_uid", uids, live_only=True)
+        soft_delete_in("poll_votes", "poll_uid", poll_uids, deleted_by, stamp=stamp)
+        soft_delete_in("poll_options", "poll_uid", poll_uids, deleted_by, stamp=stamp)
+        soft_delete_in("polls", "post_uid", uids, deleted_by, stamp=stamp)
     if target_type == "post" and "opinion_wars" in db.tables:
-        for uid in uids:
-            for war in db["opinion_wars"].find(post_uid=uid, deleted_at=None):
-                soft_delete(
-                    "opinion_war_fighters", deleted_by, stamp=stamp, war_uid=war["uid"]
-                )
-                soft_delete(
-                    "opinion_war_events", deleted_by, stamp=stamp, war_uid=war["uid"]
-                )
-            soft_delete("opinion_wars", deleted_by, stamp=stamp, post_uid=uid)
+        war_uids = _child_uids("opinion_wars", "post_uid", uids, live_only=True)
+        soft_delete_in("opinion_war_fighters", "war_uid", war_uids, deleted_by, stamp=stamp)
+        soft_delete_in("opinion_war_events", "war_uid", war_uids, deleted_by, stamp=stamp)
+        soft_delete_in("opinion_wars", "post_uid", uids, deleted_by, stamp=stamp)
 
 
 def delete_engagement(target_type: str, target_uids: list) -> None:
@@ -184,21 +198,21 @@ def delete_engagement(target_type: str, target_uids: list) -> None:
                 **params,
             )
     if target_type == "post" and "polls" in tables:
-        for uid in uids:
-            for poll in db["polls"].find(post_uid=uid):
-                if "poll_votes" in tables:
-                    db["poll_votes"].delete(poll_uid=poll["uid"])
-                if "poll_options" in tables:
-                    db["poll_options"].delete(poll_uid=poll["uid"])
-            db["polls"].delete(post_uid=uid)
+        poll_uids = _child_uids("polls", "post_uid", uids)
+        if poll_uids:
+            if "poll_votes" in tables:
+                _delete_in("poll_votes", "poll_uid", poll_uids)
+            if "poll_options" in tables:
+                _delete_in("poll_options", "poll_uid", poll_uids)
+        _delete_in("polls", "post_uid", uids)
     if target_type == "post" and "opinion_wars" in tables:
-        for uid in uids:
-            for war in db["opinion_wars"].find(post_uid=uid):
-                if "opinion_war_fighters" in tables:
-                    db["opinion_war_fighters"].delete(war_uid=war["uid"])
-                if "opinion_war_events" in tables:
-                    db["opinion_war_events"].delete(war_uid=war["uid"])
-            db["opinion_wars"].delete(post_uid=uid)
+        war_uids = _child_uids("opinion_wars", "post_uid", uids)
+        if war_uids:
+            if "opinion_war_fighters" in tables:
+                _delete_in("opinion_war_fighters", "war_uid", war_uids)
+            if "opinion_war_events" in tables:
+                _delete_in("opinion_war_events", "war_uid", war_uids)
+        _delete_in("opinion_wars", "post_uid", uids)
 
 
 def get_target_owner_uid(target_type: str, target_uid: str) -> str | None:

devplacepy/database/schema.py

@@ -1,6 +1,6 @@
 # retoor <retoor@molodetz.nl>
 
-from .core import DEFAULT_CORRECTION_PROMPT, DEFAULT_MODIFIER_PROMPT, INTERNAL_GATEWAY_URL, _drop_index, _index, _uid_index, db, defaultdict, get_table, logger
+from .core import AQUALITY_NEWS_GRADING_MODEL, AQUALITY_NEWS_GRADING_URL, DEFAULT_CORRECTION_PROMPT, DEFAULT_MODIFIER_PROMPT, INTERNAL_GATEWAY_URL, _drop_index, _index, _uid_index, db, defaultdict, get_table, logger
 from .settings import get_setting, set_setting
 from .soft_delete import SOFT_DELETE_TABLES, ensure_soft_delete_columns
 from .ranking import _authors_cache
@@ -30,6 +30,54 @@ def migrate_bug_tables_to_issue_tables() -> None:
         logger.info("Dropped table %s after migration", source_name)
 
 
+_TUNNEL_STATUS_RANK = {
+    "active": 4,
+    "provisioning": 3,
+    "pending": 2,
+    "failed": 1,
+    "suspended": 0,
+}
+
+
+def _dedupe_tunnel_hostnames() -> None:
+    if "tunnels" not in db.tables:
+        return
+    table = get_table("tunnels")
+    if not table.has_column("hostname") or not table.has_column("uid"):
+        return
+    groups = list(
+        db.query(
+            "SELECT hostname FROM tunnels "
+            "WHERE hostname IS NOT NULL AND hostname != '' "
+            "GROUP BY hostname HAVING COUNT(*) > 1"
+        )
+    )
+    for group in groups:
+        hostname = group["hostname"]
+        dupes = list(table.find(hostname=hostname))
+        dupes.sort(
+            key=lambda r: (
+                _TUNNEL_STATUS_RANK.get(r.get("status") or "", -1),
+                r.get("deleted_at") is None,
+                r.get("created_at") or "",
+                r.get("id") or 0,
+            ),
+            reverse=True,
+        )
+        losers = [r["uid"] for r in dupes[1:]]
+        if not losers:
+            continue
+        with db:
+            for uid in losers:
+                db.query("DELETE FROM tunnels WHERE uid = :uid", uid=uid)
+        logger.warning(
+            "Removed %d duplicate tunnel row(s) for hostname %s, kept %s",
+            len(losers),
+            hostname,
+            dupes[0]["uid"],
+        )
+
+
 def init_db():
     tables = db.tables
     _index(db, "users", "idx_users_username", ["username"])
@@ -38,6 +86,7 @@ def init_db():
     _index(db, "users", "idx_users_role", ["role", "created_at"])
     _index(db, "users", "idx_users_last_seen", ["last_seen"])
     _index(db, "users", "idx_users_created_at", ["created_at"])
+    _ensure_users_uid()
     _index(db, "posts", "idx_posts_user_uid", ["user_uid"])
     _index(db, "posts", "idx_posts_created_at", ["created_at"])
     _index(db, "posts", "idx_posts_topic", ["topic"])
@@ -129,6 +178,8 @@ def init_db():
         ("content", ""),
         ("read", False),
         ("created_at", ""),
+        ("updated_at", ""),
+        ("client_id", ""),
     ):
         if not messages.has_column(column):
             messages.create_column_by_example(column, example)
@@ -143,6 +194,8 @@ def init_db():
         "idx_messages_conversation_rev",
         ["receiver_uid", "sender_uid"],
     )
+    _index(db, "messages", "idx_messages_updated_at", ["updated_at"])
+    _index(db, "messages", "idx_messages_dedupe", ["sender_uid", "client_id"])
     _index(db, "notifications", "idx_notifications_user", ["user_uid"])
     _index(db, "notifications", "idx_notifications_user_read", ["user_uid", "read"])
     push_registration = get_table("push_registration")
@@ -154,19 +207,38 @@ def init_db():
         ("key_auth", ""),
         ("key_p256dh", ""),
         ("token", ""),
+        ("client_id", ""),
+        ("environment", ""),
         ("created_at", ""),
+        ("registered_at", ""),
         ("deleted_at", ""),
     ):
         if not push_registration.has_column(column):
             push_registration.create_column_by_example(column, example)
     _index(db, "push_registration", "idx_push_registration_user", ["user_uid"])
     _index(db, "push_registration", "idx_push_registration_provider", ["provider"])
+    _index(
+        db,
+        "push_registration",
+        "idx_push_registration_client",
+        ["user_uid", "provider", "client_id"],
+    )
+    _index(
+        db,
+        "push_registration",
+        "idx_push_registration_token",
+        ["user_uid", "provider", "token"],
+    )
     if "push_registration" in db.tables:
         with db:
             db.query(
                 "UPDATE push_registration SET provider = 'webpush' "
                 "WHERE provider IS NULL OR provider = ''"
             )
+            db.query(
+                "UPDATE push_registration SET registered_at = created_at "
+                "WHERE registered_at IS NULL OR registered_at = ''"
+            )
     _index(db, "sessions", "idx_sessions_token", ["session_token"])
     projects = get_table("projects")
     for column, example in (
@@ -224,6 +296,22 @@ def init_db():
     _index(
         db, "project_files", "idx_project_files_parent", ["project_uid", "parent_path"]
     )
+    project_file_sync_state = get_table("project_file_sync_state")
+    for column, example in (
+        ("project_uid", ""),
+        ("path", ""),
+        ("db_epoch", 0.0),
+        ("fs_epoch", 0.0),
+    ):
+        if not project_file_sync_state.has_column(column):
+            project_file_sync_state.create_column_by_example(column, example)
+    _index(
+        db,
+        "project_file_sync_state",
+        "idx_project_file_sync_state_path",
+        ["project_uid", "path"],
+        unique=True,
+    )
     _index(db, "badges", "idx_badges_user", ["user_uid"])
     _index(db, "badges", "idx_badges_user_name", ["user_uid", "badge_name"])
     _drop_index(db, "idx_follows_follower")
@@ -286,6 +374,8 @@ def init_db():
     )
     _index(db, "bookmarks", "idx_bookmarks_user", ["user_uid"])
     _index(db, "bookmarks", "idx_bookmarks_target", ["target_type", "target_uid"])
+    _index(db, "notes", "idx_notes_user", ["user_uid"])
+    _index(db, "notes", "idx_notes_target", ["target_type", "target_uid"])
     _index(db, "polls", "idx_polls_post", ["post_uid"])
     _index(db, "poll_options", "idx_poll_options_poll", ["poll_uid"])
     _index(db, "poll_votes", "idx_poll_votes_poll", ["poll_uid"])
@@ -510,7 +600,60 @@ def init_db():
         "idx_user_cust_scope",
         ["owner_kind", "owner_id", "scope", "lang"],
     )
+    gateway_usage_ledger = get_table("gateway_usage_ledger")
+    for column, example in (
+        ("created_at", ""),
+        ("owner_kind", ""),
+        ("owner_id", ""),
+        ("backend", ""),
+        ("endpoint", ""),
+        ("requested_model", ""),
+        ("model", ""),
+        ("status_code", 0),
+        ("success", 0),
+        ("error_category", ""),
+        ("upstream_latency_ms", 0.0),
+        ("gateway_overhead_ms", 0.0),
+        ("queue_wait_ms", 0.0),
+        ("connect_ms", 0.0),
+        ("total_latency_ms", 0.0),
+        ("prompt_tokens", 0),
+        ("completion_tokens", 0),
+        ("cache_hit_tokens", 0),
+        ("cache_miss_tokens", 0),
+        ("reasoning_tokens", 0),
+        ("total_tokens", 0),
+        ("tokens_per_second", 0.0),
+        ("context_window", 0),
+        ("context_utilization", 0.0),
+        ("cost_usd", 0.0),
+        ("input_cost_usd", 0.0),
+        ("output_cost_usd", 0.0),
+        ("native_cost", 0),
+        ("stream_requested", 0),
+        ("temperature", 0.0),
+        ("top_p", 0.0),
+        ("max_tokens", 0),
+        ("has_tools", 0),
+        ("retries_attempted", 0),
+        ("retry_succeeded", 0),
+        ("circuit_open", 0),
+        ("user_agent", ""),
+        ("app_reference", ""),
+        ("ttft_ms", 0.0),
+        ("inter_token_ms", 0.0),
+        ("provider", ""),
+        ("fallback_used_route", ""),
+    ):
+        if not gateway_usage_ledger.has_column(column):
+            gateway_usage_ledger.create_column_by_example(column, example)
     _index(db, "gateway_usage_ledger", "idx_gw_usage_time", ["created_at"])
+    _index(
+        db,
+        "gateway_usage_ledger",
+        "idx_gw_usage_provider_time",
+        ["provider", "created_at"],
+    )
     _index(
         db,
         "gateway_usage_ledger",
@@ -702,7 +845,9 @@ def init_db():
     _index(db, "instances", "idx_instances_workspace", ["is_workspace", "status"])
     _index(db, "instances", "idx_instances_workspace_owner", ["workspace_owner_uid"])
     _index(db, "instances", "idx_instances_tunnel_name", ["tunnel_name"])
-    _index(db, "tunnels", "idx_tunnels_hostname", ["hostname"])
+    _dedupe_tunnel_hostnames()
+    _drop_index(db, "idx_tunnels_hostname")
+    _index(db, "tunnels", "idx_tunnels_hostname", ["hostname"], unique=True)
     _index(db, "tunnels", "idx_tunnels_instance", ["instance_uid"])
     _index(db, "tunnels", "idx_tunnels_user", ["user_uid"])
     _index(db, "tunnels", "idx_tunnels_state", ["desired_state", "status"])
@@ -1107,6 +1252,12 @@ def init_db():
     _index(db, "deepsearch_sessions", "idx_deepsearch_sessions_owner", ["owner_kind", "owner_id"])
     _index(db, "deepsearch_sessions", "idx_deepsearch_sessions_status", ["status"])
     _index(db, "deepsearch_sessions", "idx_deepsearch_sessions_created", ["created_at"])
+    _index(
+        db,
+        "deepsearch_sessions",
+        "idx_deepsearch_sessions_owner_created",
+        ["owner_kind", "owner_id", "created_at"],
+    )
 
     deepsearch_messages = get_table("deepsearch_messages")
     for column, example in (
@@ -1798,8 +1949,8 @@ def init_db():
         news_defaults = {
             "news_grade_threshold": "7",
             "news_api_url": "https://news.app.molodetz.nl/api",
-            "news_ai_url": INTERNAL_GATEWAY_URL,
-            "news_ai_model": "molodetz",
+            "news_ai_url": AQUALITY_NEWS_GRADING_URL,
+            "news_ai_model": AQUALITY_NEWS_GRADING_MODEL,
         }
         for key, value in news_defaults.items():
             existing = db["site_settings"].find_one(key=key)
@@ -1831,6 +1982,7 @@ def init_db():
             "maintenance_message": "DevPlace is undergoing scheduled maintenance. Please check back shortly.",
             "customization_enabled": "1",
             "customization_js_enabled": "1",
+            "happy_404_enabled": "1",
             "audit_log_retention_days": "90",
             "statistics_tracking_enabled": "1",
             "docs_search_mode": "agent",
@@ -1920,6 +2072,7 @@ def init_db():
     _index(db, "reactions", "idx_reactions_created_at", ["created_at"])
     _index(db, "votes", "idx_votes_created_at", ["created_at"])
     _index(db, "bookmarks", "idx_bookmarks_created_at", ["created_at"])
+    _index(db, "notes", "idx_notes_created_at", ["created_at"])
     _index(db, "badges", "idx_badges_created_at", ["created_at"])
     _index(db, "audit_log", "idx_audit_result_created", ["result", "created_at"])
     _index(db, "jobs", "idx_jobs_created_at", ["created_at"])
@@ -2045,6 +2198,12 @@ def migrate_ai_gateway_settings() -> None:
         if get_setting(key, "") == OLD_GATEWAY_URL:
             set_setting(key, INTERNAL_GATEWAY_URL)
             logger.info(f"Migrated {key} to the internal gateway")
+    if get_setting("news_ai_url", "") == INTERNAL_GATEWAY_URL:
+        set_setting("news_ai_url", AQUALITY_NEWS_GRADING_URL)
+        logger.info("Migrated news_ai_url to the free aquality grading model")
+    if get_setting("news_ai_model", "") in ("molodetz", ""):
+        set_setting("news_ai_model", AQUALITY_NEWS_GRADING_MODEL)
+        logger.info("Migrated news_ai_model to aquality")
     if get_setting("bot_model", "") == "deepseek-chat":
         set_setting("bot_model", "molodetz")
         logger.info("Migrated bot_model to molodetz")
@@ -2059,6 +2218,26 @@ def migrate_ai_gateway_settings() -> None:
     migrate_retired_image_gateway()
 
 
+def _ensure_users_uid() -> int:
+    if "users" not in db.tables:
+        return 0
+    users = get_table("users")
+    if not users.has_column("uid"):
+        users.create_column_by_example("uid", "")
+    import uuid_utils
+
+    updated = 0
+    for user in users.find():
+        if not user.get("uid"):
+            users.update(
+                {"id": user["id"], "uid": str(uuid_utils.uuid7())}, ["id"]
+            )
+            updated += 1
+    if updated:
+        logger.info("Backfilled uid for %s user(s)", updated)
+    return updated
+
+
 def backfill_api_keys() -> int:
     users = get_table("users")
     if not users.has_column("api_key"):
@@ -2113,6 +2292,10 @@ def backfill_api_keys() -> int:
         users.create_column_by_example("suspension_reason", "")
     if not users.has_column("deletion_requested_at"):
         users.create_column_by_example("deletion_requested_at", "")
+    if not users.has_column("active_conversation_uid"):
+        users.create_column_by_example("active_conversation_uid", "")
+    if not users.has_column("active_conversation_at"):
+        users.create_column_by_example("active_conversation_at", "")
     with db:
         db.query(
             "UPDATE users SET ai_modifier_enabled = 1 WHERE ai_modifier_enabled IS NULL"

devplacepy/database/soft_delete.py

@@ -16,6 +16,7 @@ SOFT_DELETE_TABLES = [
     "votes",
     "reactions",
     "bookmarks",
+    "notes",
     "follows",
     "poll_votes",
     "polls",

devplacepy/database/users.py

@@ -15,8 +15,6 @@ def get_users_by_uids(uids):
 
 
 _admins_cache = TTLCache(ttl=300, max_size=4)
-# The primary administrator must be an account that can actually authenticate, so scan a
-# few of the earliest admins and skip any that are soft-deleted or deactivated.
 PRIMARY_ADMIN_CANDIDATES = 50
 
 
@@ -32,6 +30,9 @@ def get_admin_uids():
         return list(cached)
     if "users" not in db.tables:
         return []
+    users = db["users"]
+    if "uid" not in users.columns or "role" not in users.columns:
+        return []
     rows = db.query("SELECT uid FROM users WHERE role = 'Admin'")
     uids = [row["uid"] for row in rows]
     _admins_cache.set("uids", uids)
@@ -92,6 +93,9 @@ def get_primary_admin_uid():
         return cached or None
     if "users" not in db.tables:
         return None
+    users = db["users"]
+    if "uid" not in users.columns or "role" not in users.columns:
+        return None
     rows = list(
         db.query(
             "SELECT * FROM users WHERE role = 'Admin' "
@@ -114,15 +118,18 @@ def search_users_by_username(q, *, exclude_uid=None, limit=10):
         return []
     if exclude_uid is not None:
         rows = db.query(
-            "SELECT uid, username FROM users WHERE username LIKE :q AND uid != :me LIMIT :limit",
+            "SELECT uid, username, avatar_seed FROM users WHERE username LIKE :q AND uid != :me LIMIT :limit",
             q=f"%{q}%",
             me=exclude_uid,
             limit=limit,
         )
     else:
         rows = db.query(
-            "SELECT uid, username FROM users WHERE username LIKE :q LIMIT :limit",
+            "SELECT uid, username, avatar_seed FROM users WHERE username LIKE :q LIMIT :limit",
             q=f"%{q}%",
             limit=limit,
         )
-    return [{"uid": r["uid"], "username": r["username"]} for r in rows]
+    return [
+        {"uid": r["uid"], "username": r["username"], "avatar_seed": r["avatar_seed"]}
+        for r in rows
+    ]

devplacepy/dependencies.py

@@ -1,9 +1,4 @@
 # retoor <retoor@molodetz.nl>
-"""
-Generic FastAPI dependency that accepts JSON or form-encoded data,
-validated against a Pydantic model.
-"""
-
 import json
 import logging
 from typing import Any, TypeVar, get_origin
@@ -17,21 +12,10 @@ logger = logging.getLogger(__name__)
 
 _TModel = TypeVar("_TModel", bound=BaseModel)
 
-# Container origins recognised as sequence fields that may receive
-# multiple values from form data.
 _SEQUENCE_ORIGINS = frozenset({list, set, tuple, frozenset})
 
 
 def _formdata_to_dict(form: FormData, model: type[BaseModel]) -> dict[str, Any]:
-    """Convert FormData to a dict suitable for Pydantic validation.
-
-    * Sequence-typed model fields collect every submitted value via
-      ``getlist()``; a lone empty string is dropped (browsers emit empty
-      hidden inputs by default).
-    * Scalar fields use ``get()`` (the last value).
-    * Fields absent from the form are omitted so that Pydantic applies
-      the model default.
-    """
     body: dict[str, Any] = {}
     for field_name, field_info in model.model_fields.items():
         origin = get_origin(field_info.annotation)
@@ -50,8 +34,6 @@ def _formdata_to_dict(form: FormData, model: type[BaseModel]) -> dict[str, Any]:
 
 
 class _JsonOrForm:
-    """Internal callable that parses JSON or form data and validates."""
-
     def __init__(self, model: type[BaseModel]):
         self.model = model
 
@@ -70,7 +52,6 @@ class _JsonOrForm:
                         status_code=400, detail="JSON body must be an object"
                     )
                 return self.model.model_validate(body)
-            # Default: form-encoded (multipart or url-encoded)
             try:
                 form = await request.form()
             except Exception as exc:
@@ -85,11 +66,4 @@ class _JsonOrForm:
 
 
 def json_or_form(model: type[_TModel]) -> _JsonOrForm:
-    """Dependency factory: accept JSON or form-encoded data for a Pydantic model.
-
-    Usage:
-        @router.post("/create")
-        async def create(data: Annotated[PostForm, Depends(json_or_form(PostForm))]):
-            ...
-    """
     return _JsonOrForm(model)

devplacepy/docs_api/_shared.py

@@ -3,6 +3,7 @@
 VOTE_TARGETS = ["post", "comment", "gist", "project", "quiz"]
 REACTION_TARGETS = ["post", "comment", "gist", "project", "quiz"]
 BOOKMARK_TARGETS = ["post", "gist", "project", "news", "quiz"]
+NOTE_TARGETS = ["post", "gist", "project", "news"]
 COMMENT_TARGETS = ["post", "project", "news", "issue", "gist", "quiz"]
 PROJECT_TYPES = ["game", "game_asset", "software", "mobile_app", "website"]
 GIST_LANGUAGES = [
@@ -38,6 +39,7 @@ def field(
     example="",
     description="",
     options=None,
+    nullable=False,
 ):
     spec = {
         "name": name,
@@ -46,6 +48,7 @@ def field(
         "required": required,
         "example": example,
         "description": description,
+        "nullable": nullable,
     }
     if options:
         spec["options"] = list(options)

devplacepy/docs_api/groups/admin.py

@@ -265,6 +265,29 @@ four ways to sign requests.
                     ),
                 ],
             ),
+            endpoint(
+                id="admin-issues-planning",
+                method="GET",
+                path="/admin/issues/planning",
+                title="Ticket planning report",
+                summary=(
+                    "Admin page listing every open Gitea ticket so an admin can pick a subset "
+                    "and generate a grouped, ordered planning document for a coding agent."
+                ),
+                auth="admin",
+                interactive=False,
+                sample_response={
+                    "configured": True,
+                    "tickets": [
+                        {"number": 42, "title": "Fix login redirect", "labels": ["bug"]},
+                    ],
+                    "tickets_error": False,
+                },
+                notes=[
+                    "`configured` is false when the issue tracker (Gitea) has not been set up in Services yet - `tickets` is then empty.",
+                    "`tickets_error` is true when the tracker is configured but the live fetch failed - retry shortly.",
+                ],
+            ),
             endpoint(
                 id="admin-ai-usage",
                 method="GET",
@@ -300,7 +323,7 @@ four ways to sign requests.
                     ),
                 ],
                 notes=[
-                    "TTFT and inter-token latency are not reported: the gateway forwards non-streaming to the upstream."
+                    "TTFT and inter-token latency (latency.ttft_ms/latency.inter_token_ms) are measured only for calls that requested streaming (stream: true); a zero count means no streaming calls occurred in the window, not that the metric is unavailable."
                 ],
             ),
             endpoint(
@@ -613,6 +636,85 @@ four ways to sign requests.
                 auth="admin",
                 destructive=True,
             ),
+            endpoint(
+                id="admin-gateway-page",
+                method="GET",
+                path="/admin/gateway",
+                title="Gateway routing dashboard",
+                summary="Admin HTML page for managing AI gateway providers and per-model routing.",
+                auth="admin",
+                interactive=False,
+            ),
+            endpoint(
+                id="admin-gateway-providers",
+                method="GET",
+                path="/admin/gateway/providers",
+                title="List AI gateway providers",
+                summary="List every configured upstream provider plus the default provider summary.",
+                auth="admin",
+                interactive=True,
+            ),
+            endpoint(
+                id="admin-gateway-provider-set",
+                method="POST",
+                path="/admin/gateway/providers",
+                title="Create or update an AI gateway provider",
+                summary="Save an upstream provider (base URL, model, and API key) by name. Pass name to update an existing provider.",
+                auth="admin",
+                params=[
+                    field("name", "json", "string", True, "openrouter", "Provider name; existing name updates in place."),
+                    field("base_url", "json", "string", True, "https://openrouter.ai/api/v1", "Upstream chat completions base URL."),
+                    field("model", "json", "string", True, "x-ai/grok-4.3", "Default model for this provider."),
+                    field("api_key", "json", "string", False, "", "Upstream API key. Blank keeps the current key."),
+                ],
+            ),
+            endpoint(
+                id="admin-gateway-provider-delete",
+                method="DELETE",
+                path="/admin/gateway/providers/{name}",
+                title="Delete an AI gateway provider",
+                summary="Delete a configured upstream provider by name.",
+                auth="admin",
+                destructive=True,
+                params=[
+                    field("name", "path", "string", True, "openrouter", "Provider name."),
+                ],
+            ),
+            endpoint(
+                id="admin-gateway-models",
+                method="GET",
+                path="/admin/gateway/models",
+                title="List AI gateway model routes",
+                summary="List every source-to-target model route plus the configured provider names.",
+                auth="admin",
+                interactive=True,
+            ),
+            endpoint(
+                id="admin-gateway-model-set",
+                method="POST",
+                path="/admin/gateway/models",
+                title="Create or update an AI gateway model route",
+                summary="Route a source model name to a target model, optionally on a specific provider. Pass source_model to update an existing route.",
+                auth="admin",
+                params=[
+                    field("source_model", "json", "string", True, "gpt-4", "Model name callers request."),
+                    field("target_model", "json", "string", True, "x-ai/grok-4.3", "Model actually sent upstream."),
+                    field("provider", "json", "string", False, "openrouter", "Provider name to route through. Blank = the default provider."),
+                    field("fallback_model", "json", "string", False, "molodetz-pro", "Another already-configured source_model of the same kind, tried once automatically when this route fails after its own retries are exhausted. Blank = no fallback."),
+                ],
+            ),
+            endpoint(
+                id="admin-gateway-model-delete",
+                method="DELETE",
+                path="/admin/gateway/models/{source_model}",
+                title="Delete an AI gateway model route",
+                summary="Delete a source-to-target model route by source model name.",
+                auth="admin",
+                destructive=True,
+                params=[
+                    field("source_model", "path", "string", True, "gpt-4", "Source model name."),
+                ],
+            ),
             endpoint(
                 id="admin-gateway-quota-rules",
                 method="GET",
@@ -975,5 +1077,190 @@ four ways to sign requests.
                 destructive=True,
                 sample_response={"ok": True, "redirect": "/admin/game"},
             ),
+            endpoint(
+                id="admin-workspaces-page",
+                method="GET",
+                path="/admin/workspaces",
+                title="Workspaces dashboard",
+                summary="Admin HTML page listing every dev workspace across all projects, with owner, project, and moderation flags.",
+                auth="admin",
+                interactive=False,
+            ),
+            endpoint(
+                id="admin-workspaces-data",
+                method="GET",
+                path="/admin/workspaces/data",
+                title="List workspaces",
+                summary="Every dev workspace across all projects plus open moderation flags, as JSON.",
+                auth="admin",
+                interactive=True,
+                sample_response={"workspaces": [], "flags": []},
+            ),
+            endpoint(
+                id="admin-workspaces-suspend",
+                method="POST",
+                path="/admin/workspaces/{uid}/suspend",
+                title="Suspend a workspace",
+                summary="Suspend a workspace with a reason shown to its owner. The owner is notified.",
+                auth="admin",
+                destructive=True,
+                params=[
+                    field("uid", "path", "string", True, "INSTANCE_UID", "Workspace instance uid."),
+                    field("reason", "form", "string", True, "Excessive resource usage", "Shown to the workspace owner."),
+                ],
+            ),
+            endpoint(
+                id="admin-workspaces-unsuspend",
+                method="POST",
+                path="/admin/workspaces/{uid}/unsuspend",
+                title="Unsuspend a workspace",
+                summary="Lift a suspension. The owner is notified the workspace is available again.",
+                auth="admin",
+                params=[
+                    field("uid", "path", "string", True, "INSTANCE_UID", "Workspace instance uid."),
+                ],
+            ),
+            endpoint(
+                id="admin-workspaces-stop",
+                method="POST",
+                path="/admin/workspaces/{uid}/stop",
+                title="Stop a workspace",
+                summary="Stop the workspace container. Files and tunnels are kept.",
+                auth="admin",
+                destructive=True,
+                params=[
+                    field("uid", "path", "string", True, "INSTANCE_UID", "Workspace instance uid."),
+                ],
+            ),
+            endpoint(
+                id="admin-workspaces-start",
+                method="POST",
+                path="/admin/workspaces/{uid}/start",
+                title="Start a workspace",
+                summary="Resume a stopped workspace container.",
+                auth="admin",
+                params=[
+                    field("uid", "path", "string", True, "INSTANCE_UID", "Workspace instance uid."),
+                ],
+            ),
+            endpoint(
+                id="admin-workspaces-delete",
+                method="POST",
+                path="/admin/workspaces/{uid}/delete",
+                title="Delete a workspace",
+                summary="Remove a workspace and its tunnels.",
+                auth="admin",
+                destructive=True,
+                params=[
+                    field("uid", "path", "string", True, "INSTANCE_UID", "Workspace instance uid."),
+                ],
+            ),
+            endpoint(
+                id="admin-workspaces-flag",
+                method="POST",
+                path="/admin/workspaces/{uid}/flag",
+                title="Flag a workspace",
+                summary="Raise a moderation flag on a workspace. The owner is notified.",
+                auth="admin",
+                params=[
+                    field("uid", "path", "string", True, "INSTANCE_UID", "Workspace instance uid."),
+                    field("kind", "form", "string", False, "manual", "Flag kind."),
+                    field("severity", "form", "string", False, "warn", "Flag severity."),
+                    field("detail", "form", "string", False, "", "Detail shown to the owner."),
+                ],
+            ),
+            endpoint(
+                id="admin-workspaces-flag-resolve",
+                method="POST",
+                path="/admin/workspaces/flags/{flag_uid}/resolve",
+                title="Resolve or dismiss a workspace flag",
+                summary="Set a moderation flag's status.",
+                auth="admin",
+                params=[
+                    field("flag_uid", "path", "string", True, "FLAG_UID", "Flag uid."),
+                    field("status", "query", "string", False, "resolved", "resolved or dismissed."),
+                ],
+            ),
+            endpoint(
+                id="admin-workspaces-editor",
+                method="POST",
+                path="/admin/workspaces/{uid}/editor",
+                title="Set a workspace owner's editor preferences",
+                summary="Change the workspace owner's editor preferences on their behalf, or reset them. Subject to admin seniority.",
+                auth="admin",
+                params=[
+                    field("uid", "path", "string", True, "INSTANCE_UID", "Workspace instance uid."),
+                    field("theme", "form", "string", False, "devplace-dark", "devplace-dark, devplace-light or system."),
+                    field("layout", "form", "string", False, "standard", "standard, terminal-focus or zen."),
+                    field("panel_preset", "form", "string", False, "tall", "short, normal, tall or maximized."),
+                    field("font_size", "form", "integer", False, "14", "Editor font size in pixels."),
+                    field("terminal_font_size", "form", "integer", False, "13", "Terminal font size in pixels."),
+                    field("zoom_level", "form", "integer", False, "0", "Window zoom, -5 to 5."),
+                    field("boot_agent", "form", "string", False, "dpc", "dpc or none."),
+                    field("boot_shell", "form", "integer", False, "1", "1 opens a shell on boot, 0 skips it."),
+                    field("window_mode", "form", "string", False, "tab", "tab, window or fullscreen."),
+                    field("window_width", "form", "integer", False, "1600", "Editor window width in pixels."),
+                    field("window_height", "form", "integer", False, "1000", "Editor window height in pixels."),
+                    field("reset", "form", "boolean", False, "false", "Drop every preference for this owner."),
+                ],
+            ),
+            endpoint(
+                id="admin-workspaces-quota",
+                method="POST",
+                path="/admin/workspaces/quota",
+                title="Set a user's workspace quota override",
+                summary="Set or update a per-user override of the workspace count/disk/egress/idle/retention/CPU/memory limits.",
+                auth="admin",
+                params=[
+                    field("owner_id", "form", "string", True, "USER_UID", "Target user uid."),
+                    field("label", "form", "string", False, "", "Optional admin-facing note."),
+                    field("max_workspaces", "form", "int", False, "0", "0 = use the site default."),
+                    field("max_tunnels", "form", "int", False, "0", "0 = use the site default."),
+                    field("disk_quota_mb", "form", "int", False, "0", "0 = use the site default."),
+                    field("egress_quota_mb", "form", "int", False, "0", "0 = use the site default."),
+                    field("idle_stop_minutes", "form", "int", False, "0", "0 = use the site default."),
+                    field("retention_days", "form", "int", False, "0", "0 = use the site default."),
+                    field("cpu_millicores", "form", "int", False, "0", "0 = use the site default."),
+                    field("memory_mb", "form", "int", False, "0", "0 = use the site default."),
+                ],
+            ),
+            endpoint(
+                id="admin-trash-list",
+                method="GET",
+                path="/admin/trash",
+                title="Trash",
+                summary="Admin HTML page listing soft-deleted rows for one table, with restore/purge controls per row.",
+                auth="admin",
+                interactive=True,
+                params=[
+                    field("table", "query", "string", False, "posts", "Trash table key (posts, comments, gists, projects, news, awards, quizzes, project_files, attachments)."),
+                    field("page", "query", "int", False, "1", "Page number."),
+                ],
+            ),
+            endpoint(
+                id="admin-trash-restore",
+                method="POST",
+                path="/admin/trash/{table}/{uid}/restore",
+                title="Restore a soft-deleted row",
+                summary="Restore every row soft-deleted under the same event timestamp as the given row (a whole delete cascade at once).",
+                auth="admin",
+                params=[
+                    field("table", "path", "string", True, "posts", "Trash table key."),
+                    field("uid", "path", "string", True, "ROW_UID", "Row uid."),
+                ],
+            ),
+            endpoint(
+                id="admin-trash-purge",
+                method="POST",
+                path="/admin/trash/{table}/{uid}/purge",
+                title="Purge a soft-deleted row",
+                summary="Permanently delete every row soft-deleted under the same event timestamp as the given row, unlinking any attachment/project-file blobs.",
+                auth="admin",
+                destructive=True,
+                params=[
+                    field("table", "path", "string", True, "posts", "Trash table key."),
+                    field("uid", "path", "string", True, "ROW_UID", "Row uid."),
+                ],
+            ),
         ],
 }

devplacepy/docs_api/groups/content.py

@@ -62,6 +62,75 @@ four ways to sign requests.
                         "Search post title, content, and author username.",
                     ),
                     field("before", "query", "string", False, "", "Pagination cursor."),
+                    field(
+                        "items[].poll",
+                        "response",
+                        "object",
+                        description="Present only when the post has a poll attached.",
+                        nullable=True,
+                    ),
+                    field(
+                        "items[].war",
+                        "response",
+                        "object",
+                        description="Present only when an Opinion War is running on the post.",
+                        nullable=True,
+                    ),
+                    field(
+                        "items[].project_link",
+                        "response",
+                        "object",
+                        description="Present only when the post is attached to a project.",
+                        nullable=True,
+                    ),
+                    field(
+                        "items[].maturity",
+                        "response",
+                        "string",
+                        description="Content maturity label; absent for unrated posts.",
+                        nullable=True,
+                    ),
+                    field(
+                        "items[].my_vote",
+                        "response",
+                        "integer",
+                        description="Viewer's own vote on the post, defaults to 0 (never null).",
+                        nullable=False,
+                    ),
+                    field(
+                        "items[].comment_count",
+                        "response",
+                        "integer",
+                        description="Total comment count, defaults to 0 (never null).",
+                        nullable=False,
+                    ),
+                ],
+            ),
+            endpoint(
+                id="topics-hub",
+                method="GET",
+                path="/topics",
+                title="Browse topics",
+                summary="The topics hub - every post topic with its live post count, linking to its own crawlable listing page.",
+                auth="public",
+                interactive=True,
+            ),
+            endpoint(
+                id="topics-list",
+                method="GET",
+                path="/topics/{topic}",
+                title="Browse one topic",
+                summary=(
+                    "A single topic's post listing, on its own permanent, crawlable URL (unlike /feed?topic=, "
+                    "whose canonical collapses back to /feed). Same author-interleaved pagination as the feed."
+                ),
+                auth="public",
+                interactive=True,
+                params=[
+                    field(
+                        "topic", "path", "enum", True, "devlog", "Topic key.", TOPICS
+                    ),
+                    field("before", "query", "string", False, "", "Pagination cursor."),
                 ],
             ),
             endpoint(
@@ -152,7 +221,49 @@ four ways to sign requests.
                         True,
                         "POST_SLUG",
                         "Slug or UID of the post.",
-                    )
+                    ),
+                    field(
+                        "post.title",
+                        "response",
+                        "string",
+                        description="Optional title, blank when the author posted without one.",
+                        nullable=True,
+                    ),
+                    field(
+                        "post.image",
+                        "response",
+                        "string",
+                        description="Cover image URL; null unless one was attached.",
+                        nullable=True,
+                    ),
+                    field(
+                        "post.project_uid",
+                        "response",
+                        "string",
+                        description="Attached project uid; null unless the post is linked to a project.",
+                        nullable=True,
+                    ),
+                    field(
+                        "post.updated_at",
+                        "response",
+                        "string",
+                        description="ISO timestamp of the last edit; null until the post is edited for the first time.",
+                        nullable=True,
+                    ),
+                    field(
+                        "post.content",
+                        "response",
+                        "string",
+                        description="Post body, always present (minimum 10 characters at creation).",
+                        nullable=False,
+                    ),
+                    field(
+                        "post.slug",
+                        "response",
+                        "string",
+                        description="Always generated at creation, never null.",
+                        nullable=False,
+                    ),
                 ],
             ),
             endpoint(

devplacepy/docs_api/groups/gateway.py

@@ -77,9 +77,16 @@ managed by administrators on the **Gateway** page (`/admin/gateway`).
 
 ## Per-call cost and usage headers
 
-Every gateway response - chat, embeddings, and passthrough, on both success and error - carries
-`X-Gateway-*` response headers describing that single call, so a client can read its own token usage
-and dollar cost directly from the response with no extra request:
+Every NON-STREAMING gateway response - chat, embeddings, images, and passthrough, on both success
+and error - carries `X-Gateway-*` response headers describing that single call, so a client can read
+its own token usage and dollar cost directly from the response with no extra request. A streaming
+chat response (`"stream": true`) is the one exception: it carries only `X-Gateway-Model`,
+`X-Gateway-Backend`, and `X-App-Reference` - HTTP headers must be sent before the body, and cost/token
+counts for a streamed call are only known once the stream ends, so they cannot be response headers on
+that same response. The call is still fully metered server-side, and a client that sends
+`"stream_options": {"include_usage": true}` still receives the upstream's real `usage` object on the
+final SSE chunk, exactly as the underlying provider's own streaming API works - it is just not
+summarized into headers.
 
 | Header | Meaning |
 |--------|---------|
@@ -123,9 +130,17 @@ X-App-Reference: devplace-devii-v-1-0-0
 Administrators enable and configure this gateway under [Background Services](/docs/services.html)
 (the `openai` service).
 
-The gateway is exempt from rate limiting, but every other endpoint follows the shared
+The gateway is exempt from general rate limiting, but every other endpoint follows the shared
 [Conventions & Errors](/docs/conventions.html); see [Authentication](/docs/authentication.html)
 for signing DevPlace's own requests.
+
+A dedicated failed-authentication throttle protects the gateway from unauthenticated probing:
+an IP that repeatedly presents no credentials at all and fails authentication is answered `429`
+(with a `Retry-After` header) once it crosses a configurable threshold within a rolling window.
+This never affects a request that presents a valid key or session, even from an IP that has
+recently failed - a legitimate caller sharing a network address with a prior bad actor is never
+locked out. A `429` returned to a properly-authenticated call is a separate daily quota limit,
+not this throttle.
 """,
         "endpoints": [
             endpoint(
@@ -161,11 +176,20 @@ for signing DevPlace's own requests.
                         "false",
                         "Set true for a streamed SSE response.",
                     ),
+                    field(
+                        "think",
+                        "json",
+                        "string",
+                        False,
+                        "false",
+                        "Optional thinking override. Omitted: the gateway disables thinking (fast path). true / high / medium / low enables it; false disables it. DeepSeek-native `thinking.type` and OpenRouter `reasoning.effort` are also accepted.",
+                    ),
                 ],
                 notes=[
                     "Returns `503` when the gateway service is not running.",
-                    "Every response carries the `X-Gateway-*` token and dollar-cost headers (see Per-call cost and usage headers above), including the streamed SSE response.",
+                    "A non-streaming response carries the `X-Gateway-*` token and dollar-cost headers (see Per-call cost and usage headers above). A streaming response (`stream: true`) is forwarded from the upstream in real time and carries only `X-Gateway-Model`/`X-Gateway-Backend`/`X-App-Reference` - request `stream_options: {\"include_usage\": true}` to receive the real token usage on the final SSE chunk instead.",
                     "If `model` matches a configured model route it is forwarded to that route's provider, upstream model, and per-model pricing (with an optional vision model); otherwise it falls through to the default upstream (see Model routing and providers above).",
+                    "Thinking is disabled by default. Pass `think: true` (or `thinking: {\"type\": \"enabled\"}`) to turn it on for that call.",
                 ],
             ),
             endpoint(

devplacepy/docs_api/groups/lookups.py

@@ -34,7 +34,13 @@ four ways to sign requests.
                     )
                 ],
                 sample_response={
-                    "results": [{"uid": "8f14e45f-...", "username": "alice_test"}]
+                    "results": [
+                        {
+                            "uid": "8f14e45f-...",
+                            "username": "alice_test",
+                            "avatar_seed": None,
+                        }
+                    ]
                 },
             ),
             endpoint(
@@ -54,7 +60,13 @@ four ways to sign requests.
                     )
                 ],
                 sample_response={
-                    "results": [{"uid": "0cc175b9-...", "username": "bob_test"}]
+                    "results": [
+                        {
+                            "uid": "0cc175b9-...",
+                            "username": "bob_test",
+                            "avatar_seed": None,
+                        }
+                    ]
                 },
             ),
         ],

devplacepy/docs_api/groups/messaging.py

@@ -41,6 +41,14 @@ four ways to sign requests.
                         "",
                         "Jump to a conversation by username.",
                     ),
+                    field(
+                        "before",
+                        "query",
+                        "string",
+                        False,
+                        "",
+                        "ISO timestamp. When set, return the page of messages strictly older than this instant (for loading earlier history).",
+                    ),
                 ],
             ),
             endpoint(

devplacepy/docs_api/groups/profiles.py

@@ -65,6 +65,55 @@ four ways to sign requests.
                         "Profile tab.",
                         ["posts", "activity", "followers", "following", "media", "awards"],
                     ),
+                    field(
+                        "api_key",
+                        "response",
+                        "string",
+                        description="The profile's API key; null unless the viewer is the owner or an admin (see can_view_api_key).",
+                        nullable=True,
+                    ),
+                    field(
+                        "rank",
+                        "response",
+                        "integer",
+                        description="Leaderboard rank; null for a user with no stars yet.",
+                        nullable=True,
+                    ),
+                    field(
+                        "follow_pagination",
+                        "response",
+                        "object",
+                        description="Pagination metadata for the followers/following tabs; null on every other tab.",
+                        nullable=True,
+                    ),
+                    field(
+                        "profile_user.avatar_seed",
+                        "response",
+                        "string",
+                        description="Nullable users.avatar_seed override; null falls back to the username as the avatar seed.",
+                        nullable=True,
+                    ),
+                    field(
+                        "profile_user.bio",
+                        "response",
+                        "string",
+                        description="Optional profile bio; null when never set.",
+                        nullable=True,
+                    ),
+                    field(
+                        "is_following",
+                        "response",
+                        "boolean",
+                        description="Always a boolean, defaults to false, never null.",
+                        nullable=False,
+                    ),
+                    field(
+                        "profile_user.username",
+                        "response",
+                        "string",
+                        description="Always present, never null.",
+                        nullable=False,
+                    ),
                 ],
             ),
             endpoint(

devplacepy/docs_api/groups/push.py

@@ -14,13 +14,15 @@ implements the Web Push protocol: fetch the public VAPID key, then register a
 Push Notification service device token and is only offered when an administrator has
 configured it.
 
-`GET /push.json` lists the providers that currently accept registrations. A registration
-body without a `provider` field is a `webpush` registration, so existing clients need no
-change.
+`GET /push.json` lists the providers that currently accept registrations. When `apns` is
+active it includes `environment` (`production` or `sandbox`) so a native client can match
+its build. A registration body without a `provider` field is a `webpush` registration, so
+existing clients need no change. An APNs body may include a stable `client_id` so a later
+token rotation updates the same device instead of inserting another row.
 
-There is no server-side unsubscribe endpoint: unsubscription is handled entirely in the
-browser by calling `PushManager.unsubscribe()` on the subscription. The server stops delivering
-to a subscription once its push endpoint reports it as gone. These mirror the in-app
+`DELETE /push.json` removes a single registration by the same identity used to create it
+(`endpoint` for webpush, `token` or `client_id` for apns). The server also stops delivering to a
+subscription once its push endpoint reports it as gone. These mirror the in-app
 [Notifications](/docs/notifications.html) feed.
 
 Every endpoint follows the shared [Conventions & Errors](/docs/conventions.html) (auth, content
@@ -37,7 +39,10 @@ four ways to sign requests.
                 auth="public",
                 sample_response={
                     "publicKey": "BASE64_VAPID_KEY",
-                    "providers": {"webpush": {"publicKey": "BASE64_VAPID_KEY"}},
+                    "providers": {
+                        "webpush": {"publicKey": "BASE64_VAPID_KEY"},
+                        "apns": {"environment": "production"},
+                    },
                 },
             ),
             endpoint(
@@ -80,15 +85,75 @@ four ways to sign requests.
                         "string",
                         False,
                         "a1b2c3...",
-                        "Hexadecimal device token. Required for apns.",
+                        "Hexadecimal device token. Required for apns. Spaces and angle brackets are stripped.",
+                    ),
+                    field(
+                        "client_id",
+                        "json",
+                        "string",
+                        False,
+                        "vendor-uuid",
+                        "Stable per-device id for apns. When present, a new token updates this device instead of inserting a row.",
                     ),
                 ],
                 notes=[
                     'A webpush body is JSON: `{"endpoint": "...", "keys": {"p256dh": "...", "auth": "..."}}`.',
-                    'An APNs body is JSON: `{"provider": "apns", "token": "..."}`.',
+                    'An APNs body is JSON: `{"provider": "apns", "token": "...", "client_id": "..."}`. `client_id` is optional; token-only bodies keep working and revive a previously dead token.',
                     "A provider that is unknown, disabled or unconfigured returns 400.",
+                    "A newly created or revived registration is probed immediately. The response then includes `delivered` and, on failure, `error` with the provider reason. `registered` stays true so existing clients keep working.",
                 ],
-                sample_response={"registered": True},
+                sample_response={"registered": True, "delivered": True},
+            ),
+            endpoint(
+                id="push-unregister",
+                method="DELETE",
+                path="/push.json",
+                title="Unregister a subscription",
+                summary="Remove one push registration by its identity.",
+                auth="user",
+                encoding="json",
+                interactive=False,
+                params=[
+                    field(
+                        "provider",
+                        "json",
+                        "string",
+                        False,
+                        "webpush",
+                        "Provider the registration belongs to. Omit for webpush.",
+                    ),
+                    field(
+                        "endpoint",
+                        "json",
+                        "string",
+                        False,
+                        "https://fcm.googleapis.com/...",
+                        "Subscription endpoint URL. Identifies a webpush registration.",
+                    ),
+                    field(
+                        "token",
+                        "json",
+                        "string",
+                        False,
+                        "a1b2c3...",
+                        "Hexadecimal device token. Identifies an apns registration.",
+                    ),
+                    field(
+                        "client_id",
+                        "json",
+                        "string",
+                        False,
+                        "vendor-uuid",
+                        "Stable per-device id. Identifies an apns registration when set.",
+                    ),
+                ],
+                notes=[
+                    "Exactly one identity field is required: `endpoint` for webpush, `token` or `client_id` for apns.",
+                    "Matches the same identity priority as registration: `client_id`, then `token`, then `endpoint`.",
+                    "Idempotent: unregistering an unknown or already-removed identity still returns 200 with `unregistered: false`.",
+                    "Call this before logging out to stop delivery to the device that is logging out - the server has no way to know a browser tab or native app closed on its own.",
+                ],
+                sample_response={"unregistered": True},
             ),
         ],
 }

devplacepy/docs_api/groups/social_actions.py

@@ -1,18 +1,29 @@
 # retoor <retoor@molodetz.nl>
 
-from .._shared import BOOKMARK_TARGETS, REACTION_TARGETS, VOTE_TARGETS, endpoint, field
+from .._shared import (
+    BOOKMARK_TARGETS,
+    NOTE_TARGETS,
+    REACTION_TARGETS,
+    VOTE_TARGETS,
+    endpoint,
+    field,
+)
 from devplacepy.constants import REACTION_EMOJI
 
 GROUP = {
         "slug": "social-actions",
-        "title": "Votes, Reactions, Bookmarks & Polls",
+        "title": "Votes, Reactions, Bookmarks, Notes & Polls",
         "intro": """
-# Votes, Reactions, Bookmarks & Polls
+# Votes, Reactions, Bookmarks, Notes & Polls
 
-Lightweight engagement actions. The POST endpoints here are **toggles** - sending the same
-action again removes it. They return JSON when called with `X-Requested-With: fetch` (sent
-automatically by the panels below); the [Conventions & Errors](/docs/conventions.html) page
-explains that header rule and the response envelope.
+Lightweight engagement actions. The vote/reaction/bookmark POST endpoints here are **toggles** -
+sending the same action again removes it. They return JSON when called with
+`X-Requested-With: fetch` (sent automatically by the panels below); the
+[Conventions & Errors](/docs/conventions.html) page explains that header rule and the response
+envelope.
+
+Personal notes are private, per-user annotations attached to a piece of content - only you can
+ever see your own notes.
 
 Every endpoint follows the shared [Conventions & Errors](/docs/conventions.html) (auth, content
 negotiation, pagination, status codes); see [Authentication](/docs/authentication.html) for the
@@ -24,7 +35,7 @@ four ways to sign requests.
                 method="POST",
                 path="/votes/{target_type}/{target_uid}",
                 title="Cast or toggle a vote",
-                summary="Upvote or downvote a target. Re-sending the same value removes the vote.",
+                summary="Upvote or downvote a target. Re-sending the same value, or sending 0, removes the vote.",
                 auth="user",
                 ajax=True,
                 encoding="form",
@@ -52,8 +63,8 @@ four ways to sign requests.
                         "enum",
                         True,
                         "1",
-                        "1 to upvote, -1 to downvote.",
-                        ["1", "-1"],
+                        "1 to upvote, -1 to downvote, 0 to retract your existing vote.",
+                        ["1", "-1", "0"],
                     ),
                 ],
                 sample_response={"net": 3, "up": 4, "down": 1, "value": 1},
@@ -151,6 +162,97 @@ four ways to sign requests.
                     "Bookmarks target posts, projects, gists, and news; see [Posts, Comments, Projects, Gists & News](/docs/content.html)."
                 ],
             ),
+            endpoint(
+                id="notes-set",
+                method="POST",
+                path="/notes/{target_type}/{target_uid}",
+                title="Add or update a personal note",
+                summary="Save a private note on a target. Only you can ever see it.",
+                auth="user",
+                ajax=True,
+                encoding="form",
+                params=[
+                    field(
+                        "target_type",
+                        "path",
+                        "enum",
+                        True,
+                        "post",
+                        "Type of content to annotate.",
+                        NOTE_TARGETS,
+                    ),
+                    field(
+                        "target_uid",
+                        "path",
+                        "string",
+                        True,
+                        "POST_UID",
+                        "UID of the target.",
+                    ),
+                    field(
+                        "content",
+                        "form",
+                        "string",
+                        True,
+                        "Remember to check this later.",
+                        "Note body, up to 4000 characters. Sending again on the same target replaces the note.",
+                    ),
+                ],
+                sample_response={"uid": "NOTE_UID", "content": "Remember to check this later."},
+            ),
+            endpoint(
+                id="notes-delete",
+                method="POST",
+                path="/notes/{target_type}/{target_uid}/delete",
+                title="Delete a personal note",
+                summary="Remove your private note from a target.",
+                auth="user",
+                ajax=True,
+                encoding="none",
+                params=[
+                    field(
+                        "target_type",
+                        "path",
+                        "enum",
+                        True,
+                        "post",
+                        "Type of content the note is on.",
+                        NOTE_TARGETS,
+                    ),
+                    field(
+                        "target_uid",
+                        "path",
+                        "string",
+                        True,
+                        "POST_UID",
+                        "UID of the target.",
+                    ),
+                ],
+                sample_response={"deleted": True},
+            ),
+            endpoint(
+                id="notes-saved",
+                method="GET",
+                path="/notes/saved",
+                title="View your personal notes",
+                summary="Render your saved notes. Returns an HTML page.",
+                auth="user",
+                interactive=True,
+                params=[
+                    field(
+                        "before",
+                        "query",
+                        "string",
+                        False,
+                        "",
+                        "Pagination cursor (created_at of the last item).",
+                    )
+                ],
+                notes=[
+                    "Notes target posts, projects, gists, and news; see [Posts, Comments, Projects, Gists & News](/docs/content.html).",
+                    "Nobody else can ever see your notes, not even the author of the content you annotated.",
+                ],
+            ),
             endpoint(
                 id="polls-vote",
                 method="POST",

devplacepy/docs_api/groups/tools.py

@@ -226,12 +226,43 @@ status and report.
                         {"title": "Invention", "detail": "...", "confidence": 0.8, "citations": [1]}
                     ],
                     "sources": [{"url": "https://example.com", "title": "Example", "source": "httpx"}],
+                    "follow_up_questions": ["Who else worked on the transistor?", "How did it replace vacuum tubes?"],
                     "chat_ws_url": "/tools/deepsearch/DEEPSEARCH_JOB_UID/chat",
                     "export_md_url": "/tools/deepsearch/DEEPSEARCH_JOB_UID/export.md",
                     "export_json_url": "/tools/deepsearch/DEEPSEARCH_JOB_UID/export.json",
                     "export_pdf_url": "/tools/deepsearch/DEEPSEARCH_JOB_UID/export.pdf",
                 },
             ),
+            endpoint(
+                id="tools-deepsearch-history",
+                method="GET",
+                path="/tools/deepsearch/history",
+                title="My DeepSearch history",
+                summary="List the caller's own past DeepSearch runs, newest first, with a link back to each report/chat. Member history is account-bound; guest history is scoped to the visitor's address. Negotiates HTML or JSON.",
+                auth="public",
+                params=[
+                    field("limit", "query", "integer", False, "20", "Maximum sessions to return (1-100)."),
+                ],
+                sample_response={
+                    "sessions": [
+                        {
+                            "uid": "DEEPSEARCH_JOB_UID",
+                            "query": "history of the transistor",
+                            "status": "done",
+                            "score": 78,
+                            "confidence": 0.72,
+                            "page_count": 11,
+                            "chunk_count": 240,
+                            "summary": "The transistor was invented at Bell Labs in 1947...",
+                            "reopen_url": "/tools/deepsearch/DEEPSEARCH_JOB_UID/session",
+                            "chat_available": True,
+                            "available": True,
+                            "created_at": "2026-06-14T10:00:00+00:00",
+                            "completed_at": "2026-06-14T10:01:40+00:00",
+                        }
+                    ]
+                },
+            ),
             endpoint(
                 id="tools-isslop-run",
                 method="POST",

devplacepy/docs_api/groups/workspaces.py

@@ -37,7 +37,10 @@ arrives as a `workspace` notification and states exactly what happens next and w
             title="Read workspace",
             summary=(
                 "State, quota usage, idle countdown, tunnels and open moderation flags "
-                "for your workspace on this project."
+                "for your workspace on this project. The phase (stopped, starting, ready, "
+                "stopping, crashed, suspended) is derived from the desired state, the "
+                "container status and a live probe of the editor port, so editor_ready "
+                "is true only when the editor will actually open."
             ),
             auth="user",
             params=[
@@ -51,7 +54,12 @@ arrives as a `workspace` notification and states exactly what happens next and w
                 "editor_url": "/projects/my-project/containers/instances/INSTANCE_UID/code/",
                 "workspace": {
                     "uid": "INSTANCE_UID",
+                    "owner_uid": "USER_UID",
                     "status": "running",
+                    "desired_state": "running",
+                    "phase": "ready",
+                    "phase_label": "Ready",
+                    "editor_ready": True,
                     "suspended": False,
                     "tunnel_name": "brave-otter",
                     "primary_url": "https://brave-otter.tunnel.pravda.education",
@@ -131,7 +139,7 @@ arrives as a `workspace` notification and states exactly what happens next and w
                     "terminal_font_size": 13,
                     "zoom_level": 0,
                     "layout": "standard",
-                    "panel_preset": "tall",
+                    "panel_preset": "normal",
                     "boot_agent": "dpc",
                     "boot_shell": True,
                     "window_mode": "tab",

devplacepy/docs_api/negotiation.py

@@ -37,6 +37,7 @@ _PAGE_RESPONSES = {
     "issues-detail": schemas.IssueDetailOut,
     "issues-attachments-list": schemas.IssueAttachmentsOut,
     "bookmarks-saved": schemas.SavedOut,
+    "notes-saved": schemas.NotesOut,
     "admin-users": schemas.AdminUsersOut,
     "admin-news-list": schemas.AdminNewsOut,
     "admin-settings-get": schemas.AdminSettingsOut,

devplacepy/docs_api/services_group.py

@@ -117,6 +117,25 @@ def build_services_group(services, base):
             )
         )
     control_endpoints = [
+        endpoint(
+            id="services-page",
+            method="GET",
+            path="/admin/services",
+            title="Services dashboard",
+            summary="Admin HTML index of every registered background service, its status, and controls.",
+            auth="admin",
+            interactive=False,
+        ),
+        endpoint(
+            id="services-detail-page",
+            method="GET",
+            path="/admin/services/{name}",
+            title="Service detail page",
+            summary="Admin HTML detail page for a single service: overview, configuration form, and logs.",
+            auth="admin",
+            interactive=False,
+            params=[field("name", "path", "string", True, "news", "Service name.")],
+        ),
         endpoint(
             id="services-data",
             method="GET",

devplacepy/docs_export.py

@@ -31,8 +31,8 @@ def _params_table(params: list) -> str:
     if not params:
         return ""
     rows = [
-        "| Name | In | Type | Required | Description |",
-        "|------|----|------|----------|-------------|",
+        "| Name | In | Type | Required | Nullable | Description |",
+        "|------|----|------|----------|----------|-------------|",
     ]
     for p in params:
         desc = (p.get("description", "") or "").replace("|", "\\|")
@@ -43,7 +43,8 @@ def _params_table(params: list) -> str:
             desc = f"{desc} Allowed: {allowed}.".strip()
         rows.append(
             f"| `{p['name']}` | {p['location']} | {p['type']} | "
-            f"{'yes' if p['required'] else 'no'} | {desc} |"
+            f"{'yes' if p['required'] else 'no'} | "
+            f"{'yes' if p.get('nullable') else 'no'} | {desc} |"
         )
     return "\n".join(rows)
 

devplacepy/happy404.py

@@ -0,0 +1,70 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import logging
+import os
+import random
+
+from fastapi import Request
+from fastapi.responses import HTMLResponse
+
+from devplacepy.cache import TTLCache
+from devplacepy.content import load_detail
+from devplacepy.database import db, get_setting
+from devplacepy.routers.posts import post_page_context
+from devplacepy.templating import templates
+from devplacepy.utils import get_current_user
+
+logger = logging.getLogger("happy404")
+
+POOL_TTL_SECONDS = int(os.environ.get("DEVPLACE_HAPPY_404_POOL_TTL", "300"))
+POOL_SIZE = 100
+
+API_PATH_PREFIXES = ("/api", "/dbapi", "/openai", "/xmlrpc", "/swagger", "/openapi.json")
+
+_pool_cache = TTLCache(ttl=POOL_TTL_SECONDS, max_size=1)
+
+
+def _post_pool() -> list[str]:
+    cached = _pool_cache.get("slugs")
+    if cached is not None:
+        return cached
+    slugs: list[str] = []
+    if "posts" in db.tables:
+        rows = db.query(
+            "SELECT slug, uid FROM posts WHERE deleted_at IS NULL ORDER BY RANDOM() LIMIT :limit",
+            limit=POOL_SIZE,
+        )
+        slugs = [row["slug"] or row["uid"] for row in rows]
+    _pool_cache.set("slugs", slugs)
+    return slugs
+
+
+def _eligible(request: Request) -> bool:
+    if request.method != "GET":
+        return False
+    if request.url.path.startswith(API_PATH_PREFIXES):
+        return False
+    return get_setting("happy_404_enabled", "1") == "1"
+
+
+def render(request: Request) -> HTMLResponse | None:
+    try:
+        if not _eligible(request):
+            return None
+        pool = _post_pool()
+        if not pool:
+            return None
+        slug = random.choice(pool)
+        user = get_current_user(request)
+        detail = load_detail("posts", "post", slug, user)
+        if not detail:
+            return None
+        context = post_page_context(
+            request, user, detail, robots="noindex,nofollow"
+        )
+        return templates.TemplateResponse(request, "post.html", context)
+    except Exception as exc:  # noqa: BLE001 - a happy-404 bug must never break the 404 page
+        logger.warning("happy_404 render failed: %s", exc)
+        return None

devplacepy/main.py

@@ -20,6 +20,7 @@ from devplacepy.config import (
     PORT,
     SERVICE_LOCK_FILE,
     INIT_LOCK_FILE,
+    LOG_LEVEL,
     ensure_data_dirs,
 )
 from devplacepy.database import (
@@ -33,7 +34,7 @@ from devplacepy.database import (
     get_news_images_by_uids,
     get_setting,
     get_int_setting,
-    interleave_by_author,
+    paginate_diverse,
     get_user_post_count,
     get_user_stars,
     get_blocked_uids,
@@ -43,6 +44,7 @@ from devplacepy.database import (
 from devplacepy.templating import templates, jinja_unread_count
 from devplacepy.cache import TTLCache
 from devplacepy.responses import respond, wants_json, json_error
+from devplacepy import happy404
 from devplacepy.schemas import LandingOut, ValidationErrorOut
 from fastapi.responses import JSONResponse
 from devplacepy.utils import get_current_user, time_ago, safe_next, client_ip
@@ -52,6 +54,7 @@ from devplacepy.routers import (
     battles,
     feed,
     posts,
+    topics,
     comments,
     projects,
     profile,
@@ -75,6 +78,7 @@ from devplacepy.routers import (
     reactions,
     reports,
     bookmarks,
+    notes,
     polls,
     docs,
     openai_gateway,
@@ -128,7 +132,7 @@ from devplacepy.services.telegram import TelegramService
 from devplacepy.services.telegram.outbox_service import TelegramOutboxService
 
 logging.basicConfig(
-    level=logging.INFO,
+    level=getattr(logging, LOG_LEVEL, logging.INFO),
     format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
 )
 logger = logging.getLogger(__name__)
@@ -261,6 +265,9 @@ async def lifespan(app: FastAPI):
         from devplacepy.push import ensure_certificates
 
         ensure_certificates()
+        from devplacepy.services.openai_gateway import model_health
+
+        model_health.seed_from_ledger()
     service_manager.register(NewsService())
     service_manager.register(BotsService())
     service_manager.register(GatewayService())
@@ -315,6 +322,9 @@ async def lifespan(app: FastAPI):
     from devplacepy.services.containers import forward
 
     await forward.close_client()
+    from devplacepy import push
+
+    await push.shutdown_providers()
 
 
 app = FastAPI(
@@ -341,6 +351,9 @@ app.mount("/static", FallbackStaticFiles(directory=str(STATIC_DIR)), name="stati
 async def not_found(request: Request, exc):
     if wants_json(request):
         return json_error(404, "Not found")
+    happy_response = happy404.render(request)
+    if happy_response is not None:
+        return happy_response
     seo_ctx = base_seo_context(
         request,
         title="Not Found - DevPlace",
@@ -471,6 +484,7 @@ async def on_validation_error(request: Request, exc: RequestValidationError):
 app.include_router(auth.router, prefix="/auth")
 app.include_router(feed.router, prefix="/feed")
 app.include_router(posts.router, prefix="/posts")
+app.include_router(topics.router, prefix="/topics")
 app.include_router(comments.router, prefix="/comments")
 app.include_router(projects.router, prefix="/projects")
 app.include_router(profile.router, prefix="/profile")
@@ -480,6 +494,7 @@ app.include_router(votes.router, prefix="/votes")
 app.include_router(reactions.router, prefix="/reactions")
 app.include_router(reports.router, prefix="/reports")
 app.include_router(bookmarks.router, prefix="/bookmarks")
+app.include_router(notes.router, prefix="/notes")
 app.include_router(polls.router, prefix="/polls")
 app.include_router(avatar.router, prefix="/avatar")
 app.include_router(awards.router, prefix="/awards")
@@ -785,7 +800,8 @@ def _landing_news():
     return articles
 
 
-def _landing_recent_posts(blocked):
+def _landing_recent_posts(viewer_uid):
+    blocked = get_blocked_uids(viewer_uid) if viewer_uid else frozenset()
     if not blocked:
         cached = _home_cache.get("posts")
         if cached is not None:
@@ -793,15 +809,9 @@ def _landing_recent_posts(blocked):
     posts = []
     if "posts" in db.tables:
         posts_table = get_table("posts")
-        clauses = []
-        if blocked:
-            clauses.append(posts_table.table.columns.user_uid.notin_(blocked))
-        raw_posts = list(
-            posts_table.find(
-                *clauses, deleted_at=None, order_by=["-created_at"], _limit=6
-            )
+        raw_posts, _ = paginate_diverse(
+            posts_table, order=["-created_at"], viewer_uid=viewer_uid, limit=6
         )
-        raw_posts = interleave_by_author(raw_posts)
         if raw_posts:
             post_uids = [p["uid"] for p in raw_posts]
             author_uids = [p["user_uid"] for p in raw_posts]
@@ -829,8 +839,9 @@ async def landing(request: Request):
     user = get_current_user(request)
 
     landing_articles = _landing_news()
-    blocked = get_blocked_uids(user["uid"]) if user else frozenset()
-    landing_posts = _landing_recent_posts(blocked)
+    viewer_uid = user["uid"] if user else None
+    blocked = get_blocked_uids(viewer_uid) if viewer_uid else frozenset()
+    landing_posts = _landing_recent_posts(viewer_uid)
 
     base = site_url(request)
     seo_ctx = base_seo_context(

devplacepy/models.py

@@ -258,6 +258,10 @@ class CommentEditForm(BaseModel):
     content: str = Field(min_length=3, max_length=125000)
 
 
+class NoteForm(BaseModel):
+    content: str = Field(min_length=1, max_length=4000)
+
+
 class ProjectForm(BaseModel):
     title: str = Field(min_length=1, max_length=200)
     description: str = Field(min_length=1, max_length=5000)
@@ -547,8 +551,8 @@ class VoteForm(BaseModel):
     @field_validator("value")
     @classmethod
     def valid_value(cls, value):
-        if value not in (1, -1):
-            raise ValueError("value must be 1 or -1")
+        if value not in (1, -1, 0):
+            raise ValueError("value must be 1, -1 or 0")
         return value
 
 
@@ -681,6 +685,7 @@ class AdminSettingsForm(BaseModel):
     registration_open: str = Field(default="", max_length=1)
     maintenance_mode: str = Field(default="", max_length=1)
     maintenance_message: str = Field(default="", max_length=300)
+    happy_404_enabled: str = Field(default="", max_length=1)
     docs_search_mode: str = Field(default="", max_length=20)
     outbound_proxy_url: str = Field(default="", max_length=500)
     moderation_sla_hours: str = Field(default="", max_length=10)
@@ -695,6 +700,9 @@ class AdminSettingsForm(BaseModel):
     privacy_version: str = Field(default="", max_length=20)
     guidelines_version: str = Field(default="", max_length=20)
     ai_third_party_provider: str = Field(default="", max_length=120)
+    correction_model: str = Field(default="", max_length=120)
+    modifier_model: str = Field(default="", max_length=120)
+    quiz_grading_model: str = Field(default="", max_length=120)
     extra_head: str = Field(default="", max_length=50000)
 
     @field_validator("moderation_filter_mode")

devplacepy/project_files.py

@@ -1,7 +1,9 @@
 # retoor <retoor@molodetz.nl>
 
 import logging
+import os
 import shutil
+import time
 from datetime import datetime, timezone
 from pathlib import Path
 
@@ -17,6 +19,8 @@ MAX_FILES_PER_PROJECT = 5000
 MAX_PATH_LENGTH = 1024
 MAX_SEGMENT_LENGTH = 255
 MAX_DEPTH = 32
+_MISSING_BLOB_LOG_SECONDS = 60
+_missing_blob_logged: dict[str, tuple[int, float]] = {}
 
 TEXT_EXTENSIONS = {
     ".txt",
@@ -477,6 +481,7 @@ def delete_node(project_uid: str, raw_path: str, deleted_by: str = "system") ->
 def soft_delete_all_project_files(project_uid: str, deleted_by: str) -> None:
     if "project_files" not in db.tables:
         return
+    clear_sync_state(project_uid)
     stamp = _now()
     for row in _table().find(project_uid=project_uid, deleted_at=None):
         _table().update(
@@ -486,6 +491,7 @@ def soft_delete_all_project_files(project_uid: str, deleted_by: str) -> None:
 
 
 _SYNC_CLOCK_SKEW_SECONDS = 1.0
+SYNC_STATE_TABLE = "project_file_sync_state"
 
 
 def _epoch_of(iso_timestamp) -> float:
@@ -497,6 +503,48 @@ def _epoch_of(iso_timestamp) -> float:
         return 0.0
 
 
+def _sync_state_table():
+    return get_table(SYNC_STATE_TABLE)
+
+
+def _load_sync_manifest(project_uid: str) -> dict:
+    if SYNC_STATE_TABLE not in db.tables:
+        return {}
+    return {
+        row["path"]: {"db_epoch": row["db_epoch"], "fs_epoch": row["fs_epoch"]}
+        for row in _sync_state_table().find(project_uid=project_uid)
+    }
+
+
+def _save_sync_manifest(project_uid: str, old: dict, new: dict) -> None:
+    table = _sync_state_table()
+    for path in old:
+        if path not in new:
+            table.delete(project_uid=project_uid, path=path)
+    for path, entry in new.items():
+        if old.get(path) == entry:
+            continue
+        table.upsert(
+            {"project_uid": project_uid, "path": path, **entry},
+            ["project_uid", "path"],
+        )
+
+
+def clear_sync_state(project_uid: str) -> None:
+    if SYNC_STATE_TABLE not in db.tables:
+        return
+    _sync_state_table().delete(project_uid=project_uid)
+
+
+def _delete_db_row_for_sync(row: dict, deleted_by: str) -> None:
+    _table().update(
+        {"uid": row["uid"], "deleted_at": _now(), "deleted_by": deleted_by},
+        ["uid"],
+    )
+    if row.get("is_binary"):
+        _unlink_blob(row)
+
+
 def _file_records(project_uid: str) -> dict:
     records: dict = {}
     for row in _table().find(project_uid=project_uid, deleted_at=None):
@@ -510,7 +558,7 @@ def _walk_workspace_files(src, skip):
     for root, dirs, files in src.walk():
         dirs[:] = [d for d in sorted(dirs) if d not in skip]
         for name in sorted(files):
-            if name in skip:
+            if name in skip or Path(name).suffix in IMPORT_SKIP_EXTENSIONS:
                 continue
             full = Path(root) / name
             if full.is_symlink() or not full.is_file():
@@ -536,71 +584,176 @@ def _workspace_records(workspace) -> dict:
 
 
 def sync_dir_bidirectional(project_uid: str, workspace, user: dict) -> dict:
+    empty = {
+        "exported": 0,
+        "imported": 0,
+        "deleted_in_project": 0,
+        "deleted_in_workspace": 0,
+    }
     if "project_files" not in db.tables:
-        return {"exported": 0, "imported": 0}
+        return empty
     readonly = is_readonly(project_uid)
     dest = Path(workspace).resolve()
     dest.mkdir(parents=True, exist_ok=True)
     db_files = _file_records(project_uid)
     fs_files = _workspace_records(dest)
-    exported = 0
-    imported = 0
+    manifest = _load_sync_manifest(project_uid)
+    new_manifest: dict = {}
+    counts = dict(empty)
+    missing_blobs = 0
 
-    for path, row in db_files.items():
+    for path in set(manifest) | set(db_files) | set(fs_files):
+        row = db_files.get(path)
         fs_full = fs_files.get(path)
-        if fs_full is None:
-            _export_node(row, dest)
-            exported += 1
-            continue
-        try:
-            fs_mtime = fs_full.stat().st_mtime
-        except OSError:
-            continue
-        db_mtime = _epoch_of(row.get("updated_at"))
-        if db_mtime >= fs_mtime - _SYNC_CLOCK_SKEW_SECONDS:
-            _export_node(row, dest)
-            exported += 1
-        elif not readonly:
-            if _import_file(project_uid, user, path, fs_full):
-                imported += 1
+        entry = manifest.get(path)
 
-    if not readonly:
-        for path, fs_full in fs_files.items():
-            if path in db_files:
+        if row is not None and fs_full is not None:
+            try:
+                fs_epoch = fs_full.stat().st_mtime
+            except OSError:
                 continue
-            if _import_file(project_uid, user, path, fs_full):
-                imported += 1
+            db_epoch = _epoch_of(row.get("updated_at"))
+            if (
+                entry is not None
+                and abs(db_epoch - entry["db_epoch"]) <= _SYNC_CLOCK_SKEW_SECONDS
+                and abs(fs_epoch - entry["fs_epoch"]) <= _SYNC_CLOCK_SKEW_SECONDS
+            ):
+                new_manifest[path] = entry
+                continue
+            if db_epoch >= fs_epoch - _SYNC_CLOCK_SKEW_SECONDS or readonly:
+                recorded = _record_export(row, dest)
+                if recorded:
+                    counts["exported"] += 1
+                    new_manifest[path] = recorded
+                elif row.get("is_binary"):
+                    missing_blobs += 1
+            else:
+                recorded = _record_import(project_uid, user, path, fs_full, fs_epoch)
+                if recorded:
+                    counts["imported"] += 1
+                    new_manifest[path] = recorded
+            continue
 
-    return {"exported": exported, "imported": imported}
+        if row is not None and fs_full is None:
+            db_epoch = _epoch_of(row.get("updated_at"))
+            if (
+                entry is None
+                or readonly
+                or db_epoch > entry["db_epoch"] + _SYNC_CLOCK_SKEW_SECONDS
+            ):
+                recorded = _record_export(row, dest)
+                if recorded:
+                    counts["exported"] += 1
+                    new_manifest[path] = recorded
+                elif row.get("is_binary"):
+                    missing_blobs += 1
+            else:
+                _delete_db_row_for_sync(row, user["uid"])
+                counts["deleted_in_project"] += 1
+            continue
+
+        if row is None and fs_full is not None:
+            try:
+                fs_epoch = fs_full.stat().st_mtime
+            except OSError:
+                continue
+            if entry is None:
+                if not readonly:
+                    recorded = _record_import(project_uid, user, path, fs_full, fs_epoch)
+                    if recorded:
+                        counts["imported"] += 1
+                        new_manifest[path] = recorded
+                continue
+            if not readonly and fs_epoch > entry["fs_epoch"] + _SYNC_CLOCK_SKEW_SECONDS:
+                recorded = _record_import(project_uid, user, path, fs_full, fs_epoch)
+                if recorded:
+                    counts["imported"] += 1
+                    new_manifest[path] = recorded
+                continue
+            try:
+                fs_full.unlink()
+                counts["deleted_in_workspace"] += 1
+            except OSError:
+                pass
+            continue
+
+    _save_sync_manifest(project_uid, manifest, new_manifest)
+    _warn_missing_blobs(project_uid, missing_blobs, "while syncing")
+    return counts
 
 
-def _export_node(row: dict, dest: Path) -> None:
+def _record_export(row: dict, dest: Path):
+    target = _export_node(row, dest)
+    if target is None:
+        return None
+    db_epoch = _epoch_of(row.get("updated_at"))
+    try:
+        fs_epoch = target.stat().st_mtime
+    except OSError:
+        fs_epoch = db_epoch
+    return {"db_epoch": db_epoch, "fs_epoch": fs_epoch}
+
+
+def _record_import(project_uid: str, user: dict, path: str, fs_full: Path, fs_epoch: float):
+    imported = _import_file(project_uid, user, path, fs_full)
+    if imported is None:
+        return None
+    return {"db_epoch": _epoch_of(imported.get("updated_at")), "fs_epoch": fs_epoch}
+
+
+def _copy_blob(src: Path, target: Path) -> bool:
+    try:
+        shutil.copyfile(src, target)
+        return True
+    except (FileNotFoundError, OSError):
+        return False
+
+
+def _warn_missing_blobs(project_uid: str, missing: int, action: str) -> None:
+    if missing <= 0:
+        return
+    now = time.monotonic()
+    previous = _missing_blob_logged.get(project_uid)
+    if (
+        previous is not None
+        and previous[0] == missing
+        and (now - previous[1]) < _MISSING_BLOB_LOG_SECONDS
+    ):
+        return
+    _missing_blob_logged[project_uid] = (missing, now)
+    logger.warning(
+        "Skipped %s missing blob file(s) %s project %s",
+        missing,
+        action,
+        project_uid,
+    )
+
+
+def _export_node(row: dict, dest: Path):
     target = (dest / row["path"]).resolve()
     if target != dest and not target.is_relative_to(dest):
-        return
+        return None
     target.parent.mkdir(parents=True, exist_ok=True)
     if target.is_symlink():
         target.unlink()
     if row.get("is_binary") and row.get("stored_name") and row.get("directory"):
         src = PROJECT_FILES_DIR / row["directory"] / row["stored_name"]
-        try:
-            shutil.copyfile(src, target)
-        except (FileNotFoundError, OSError):
-            logger.warning("Blob file missing during export: %s", src)
+        if not _copy_blob(src, target):
+            return None
     else:
         target.write_text(row.get("content") or "", encoding="utf-8")
+    return target
 
 
-def _import_file(project_uid: str, user: dict, path: str, fs_full: Path) -> bool:
+def _import_file(project_uid: str, user: dict, path: str, fs_full: Path):
     try:
         data = fs_full.read_bytes()
     except OSError:
-        return False
+        return None
     try:
-        store_upload(project_uid, user, _parent_of(path), _name_of(path), data)
-        return True
+        return store_upload(project_uid, user, _parent_of(path), _name_of(path), data)
     except ProjectFileError:
-        return False
+        return None
 
 
 def node_to_dict(row: dict) -> dict:
@@ -662,6 +815,36 @@ IMPORT_SKIP_NAMES = {
     ".DS_Store",
     ".idea",
     ".cache",
+    "build",
+    "dist",
+    "target",
+    "out",
+    "bin",
+    "obj",
+    ".next",
+    ".nuxt",
+    ".gradle",
+    ".tox",
+    "cmake-build-debug",
+    "cmake-build-release",
+}
+# Compiled/build-artifact extensions, regenerated wholesale on every build -
+# never worth importing/syncing regardless of which directory they land in
+# (unlike IMPORT_SKIP_NAMES, matched by suffix rather than exact name; see
+# _walk_workspace_files). A missing exclusion here is what let an unlocked
+# concurrent sync (see api._sync_dir_bidirectional_locked) leak millions of
+# orphaned blobs from an actively-compiling container workspace.
+IMPORT_SKIP_EXTENSIONS = {
+    ".o",
+    ".obj",
+    ".pyc",
+    ".pyo",
+    ".class",
+    ".so",
+    ".dylib",
+    ".dll",
+    ".a",
+    ".exe",
 }
 SYNC_SKIP_NAMES = IMPORT_SKIP_NAMES | {
     ".devplace_boot.py",
@@ -704,6 +887,7 @@ def export_to_dir(project_uid: str, subpath: str, dest_dir) -> int:
         rows = list(_table().find(project_uid=project_uid, deleted_at=None))
         strip = ""
     written = 0
+    missing = 0
     for row in sorted(rows, key=lambda r: r["path"]):
         relative = row["path"][len(strip) :].lstrip("/") if strip else row["path"]
         target = (dest / relative).resolve()
@@ -717,14 +901,13 @@ def export_to_dir(project_uid: str, subpath: str, dest_dir) -> int:
             target.unlink()
         if row.get("is_binary") and row.get("stored_name") and row.get("directory"):
             src = PROJECT_FILES_DIR / row["directory"] / row["stored_name"]
-            try:
-                shutil.copyfile(src, target)
-            except (FileNotFoundError, OSError):
-                logger.warning("Blob file missing: %s", src)
+            if not _copy_blob(src, target):
+                missing += 1
                 continue
         else:
             target.write_text(row.get("content") or "", encoding="utf-8")
         written += 1
+    _warn_missing_blobs(project_uid, missing, "during export of")
     return written
 
 
@@ -875,7 +1058,66 @@ def append_lines(project_uid: str, raw_path: str, content: str) -> dict:
 def delete_all_project_files(project_uid: str) -> None:
     if "project_files" not in db.tables:
         return
+    clear_sync_state(project_uid)
     for row in _table().find(project_uid=project_uid):
         if row.get("is_binary"):
             _unlink_blob(row)
     _table().delete(project_uid=project_uid)
+
+
+def purge_soft_deleted_project_files(*, dry_run: bool = False) -> tuple[int, int]:
+    if "project_files" not in db.tables:
+        return 0, 0
+    table = _table()
+    rows = list(table.find(table.table.columns.deleted_at.isnot(None), is_binary=1))
+    removed = 0
+    freed = 0
+    for row in rows:
+        directory = row.get("directory")
+        stored_name = row.get("stored_name")
+        if directory and stored_name:
+            path = PROJECT_FILES_DIR / directory / stored_name
+            try:
+                freed += path.stat().st_size
+            except OSError:
+                pass
+            else:
+                if not dry_run:
+                    _unlink_blob(row)
+        if not dry_run:
+            table.delete(id=row["id"])
+        removed += 1
+    return removed, freed
+
+
+def sweep_orphan_project_file_blobs(*, dry_run: bool = False) -> tuple[int, int]:
+    if not PROJECT_FILES_DIR.exists():
+        return 0, 0
+    referenced = set()
+    if "project_files" in db.tables:
+        for row in _table().find(is_binary=1):
+            directory = row.get("directory")
+            stored_name = row.get("stored_name")
+            if directory and stored_name:
+                referenced.add((directory, stored_name))
+    removed = 0
+    freed = 0
+    base = str(PROJECT_FILES_DIR)
+    for root, _dirs, files in os.walk(base):
+        directory = os.path.relpath(root, base)
+        for name in files:
+            if (directory, name) in referenced:
+                continue
+            file_path = os.path.join(root, name)
+            try:
+                size = os.path.getsize(file_path)
+            except OSError:
+                continue
+            if not dry_run:
+                try:
+                    os.unlink(file_path)
+                except OSError:
+                    continue
+            removed += 1
+            freed += size
+    return removed, freed

devplacepy/push/CLAUDE.md

@@ -2,33 +2,35 @@ This file documents `devplacepy/push/` - push notification delivery and its prov
 
 ## What this package is
 
-One delivery library behind one public surface. `devplacepy.push` exports `notify_user`, `register`, `ensure_certificates`, `public_key_standard_b64` and the Web Push crypto helpers; every caller in the codebase (`main.py`, `utils/notifications.py`, `routers/push.py`) imports only those names. Everything else is internal to the package.
+One delivery library behind one public surface. `devplacepy.push` exports `notify_user`, `notify_registration`, `register`, `unregister`, `ensure_certificates`, `public_key_standard_b64` and the Web Push crypto helpers; every caller in the codebase (`main.py`, `utils/notifications.py`, `routers/push.py`) imports only those names. Everything else is internal to the package.
 
 | Module | Role |
 |---|---|
 | `providers/base.py` | `PushProvider` protocol, the `Delivery` outcome and the three outcome constants |
 | `providers/webpush.py` | VAPID key material, `aesgcm` payload encryption, the Web Push provider |
-| `providers/apns.py` | Apple Push Notification service provider (token based, HTTP/2) |
+| `providers/apns.py` | Apple Push Notification service provider (token based, dedicated HTTP/2 client) |
 | `providers/__init__.py` | `PROVIDERS` registry, `get`, `active`, `is_active`, `admin_fields`, `client_config` |
-| `store.py` | Every `push_registration` read and write |
-| `delivery.py` | `notify_user`: group by provider, one shared client, one prepared body per provider |
+| `store.py` | Every `push_registration` read and write, including identity upsert and revive |
+| `delivery.py` | `notify_user` / `notify_registration`: group by provider, one client per provider, one prepared body per provider |
 
 The admin configuration surface lives in `devplacepy/services/push/service.py` (`PushService`), not here.
 
 ## Adding a provider
 
-1. Write `providers/<name>.py` with a `PushProvider` subclass: `name`, `label`, `config_fields`, `is_configured`, `parse_registration`, `prepare`, `deliver`, optionally `client_config`.
+1. Write `providers/<name>.py` with a `PushProvider` subclass: `name`, `label`, `config_fields`, `is_configured`, `parse_registration`, `prepare`, `deliver`, optionally `client_config`, `stamp_registration`, `delivery_client`.
 2. Add one entry to `PROVIDERS` in `providers/__init__.py`.
 
 That is the whole change. The registration route, the delivery loop, the admin page, the audit record, the metrics and the docs are written against the protocol and need no edit. The `Enabled` toggle (`push_<name>_enabled`) comes from the base class, so a provider never declares its own.
 
+The default `delivery_client` is `stealth_async_client`. Override it only when the destination is a first-party API that must not see Chrome impersonation, PRIORITY frames, extra browser headers, or the outbound proxy. APNs is that case.
+
 ## Invariants
 
-- **Zero cost for the request.** Delivery is reached only through `utils/notifications.py` `_schedule_push`, a fire-and-forget task. Never make a route await `notify_user`, and never add a queue or a table to this path.
+- **Zero cost for the request, except the welcome probe.** Delivery of real notifications is reached only through `utils/notifications.py` `_schedule_push`, a fire-and-forget task. `POST /push.json` awaits `notify_registration` only for a newly created or revived row, so the client can see `delivered` / `error`. Never add a queue or a table to this path.
 - **`deliver` never raises.** Return `Delivery(REJECTED, detail)` instead. `delivery.py` guards anyway, but a raising provider costs a log line per subscription.
 - **A provider that is not configured is inert, never an error.** `is_configured()` is false, `is_active()` is false, the delivery loop skips it, and `POST /push.json` refuses a registration for it with 400. Nothing else in the platform notices.
-- **`DEAD` is the only outcome that touches the database.** It soft-deletes the registration (`deleted_at`), exactly like a `404`/`410` Web Push endpoint always did. `REJECTED` keeps the row.
-- **Every insert writes `deleted_at: None`,** and every read filters `deleted_at IS NULL`. `push_registration` deliberately stays out of `SOFT_DELETE_TABLES` (no `deleted_by`, not restorable from Trash) - a dead device token has no owner action to undo.
+- **`DEAD` is the only outcome that touches the database.** It soft-deletes the registration (`deleted_at`), exactly like a `404`/`410` Web Push endpoint always did - unless `Delivery.dead_before` names an instant the row was proven live again after (APNs 410 `timestamp` vs. `registered_at`, see "APNs specifics"), in which case the delete is skipped. `REJECTED` keeps the row. The Apple/Web Push reason is logged at WARNING (`Push dead via ...: <detail>`); do not drop `outcome.detail`.
+- **Every insert writes `deleted_at: None`,** and every read of the live set filters `deleted_at IS NULL`. Identity lookups for upsert/revive intentionally ignore `deleted_at` so a client that re-registers a previously dead token comes back live. `push_registration` deliberately stays out of `SOFT_DELETE_TABLES` (no `deleted_by`, not restorable from Trash) - a dead device token has no owner action to undo.
 - **A row without a provider is a Web Push row.** `store.provider_of` resolves `None`/`""` to `DEFAULT_PROVIDER`, so a row written by an old worker during a deploy still delivers. `init_db` backfills the column once with a single converging `UPDATE`.
 
 ## Storage
@@ -40,13 +42,39 @@ That is the whole change. The registration route, the delivery loop, the admin p
 | `provider` | `webpush` | `apns` |
 | `endpoint`, `key_auth`, `key_p256dh` | set | `NULL` |
 | `token` | `NULL` | device token |
+| `client_id` | unused | optional stable per-device id |
+| `environment` | unused | `production` or `sandbox`, stamped server-side at register |
+| `registered_at` | stamped on insert/merge | stamped on insert/merge |
 
-Deduplication is generic: `store.register` looks up `user_uid` + `provider` + exactly the fields the provider's `parse_registration` returned, so a provider never writes its own identity rule.
+`registered_at` (ISO, both providers) is stamped on insert and on every `_merge` that actually changes a row (including revival). It exists solely to arbitrate the dead-token race described below - it is not a general "last seen" field.
+
+`store.register` returns `RegistrationWrite(record, created, revived)`. Identity, in order:
+
+1. `user_uid` + `provider` + `client_id` (live or dead), if `client_id` is present.
+2. `user_uid` + `provider` + `token` (or `endpoint` for Web Push), live or dead.
+3. Exact live match on the remaining fields.
+4. Insert.
+
+A match **updates** the row (token, client_id, environment) and clears `deleted_at` if it was dead. Token rotation with the same `client_id` therefore replaces the token on one row. Sibling live rows that share the new token, the previous token, or the same `client_id` are marked dead so one device cannot accumulate duplicates. A body without `client_id` still works: the same token revives, a new token inserts. `push.update` is a real update, not a no-op of an identical POST.
+
+## Unregistering
+
+`DELETE /push.json` (`routers/push.py` `push_unregister`) removes exactly the one registration named by the caller - never every registration for the user, mirroring how `POST /push.json` creates or updates exactly one row. `store.unregister(user_uid, provider, fields)` looks the row up with the **same identity priority as `register`**: `client_id`, then `token`, then `endpoint`. A match stamps `deleted_at` (the same bespoke soft-delete `mark_dead` already uses for a dead token - `push_registration` has no `deleted_by` and is not restorable from Trash by design, see "Invariants" above), so a later re-`register` with the same identity revives the row exactly like a dead token does, rather than accumulating a duplicate. No match (unknown identity, already-removed, or wrong owner) is a no-op that still returns `200 {"unregistered": false}` - the call is idempotent so a client can fire it speculatively at logout without first checking whether a subscription exists. The route validates that at least one identity field is a non-empty string before calling the store, refusing `400` on a body carrying none, exactly like `POST /push.json` refuses an unparseable registration body.
+
+**Client wiring.** `static/js/PushManager.js` intercepts every `a[href="/auth/logout"]` click, resolves the current `PushManager.getSubscription()` (only ever a webpush one - browser JS has no access to a native APNs device token), calls `DELETE /push.json` with `{endpoint}`, then `subscription.unsubscribe()` client-side, then navigates to the logout link's `href`. This closes the webpush half of the "token stays registered after logout" gap. **A native (iOS/Android) client integrating APNs must call `DELETE /push.json` with `{provider: "apns", token}` (or `client_id`) itself before or alongside its own logout** - there is no way for server-side `GET /auth/logout` (a plain, bodyless navigation) or this web frontend to know about, let alone unregister, a native app's device token.
+
+**No Devii action.** Unlike most mutating endpoints, `push.json` (register OR unregister) has no Devii catalog entry - a push identity (a webpush subscription endpoint/keys or an APNs device token) is private per-device browser/OS state that Devii cannot obtain, generate, or usefully ask the user to paste into a chat turn. This mirrors `POST /push.json` already having no Devii action for the identical reason.
 
 ## APNs specifics
 
-- `POST https://{host}/3/device/{token}` over HTTP/2, host from `push_apns_environment` (`api.push.apple.com` or `api.sandbox.push.apple.com`; an unrecognised value falls back to production). HTTP/2 comes from `stealth_async_client` because the origin is `https` - the cleartext downgrade in `curl_transport` does not apply.
-- Provider token: `ES256`, header `kid` = key id, claims `iss` = team id and `iat`. Cached per credential fingerprint for 45 minutes, so a worker signs at most one token per 45 minutes; Apple refuses tokens regenerated faster than every 20 minutes. Editing any credential changes the fingerprint and takes effect on the next delivery, with no restart.
+- **Persistent HTTP/2 connection, not one per notification.** `apns.cached_client(timeout)` lazily creates ONE module-level `httpx.AsyncClient(http2=True, ...)` and reuses it across every `notify_user`/`notify_registration` call for the lifetime of the process, following Apple's explicit guidance to keep the connection open rather than repeatedly opening/closing (`sending-notification-requests-to-apns`). `ApnsProvider.closes_delivery_client()` returns `False` so `delivery.py` never closes it after a batch (Web Push still opens/closes per call via `stealth_async_client`, `closes_delivery_client()` defaulting `True` on the base class). Closed once, gracefully, in `main.py`'s shutdown via `push.shutdown_providers()` -> `ApnsProvider.aclose()` -> `apns.close_client()`, mirroring the identical `services/containers/forward.py` `client()`/`close_client()` pattern. Not stealth, not curl_cffi Chrome impersonation, not the outbound proxy - Apple's provider API is a first-party HTTP/2 service; browser PRIORITY frames, HPACK indexing of `:path`/`authorization`, extra `sec-ch-ua` headers, and a scraping proxy all violate that contract. This is the second documented exception to the stealth-only outbound rule (the other is container reverse-proxy forwarding). If the admin-configured delivery timeout changes, the cached client is rebuilt with the new timeout on next use and the old one is left for GC (not explicitly closed) - a deliberate, rare-path simplification.
+- `POST https://{host}/3/device/{token}` over HTTP/2. Host comes from the **row's** `environment` if set, else `push_apns_environment` (`api.push.apple.com` or `api.sandbox.push.apple.com`; an unrecognised value falls back to production). Stamping environment per row lets a sandbox debug token and a production TestFlight token coexist.
+- `GET /push.json` advertises `providers.apns.environment` when APNs is active so a native client can refuse to register a sandbox token against production.
+- `parse_registration` requires a hexadecimal `token` (64-200 digits after stripping spaces and `<>`) and optionally `client_id` (string, max 128). A `client_id` that is not a string is a 400, not a silent drop. Server stamps `environment`; the client cannot pick the host.
+- **Provider token (JWT):** `ES256`, header `kid` = key id, claims `iss` = team id and `iat`. Cached per credential fingerprint for 45 minutes (Apple refuses tokens regenerated faster than every 20 minutes, and requires a fresh one at least once an hour). **Shared cross-worker via `site_settings` key `push_apns_shared_token`** (JSON `{fingerprint, token, issued_at}`, read/written through the existing `get_setting`/`set_setting` cache-version machinery, propagating to sibling workers within ~1s like every other settings key) - a worker that finds no valid in-process cache checks the shared copy before signing a new one, so `make prod`'s multiple uvicorn workers converge on presenting Apple the SAME token instead of each independently re-signing on its own clock (which could otherwise interleave closer together than Apple's per-credential minimum spacing, worst case when workers boot near-simultaneously). Editing any credential changes the fingerprint and takes effect on the next delivery, with no restart.
+- **A rejected provider token is evicted immediately, not left to expire from cache.** A 403 status, or any reason in `AUTH_REASONS` (`InvalidProviderToken`, `ExpiredProviderToken`, `BadCertificate`, `BadCertificateEnvironment`, `Forbidden`, `MissingProviderToken` - all provider-credential-shaped per Apple's reason table, never about a specific device), calls `invalidate_provider_token()`: clears the in-process cache AND the shared DB copy, so the very next delivery attempt (any worker) re-signs instead of retrying the same rejected token for up to 45 minutes.
 - A `.p8` that does not parse is cached as a failure for the same window, so a misconfiguration costs one error log per window rather than one parse per notification.
-- `410`, or any status carrying reason `BadDeviceToken`, `Unregistered`, `ExpiredToken`, `DeviceTokenNotForTopic` or `TopicDisallowed`, is `DEAD`. Everything else is `REJECTED`.
+- **`DEAD_REASONS` is deliberately narrow: only `BadDeviceToken` (400), `Unregistered` (410) and `ExpiredToken` (410).** These are the only reasons in Apple's documented table that describe the TOKEN itself as permanently invalid. `DeviceTokenNotForTopic` and `TopicDisallowed` are 400 errors about the **topic/provisioning matching the connection**, not the token - they fire identically for every token when `push_apns_topic` is misconfigured or the certificate/entitlements don't match, so treating them as dead would soft-delete the entire APNs subscriber base (a table deliberately kept OUT of `SOFT_DELETE_TABLES`, hence unrestorable) on the first delivery after a one-field admin typo. Never add a topic/provisioning-shaped reason to `DEAD_REASONS`.
+- **A 410's `timestamp` is honored before deleting.** Apple's 410 body carries `{"reason": "Unregistered", "timestamp": <ms epoch>}` - the instant Apple last confirmed the token dead, which can trail real device state by "several days" per Apple engineering guidance (410 delivery is intentionally non-deterministic; do not use it to infer app-uninstall timing). `apns._dead_before` converts it to ISO and it rides on `Delivery.dead_before`; `store.mark_dead(id, dead_before)` compares it against the row's `registered_at` and **skips the delete** (logs at INFO instead) if the registration was re-registered/revived after that instant - closing the race where an in-flight delivery against a stale row state would otherwise undo a concurrent revival. `dead_before` is only ever set for a 410; the `BadDeviceToken`/`ExpiredToken` paths pass `None` (unconditional delete, as before), since those reasons carry no `timestamp`.
 - The shared payload dict (`title`, `message`, `icon`, `url`) is translated once per batch into `aps.alert` plus the custom `url`/`icon` keys, mirroring what `service-worker.js` does for Web Push. `thread-id` mirrors the service worker's notification `tag`.
+- `POST /push.json` probes a created or revived row immediately via `notify_registration`. The JSON is `{registered: true, delivered: bool, error?: string}`. `registered` stays true so existing clients keep working; `delivered`/`error` surface Apple's reason instead of killing the row silently from the client's point of view. The row is still marked dead on a `DEAD` outcome (subject to the `dead_before` guard above) so later `notify_user` calls do not keep hitting a known-bad token.

devplacepy/push/__init__.py

@@ -1,6 +1,7 @@
 # retoor <retoor@molodetz.nl>
 
-from devplacepy.push.delivery import notify_user
+from devplacepy.push.delivery import notify_registration, notify_user
+from devplacepy.push.providers import shutdown as shutdown_providers
 from devplacepy.push.providers.webpush import (
     browser_base64,
     create_notification_authorization,
@@ -12,7 +13,7 @@ from devplacepy.push.providers.webpush import (
     hkdf,
     public_key_standard_b64,
 )
-from devplacepy.push.store import register
+from devplacepy.push.store import register, unregister
 
 __all__ = [
     "browser_base64",
@@ -23,7 +24,10 @@ __all__ = [
     "generate_private_key",
     "generate_public_key",
     "hkdf",
+    "notify_registration",
     "notify_user",
     "public_key_standard_b64",
     "register",
+    "shutdown_providers",
+    "unregister",
 ]

devplacepy/push/delivery.py

@@ -3,9 +3,9 @@
 import logging
 from typing import Any
 
-from devplacepy import stealth
 from devplacepy.database import get_int_setting
 from devplacepy.push import providers, store
+from devplacepy.push.providers.base import Delivery
 
 logger = logging.getLogger(__name__)
 
@@ -29,6 +29,10 @@ def group_by_provider(
     return grouped
 
 
+def _open_client(provider, timeout: float):
+    return provider.delivery_client(timeout)
+
+
 async def notify_user(user_uid: str, payload: dict[str, Any]) -> None:
     registrations = store.active_for_user(user_uid)
     if not registrations:
@@ -36,48 +40,86 @@ async def notify_user(user_uid: str, payload: dict[str, Any]) -> None:
         return
 
     grouped = group_by_provider(registrations)
-    async with stealth.stealth_async_client(timeout=timeout_seconds()) as client:
-        for name, rows in grouped.items():
-            provider = providers.PROVIDERS.get(name)
-            if provider is None:
-                logger.warning(
-                    "Unknown push provider %s on %s subscriptions of user %s",
-                    name,
-                    len(rows),
-                    user_uid,
-                )
-                continue
-            if not providers.is_active(provider):
-                logger.debug(
-                    "Push provider %s is not active; skipping %s subscriptions",
-                    name,
-                    len(rows),
-                )
-                continue
-            try:
-                prepared = provider.prepare(payload)
-            except Exception as exc:
-                logger.error("Push provider %s could not build a payload: %s", name, exc)
-                continue
+    timeout = timeout_seconds()
+    for name, rows in grouped.items():
+        provider = providers.PROVIDERS.get(name)
+        if provider is None:
+            logger.warning(
+                "Unknown push provider %s on %s subscriptions of user %s",
+                name,
+                len(rows),
+                user_uid,
+            )
+            continue
+        if not providers.is_active(provider):
+            logger.debug(
+                "Push provider %s is not active; skipping %s subscriptions",
+                name,
+                len(rows),
+            )
+            continue
+        try:
+            prepared = provider.prepare(payload)
+        except Exception as exc:
+            logger.error("Push provider %s could not build a payload: %s", name, exc)
+            continue
+        try:
+            client = _open_client(provider, timeout)
+        except Exception as exc:
+            logger.error("Push provider %s could not open a client: %s", name, exc)
+            continue
+        try:
             for registration in rows:
                 await _deliver_one(provider, client, registration, prepared, user_uid)
+        finally:
+            if provider.closes_delivery_client():
+                await client.aclose()
 
 
-async def _deliver_one(provider, client, registration, prepared, user_uid) -> None:
+async def notify_registration(
+    registration: dict[str, Any], payload: dict[str, Any]
+) -> Delivery:
+    name = store.provider_of(registration)
+    provider = providers.PROVIDERS.get(name)
+    if provider is None:
+        return Delivery(providers.REJECTED, f"unknown provider {name}")
+    if not providers.is_active(provider):
+        return Delivery(providers.REJECTED, f"provider {name} is not active")
+    try:
+        prepared = provider.prepare(payload)
+    except Exception as exc:
+        return Delivery(providers.REJECTED, str(exc))
+    user_uid = registration.get("user_uid") or ""
+    try:
+        client = _open_client(provider, timeout_seconds())
+    except Exception as exc:
+        return Delivery(providers.REJECTED, str(exc))
+    try:
+        return await _deliver_one(provider, client, registration, prepared, user_uid)
+    finally:
+        if provider.closes_delivery_client():
+            await client.aclose()
+
+
+async def _deliver_one(provider, client, registration, prepared, user_uid) -> Delivery:
     try:
         outcome = await provider.deliver(client, registration, prepared)
     except Exception as exc:
         logger.error("Push provider %s raised for %s: %s", provider.name, user_uid, exc)
-        return
+        return Delivery(providers.REJECTED, str(exc))
     if outcome.status == providers.ACCEPTED:
         logger.debug("Push delivered to %s via %s", user_uid, provider.name)
-        return
+        return outcome
     if outcome.status == providers.DEAD:
+        logger.warning(
+            "Push dead via %s for %s: %s", provider.name, user_uid, outcome.detail
+        )
         try:
-            store.mark_dead(registration["id"])
+            store.mark_dead(registration["id"], outcome.dead_before)
         except Exception as exc:
             logger.error("Could not soft-delete push subscription: %s", exc)
-        return
+        return outcome
     logger.warning(
         "Push rejected by %s for %s: %s", provider.name, user_uid, outcome.detail
     )
+    return outcome

devplacepy/push/providers/__init__.py

@@ -35,6 +35,7 @@ __all__ = [
     "get",
     "is_active",
     "names",
+    "shutdown",
 ]
 
 
@@ -74,3 +75,11 @@ def _client_config(provider: PushProvider) -> dict[str, Any]:
     except Exception as exc:
         logger.error("Push provider %s failed to describe itself: %s", provider.name, exc)
         return {}
+
+
+async def shutdown() -> None:
+    for provider in PROVIDERS.values():
+        try:
+            await provider.aclose()
+        except Exception as exc:
+            logger.error("Push provider %s failed to close: %s", provider.name, exc)

devplacepy/push/providers/apns.py

@@ -5,13 +5,14 @@ import json
 import logging
 import string
 import time
+from datetime import datetime, timezone
 from typing import Any
 
 import httpx
 import jwt
 
 from devplacepy.config import SECONDS_PER_DAY
-from devplacepy.database import get_setting
+from devplacepy.database import get_setting, set_setting
 from devplacepy.push.providers.base import (
     ACCEPTED,
     DEAD,
@@ -44,20 +45,27 @@ ENVIRONMENT_OPTIONS = [
 TOKEN_REFRESH_SECONDS = 45 * 60
 TOKEN_MIN_LENGTH = 64
 TOKEN_MAX_LENGTH = 200
+CLIENT_ID_MAX_LENGTH = 128
 THREAD_ID = "devplace-notification"
 PUSH_TYPE = "alert"
 PRIORITY = "10"
-DEAD_REASONS = frozenset(
+SHARED_TOKEN_KEY = "push_apns_shared_token"
+
+DEAD_REASONS = frozenset({"BadDeviceToken", "ExpiredToken", "Unregistered"})
+
+AUTH_REASONS = frozenset(
     {
-        "BadDeviceToken",
-        "DeviceTokenNotForTopic",
-        "ExpiredToken",
-        "Unregistered",
-        "TopicDisallowed",
+        "BadCertificate",
+        "BadCertificateEnvironment",
+        "ExpiredProviderToken",
+        "Forbidden",
+        "InvalidProviderToken",
+        "MissingProviderToken",
     }
 )
 
 _token_state: dict[str, Any] = {}
+_client: httpx.AsyncClient | None = None
 
 
 def _setting(key: str) -> str:
@@ -70,13 +78,78 @@ def _environment() -> str:
 
 
 def host() -> str:
-    return HOSTS[_environment()]
+    return host_for(_environment())
+
+
+def host_for(environment: str | None) -> str:
+    value = (environment or "").strip() or _environment()
+    return HOSTS[value] if value in HOSTS else HOSTS[DEFAULT_ENVIRONMENT]
+
+
+def gateway_client(timeout: float) -> httpx.AsyncClient:
+    return httpx.AsyncClient(
+        http2=True,
+        timeout=timeout,
+        trust_env=False,
+        verify=True,
+        follow_redirects=False,
+    )
+
+
+def cached_client(timeout: float) -> httpx.AsyncClient:
+    global _client
+    if _client is None or _client.is_closed:
+        _client = gateway_client(timeout)
+    return _client
+
+
+async def close_client() -> None:
+    global _client
+    if _client is not None and not _client.is_closed:
+        await _client.aclose()
+    _client = None
+
+
+def _normalize_token(token: str) -> str:
+    return "".join(
+        character.lower() for character in token if character in string.hexdigits
+    )
 
 
 def _fingerprint(team_id: str, key_id: str, auth_key: str) -> str:
     return hashlib.sha256(f"{team_id}:{key_id}:{auth_key}".encode("utf-8")).hexdigest()
 
 
+def _read_shared_token() -> dict[str, Any] | None:
+    raw = get_setting(SHARED_TOKEN_KEY, "")
+    if not raw:
+        return None
+    try:
+        data = json.loads(raw)
+    except ValueError:
+        return None
+    if (
+        not isinstance(data, dict)
+        or not isinstance(data.get("fingerprint"), str)
+        or not isinstance(data.get("token"), str)
+        or not isinstance(data.get("issued_at"), int)
+    ):
+        return None
+    return data
+
+
+def _write_shared_token(fingerprint: str, token: str, issued_at: int) -> None:
+    set_setting(
+        SHARED_TOKEN_KEY,
+        json.dumps({"fingerprint": fingerprint, "token": token, "issued_at": issued_at}),
+    )
+
+
+def invalidate_provider_token() -> None:
+    _token_state.pop("current", None)
+    set_setting(SHARED_TOKEN_KEY, "")
+
+
 def provider_token(team_id: str, key_id: str, auth_key: str) -> str:
     fingerprint = _fingerprint(team_id, key_id, auth_key)
     issued_at = int(time.time())
@@ -89,6 +162,21 @@ def provider_token(team_id: str, key_id: str, auth_key: str) -> str:
         if state["token"] is None:
             raise ValueError(state["error"])
         return state["token"]
+
+    shared = _read_shared_token()
+    if (
+        shared
+        and shared["fingerprint"] == fingerprint
+        and issued_at - shared["issued_at"] < TOKEN_REFRESH_SECONDS
+    ):
+        _token_state["current"] = {
+            "token": shared["token"],
+            "error": "",
+            "issued_at": shared["issued_at"],
+            "fingerprint": fingerprint,
+        }
+        return shared["token"]
+
     try:
         token = jwt.encode(
             {"iss": team_id, "iat": issued_at},
@@ -112,6 +200,7 @@ def provider_token(team_id: str, key_id: str, auth_key: str) -> str:
         "issued_at": issued_at,
         "fingerprint": fingerprint,
     }
+    _write_shared_token(fingerprint, token, issued_at)
     return token
 
 
@@ -125,6 +214,22 @@ def _reason(response: httpx.Response) -> str:
     return ""
 
 
+def _dead_before(response: httpx.Response) -> str | None:
+    try:
+        body = response.json()
+    except ValueError:
+        return None
+    if not isinstance(body, dict):
+        return None
+    raw = body.get("timestamp")
+    if not isinstance(raw, (int, float)) or isinstance(raw, bool):
+        return None
+    try:
+        return datetime.fromtimestamp(raw / 1000, tz=timezone.utc).isoformat()
+    except (OverflowError, OSError, ValueError):
+        return None
+
+
 class ApnsProvider(PushProvider):
     name = "apns"
     label = PROVIDER_LABEL
@@ -181,16 +286,41 @@ class ApnsProvider(PushProvider):
             and _setting(TOPIC_KEY)
         )
 
+    def client_config(self) -> dict[str, Any]:
+        return {"environment": _environment()}
+
+    def stamp_registration(self, fields: dict[str, Any]) -> dict[str, Any]:
+        return {**fields, "environment": _environment()}
+
+    def delivery_client(self, timeout: float) -> httpx.AsyncClient:
+        return cached_client(timeout)
+
+    def closes_delivery_client(self) -> bool:
+        return False
+
+    async def aclose(self) -> None:
+        await close_client()
+
     def parse_registration(self, body: dict[str, Any]) -> dict[str, Any] | None:
         token = body.get("token")
         if not isinstance(token, str):
             return None
-        token = token.strip()
+        token = _normalize_token(token)
         if not TOKEN_MIN_LENGTH <= len(token) <= TOKEN_MAX_LENGTH:
             return None
-        if any(character not in string.hexdigits for character in token):
+        fields: dict[str, Any] = {"token": token}
+        client_id = body.get("client_id")
+        if client_id is None:
+            return fields
+        if not isinstance(client_id, str):
             return None
-        return {"token": token}
+        client_id = client_id.strip()
+        if not client_id:
+            return fields
+        if len(client_id) > CLIENT_ID_MAX_LENGTH:
+            return None
+        fields["client_id"] = client_id
+        return fields
 
     def prepare(self, payload: dict[str, Any]) -> str:
         return json.dumps(
@@ -228,7 +358,7 @@ class ApnsProvider(PushProvider):
         try:
             headers = self.headers()
             response = await client.post(
-                f"https://{host()}/3/device/{token}",
+                f"https://{host_for(registration.get('environment'))}/3/device/{token}",
                 headers=headers,
                 content=prepared.encode("utf-8"),
             )
@@ -238,6 +368,9 @@ class ApnsProvider(PushProvider):
             return Delivery(ACCEPTED)
         reason = _reason(response)
         detail = f"{response.status_code} {reason}".strip()
+        if response.status_code == 403 or reason in AUTH_REASONS:
+            invalidate_provider_token()
         if response.status_code == 410 or reason in DEAD_REASONS:
-            return Delivery(DEAD, detail)
+            dead_before = _dead_before(response) if response.status_code == 410 else None
+            return Delivery(DEAD, detail, dead_before)
         return Delivery(REJECTED, detail)

devplacepy/push/providers/base.py

@@ -18,6 +18,7 @@ REJECTED = "rejected"
 class Delivery:
     status: str
     detail: str = ""
+    dead_before: str | None = None
 
 
 class PushProvider(ABC):
@@ -51,6 +52,20 @@ class PushProvider(ABC):
     def client_config(self) -> dict[str, Any]:
         return {}
 
+    def stamp_registration(self, fields: dict[str, Any]) -> dict[str, Any]:
+        return fields
+
+    def delivery_client(self, timeout: float) -> httpx.AsyncClient:
+        from devplacepy import stealth
+
+        return stealth.stealth_async_client(timeout=timeout)
+
+    def closes_delivery_client(self) -> bool:
+        return True
+
+    async def aclose(self) -> None:
+        return None
+
     @abstractmethod
     def is_configured(self) -> bool: ...
 

devplacepy/push/store.py

@@ -1,6 +1,7 @@
 # retoor <retoor@molodetz.nl>
 
 import logging
+from dataclasses import dataclass
 from datetime import datetime, timezone
 from typing import Any
 
@@ -13,6 +14,17 @@ logger = logging.getLogger(__name__)
 TABLE = "push_registration"
 
 
+@dataclass(frozen=True)
+class RegistrationWrite:
+    record: dict[str, Any]
+    created: bool
+    revived: bool
+
+    @property
+    def probe(self) -> bool:
+        return self.created or self.revived
+
+
 def table():
     return get_table(TABLE)
 
@@ -25,31 +37,185 @@ def active_for_user(user_uid: str) -> list[dict[str, Any]]:
     return list(table().find(user_uid=user_uid, deleted_at=None))
 
 
+def _filled(fields: dict[str, Any]) -> dict[str, Any]:
+    filled: dict[str, Any] = {}
+    for key, value in fields.items():
+        if value is None:
+            continue
+        if isinstance(value, str):
+            value = value.strip()
+            if not value:
+                continue
+        filled[key] = value
+    return filled
+
+
+def _prefer_live(rows: list[dict[str, Any]]) -> dict[str, Any] | None:
+    if not rows:
+        return None
+    live = [row for row in rows if not row.get("deleted_at")]
+    return (live or rows)[-1]
+
+
+def _lookup(user_uid: str, provider: str, **identity: Any) -> dict[str, Any] | None:
+    return _prefer_live(
+        list(table().find(user_uid=user_uid, provider=provider, **identity))
+    )
+
+
+def _with_id(record: dict[str, Any]) -> dict[str, Any]:
+    if record.get("id"):
+        return record
+    found = table().find_one(uid=record.get("uid"))
+    return found or record
+
+
+def _retire_duplicates(
+    user_uid: str,
+    provider: str,
+    keep_id: int,
+    fields: dict[str, Any],
+    previous_token: str | None = None,
+) -> None:
+    token = fields.get("token")
+    client_id = fields.get("client_id")
+    for row in table().find(user_uid=user_uid, provider=provider, deleted_at=None):
+        if row["id"] == keep_id:
+            continue
+        if client_id and row.get("client_id") == client_id:
+            mark_dead(row["id"])
+            continue
+        if token and row.get("token") == token:
+            mark_dead(row["id"])
+            continue
+        if previous_token and row.get("token") == previous_token:
+            mark_dead(row["id"])
+
+
+def _merge(
+    existing: dict[str, Any], fields: dict[str, Any]
+) -> RegistrationWrite:
+    revived = bool(existing.get("deleted_at"))
+    patch: dict[str, Any] = {"id": existing["id"]}
+    if revived:
+        patch["deleted_at"] = None
+    changed = revived
+    for key, value in fields.items():
+        if existing.get(key) != value:
+            patch[key] = value
+            changed = True
+    if not changed:
+        return RegistrationWrite(_with_id(existing), False, False)
+    patch["registered_at"] = datetime.now(timezone.utc).isoformat()
+    previous_token = existing.get("token")
+    table().update(patch, ["id"])
+    merged = {**existing, **patch}
+    if revived:
+        merged["deleted_at"] = None
+    merged = _with_id(merged)
+    _retire_duplicates(
+        existing["user_uid"],
+        existing["provider"],
+        merged["id"],
+        fields,
+        previous_token=previous_token if previous_token != fields.get("token") else None,
+    )
+    logger.info(
+        "Updated %s push subscription for user %s%s",
+        existing.get("provider"),
+        existing.get("user_uid"),
+        " (revived)" if revived else "",
+    )
+    return RegistrationWrite(merged, False, revived)
+
+
 def register(
     user_uid: str, provider: str, fields: dict[str, Any]
-) -> tuple[dict[str, Any], bool]:
+) -> RegistrationWrite:
+    fields = _filled(fields)
     registrations = table()
-    existing = registrations.find_one(
+    client_id = fields.get("client_id")
+    token = fields.get("token")
+    endpoint = fields.get("endpoint")
+
+    if client_id:
+        existing = _lookup(user_uid, provider, client_id=client_id)
+        if existing:
+            return _merge(existing, fields)
+
+    if token:
+        existing = _lookup(user_uid, provider, token=token)
+        if existing:
+            return _merge(existing, fields)
+
+    if endpoint:
+        existing = _lookup(user_uid, provider, endpoint=endpoint)
+        if existing:
+            return _merge(existing, fields)
+
+    live = registrations.find_one(
         user_uid=user_uid, provider=provider, deleted_at=None, **fields
     )
-    if existing:
-        logger.debug("Push subscription already registered for user %s", user_uid)
-        return existing, False
+    if live:
+        return RegistrationWrite(_with_id(live), False, False)
 
+    now = datetime.now(timezone.utc).isoformat()
     record = {
         "uid": generate_uid(),
         "user_uid": user_uid,
         "provider": provider,
-        "created_at": datetime.now(timezone.utc).isoformat(),
+        "created_at": now,
+        "registered_at": now,
         "deleted_at": None,
         **fields,
     }
-    registrations.insert(record)
+    inserted = registrations.insert(record)
+    if isinstance(inserted, int):
+        record["id"] = inserted
+    record = _with_id(record)
+    if record.get("id"):
+        _retire_duplicates(user_uid, provider, record["id"], fields)
     logger.info("Registered %s push subscription for user %s", provider, user_uid)
-    return record, True
+    return RegistrationWrite(record, True, False)
 
 
-def mark_dead(registration_id: int) -> None:
+def unregister(user_uid: str, provider: str, fields: dict[str, Any]) -> bool:
+    fields = _filled(fields)
+    client_id = fields.get("client_id")
+    token = fields.get("token")
+    endpoint = fields.get("endpoint")
+
+    row = None
+    if client_id:
+        row = _lookup(user_uid, provider, client_id=client_id)
+    if row is None and token:
+        row = _lookup(user_uid, provider, token=token)
+    if row is None and endpoint:
+        row = _lookup(user_uid, provider, endpoint=endpoint)
+    if row is None or row.get("deleted_at"):
+        return False
+
+    table().update(
+        {"id": row["id"], "deleted_at": datetime.now(timezone.utc).isoformat()},
+        ["id"],
+    )
+    logger.info("Unregistered %s push subscription for user %s", provider, user_uid)
+    return True
+
+
+def mark_dead(registration_id: int, dead_before: str | None = None) -> None:
+    if dead_before is not None:
+        row = table().find_one(id=registration_id)
+        registered_at = row.get("registered_at") if row else None
+        if registered_at and registered_at > dead_before:
+            logger.info(
+                "Skipped marking push subscription id=%s dead: registered again at %s "
+                "after the provider confirmed it dead at %s",
+                registration_id,
+                registered_at,
+                dead_before,
+            )
+            return
     table().update(
         {"id": registration_id, "deleted_at": datetime.now(timezone.utc).isoformat()},
         ["id"],

devplacepy/rendering.py

@@ -70,6 +70,26 @@ _TOKEN_RE = re.compile(
 _SCHEME_RE = re.compile(r"^([a-z][a-z0-9+.\-]*):", re.I)
 _ALLOWED_SCHEMES = {"http", "https", "mailto", "tel"}
 
+_TRAILING_PUNCT = ".,;:!?"
+_TRAILING_CLOSERS = {")": "(", "]": "[", "}": "{"}
+
+
+def _split_trailing_punct(url: str) -> tuple[str, str]:
+    trailing: list[str] = []
+    while url:
+        char = url[-1]
+        if char in _TRAILING_PUNCT:
+            trailing.append(char)
+            url = url[:-1]
+            continue
+        opener = _TRAILING_CLOSERS.get(char)
+        if opener is not None and url.count(char) > url.count(opener):
+            trailing.append(char)
+            url = url[:-1]
+            continue
+        break
+    return url, "".join(reversed(trailing))
+
 _YOUTUBE_ALLOW = (
     "accelerometer; autoplay; clipboard-write; encrypted-media; "
     "gyroscope; picture-in-picture"
@@ -112,6 +132,40 @@ _content_markdown = mistune.create_markdown(
     plugins=["strikethrough", "table"],
 )
 
+_structure_markdown = mistune.create_markdown(
+    renderer=None,
+    plugins=["strikethrough", "table"],
+)
+
+_STRUCTURE_TOKEN_KEYS = {
+    "block_code": "code_fences",
+    "list_item": "list_items",
+    "heading": "headers",
+    "link": "links",
+}
+
+
+def _count_structure_tokens(tokens, signature: dict[str, int]) -> None:
+    for token in tokens or []:
+        key = _STRUCTURE_TOKEN_KEYS.get(token.get("type"))
+        if key:
+            signature[key] += 1
+        children = token.get("children")
+        if children:
+            _count_structure_tokens(children, signature)
+
+
+def markdown_structure_signature(text: str) -> dict[str, int]:
+    signature = {key: 0 for key in _STRUCTURE_TOKEN_KEYS.values()}
+    if not text or not text.strip():
+        return signature
+    try:
+        tokens = _structure_markdown(text)
+    except (TypeError, ValueError):
+        return signature
+    _count_structure_tokens(tokens, signature)
+    return signature
+
 
 def _normalize_dashes(text: str) -> str:
     text = text.replace("\u2014", "-")
@@ -183,7 +237,10 @@ def _transform_text(text: str) -> str:
         if match.start() > pos:
             out.append(html.escape(_mask_emails(text[pos:match.start()])))
         if match.group("url"):
-            out.append(_embed_url(match.group("url")))
+            url, trailing = _split_trailing_punct(match.group("url"))
+            out.append(_embed_url(url))
+            if trailing:
+                out.append(html.escape(trailing))
         else:
             user = match.group("mention")
             out.append(
@@ -335,3 +392,21 @@ def content_preview(text, length: int = 60) -> str:
     if len(plain) <= length:
         return plain
     return plain[:length].rstrip() + "..."
+
+
+def safe_truncate(text, length: int = 300) -> str:
+    if not text:
+        return ""
+    text_str = str(text)
+    if len(text_str) <= length:
+        return text_str
+    cutoff = length
+    for match in _TOKEN_RE.finditer(text_str):
+        if match.start() < cutoff < match.end():
+            cutoff = match.start()
+            break
+    if cutoff > 0 and not text_str[cutoff - 1].isspace() and not text_str[cutoff].isspace():
+        boundary = text_str.rfind(" ", 0, cutoff)
+        if boundary > 0:
+            cutoff = boundary
+    return text_str[:cutoff].rstrip()

devplacepy/routers/CLAUDE.md

@@ -11,15 +11,17 @@ Prefixes are wired in `main.py`:
 | `/auth` | auth/ package - one leaf per flow (`signup`, `login`, `logout`, `forgotpassword`, `resetpassword`) |
 | `/feed` | feed.py |
 | `/posts` | posts.py |
+| `/topics` | topics.py - crawlable per-topic category index pages (`GET /topics` hub, `GET /topics/{topic}` per-topic post listing over the same `TOPICS` set as the feed sidebar filter). Reuses `feed.py`'s `get_feed_posts`/`enrich_post_cards` so a topic page is a fully-enriched `_post_card.html` listing, not a stripped-down duplicate. Unlike `/feed?topic=X` (whose canonical strips the query string back to bare `/feed`, so it is never indexed as a distinct page), each `/topics/{topic}` page has its own canonical URL, unique title/description, breadcrumbs, and a sitemap entry - see "SEO implementation" below |
 | `/comments` | comments.py |
 | `/projects` | projects/ package - `index.py` (listing/detail/create/delete plus owner-only visibility toggles `POST /{slug}/private` and `POST /{slug}/readonly`, and async `POST /{slug}/fork`), `files.py`, and the `containers/` subpackage (below). `main.py` mounts the whole `/projects` tree from this one package. See `routers/projects/CLAUDE.md` for the deep detail on this tree |
 | `/projects/{slug}/files` | projects/files.py - per-project virtual filesystem (CRUD dirs/files, upload, inline edit, plus line-range ops: `lines` read, `replace-lines`, `insert-lines`, `delete-lines`, `append` for large text files) |
 | `/profile` | profile/ package - `index.py` (page/search/followers/update/api-key; the **Posts** tab shows only the 10 most recent posts, while the sidebar `posts_count` stat always uses the real `count()` so it is never capped to the rendered 10), `customization.py`, `notifications.py`, `ai_correction.py` (`POST /{username}/ai-correction`, owner-or-admin AI content correction toggle + prompt), `ai_modifier.py` (`POST /{username}/ai-modifier`, owner-or-admin AI modifier toggle + prompt), `interactions.py` (`POST /{username}/interactions`, owner-or-admin Devii interactive widgets preference; admin default via `devii_interactions_default`), `telegram.py` (`POST /{username}/telegram`, owner-or-admin Telegram pairing-code request/unpair), `usage.py` (`_ai_quota`, re-exported from the package) |
-| `/messages` | messages.py - real-time DM chat. `GET ""` (page; the conversation sidebar is one windowed SQL aggregate and an opened thread loads only the newest `CONVERSATION_MESSAGE_LIMIT` = 500 messages - keep these queries bounded), `GET /search`, `GET /conversations` (flat JSON conversation list, same `get_conversations` helper as the page), `POST /send` (no-JS fallback, also accepts an attachment-only empty-content message), `POST /ws-ticket` (issues a 30s single-use WS auth ticket so a browser WebSocket can authenticate cross-context, since a native `WebSocket` cannot set an `Authorization`/`X-API-KEY` header), and `WS /messages/ws` (NOT lock-owner gated - accepts on every worker; closes `1008` for guests). The WS adds live bidirectional delivery, typing, and read receipts on top of the existing `messages` table - presence is NOT part of this WS (it is the generic `PresenceManager`/pub/sub mechanism, see `devplacepy/services/CLAUDE.md`). Both the WS `send` and HTTP `POST /send` route through the shared `services/messaging/persist.py` `persist_message` choke point (identical audit/notification/mention) and then through `_finalize_and_broadcast`, which awaits any pending SYNC AI correction/modifier futures (DM content runs through both, so typing `@ai <instruction>` executes live), broadcasts the FINAL corrected/modified content with an additive `ai_processed` frame flag (the WS path passes `request=websocket` and awaits directly since HTTP middleware does not run for websockets; the frontend swaps the sender's optimistic bubble to the authoritative content). Same-worker delivery is instant via the per-worker `message_hub`; cross-worker delivery is filled by `services/messaging/relay.py` `message_relay` (per-worker ~1s DB poll on the `messages.id` watermark, deduped against the hub's delivered set). See `devplacepy/services/messaging/CLAUDE.md` |
+| `/messages` | messages.py - real-time DM chat. `GET ""` (page; the conversation sidebar is one windowed SQL aggregate and an opened thread loads only the newest `CONVERSATION_MESSAGE_LIMIT` = 500 messages - keep these queries bounded), `GET /search`, `GET /conversations` (flat JSON conversation list, same `get_conversations` helper as the page), `POST /send` (no-JS fallback, also accepts an attachment-only empty-content message), `POST /ws-ticket` (issues a 30s single-use WS auth ticket so a browser WebSocket can authenticate cross-context, since a native `WebSocket` cannot set an `Authorization`/`X-API-KEY` header), and `WS /messages/ws` (NOT lock-owner gated - accepts on every worker; closes `1008` for guests). The WS adds live bidirectional delivery, typing, and read receipts on top of the existing `messages` table - presence is NOT part of this WS (it is the generic `PresenceManager`/pub/sub mechanism, see `devplacepy/services/CLAUDE.md`). Both the WS `send` and HTTP `POST /send` route through the shared `services/messaging/persist.py` `persist_message` choke point (identical audit/notification/mention) and then through `_finalize_and_broadcast`, which broadcasts the persisted row immediately and then applies SYNC AI correction/modifier (HTTP awaits so the JSON body is final; WS schedules it so the receive loop never blocks). An in-place rewrite stamps `messages.updated_at` and emits a second `ai_processed` frame; other workers pick it up from `message_relay._tick_updates`. Same-worker delivery is instant via the per-worker `message_hub`; cross-worker delivery is filled by `services/messaging/relay.py` `message_relay` (per-worker ~1s DB poll on the `messages.id` watermark, plus `updated_at` for revisions, new rows deduped against the hub's delivered set). See `devplacepy/services/messaging/CLAUDE.md` |
 | `/notifications` | notifications.py |
 | `/votes` | votes.py |
 | `/reactions` | reactions.py - emoji reaction toggle on any target: `POST /reactions/{target_type}/{target_uid}` (optimistic `ReactionBar`) |
 | `/bookmarks` | bookmarks.py - bookmark/favorite toggle: `GET /bookmarks/saved` (the viewer's saved list) and `POST /bookmarks/{target_type}/{target_uid}` (toggle a bookmark) |
+| `/notes` | notes.py - private per-user annotation on a target: `GET /notes/saved` (the viewer's personal notes list), `POST /notes/{target_type}/{target_uid}` (add/replace the note), `POST /notes/{target_type}/{target_uid}/delete` (remove it) |
 | `/polls` | polls.py - poll voting: `POST /polls/{poll_uid}/vote` |
 | `/avatar` | avatar.py |
 | `/follow` | follow.py |
@@ -38,8 +40,8 @@ Prefixes are wired in `main.py`:
 | `/devii` | devii.py - WebSocket terminal (`/devii/ws`), page, `/devii/usage`, `/devii/session` |
 | `/zips` | zips.py - generic zip job status (`/zips/{uid}`) and archive download (`/zips/{uid}/download`); enqueued from `/projects/{slug}/zip` and `/projects/{slug}/files/zip` |
 | `/forks` | forks.py - fork job status (`/forks/{uid}`); enqueued from `/projects/{slug}/fork`. When done its `project_url` points at the new forked project |
-| `/tools` | tools/ package - public developer tools surface. `index.py` (`/tools` landing) plus `seo.py` (**SEO Diagnostics**): `GET /tools/seo` page, `POST /tools/seo/run` (enqueue `seo` job, per-owner one-active-job cap), `GET /tools/seo/{uid}` (`SeoJobOut`), `GET /tools/seo/{uid}/report` (HTML+JSON `SeoReportOut`), `WS /tools/seo/{uid}/ws` (live progress, lock-owner gated, close `4013` retry), `GET /tools/seo/{uid}/screenshot/{n}`. Also `deepsearch.py` (**DeepSearch**): `GET /tools/deepsearch` page, `POST /tools/deepsearch/run` (enqueue `deepsearch` job, per-owner one-active-job cap; resolves the user `api_key` into the payload, guests use the internal key), `GET /tools/deepsearch/{uid}` (`DeepsearchJobOut`), `GET /tools/deepsearch/{uid}/session` (HTML+JSON `DeepsearchSessionOut`), `WS /tools/deepsearch/{uid}/ws` (live progress, lock-owner gated, `4013` retry), `WS /tools/deepsearch/{uid}/chat` (grounded RAG chat over the session collection), `POST /tools/deepsearch/{uid}/{pause|resume|cancel}`, `GET /tools/deepsearch/{uid}/export.{md,json,pdf}`. The shared owner helper is `routers/tools/_shared.py` `owner_for`. Also `isslop.py` (**AI Usage Analyzer**): `GET /tools/isslop` page, `POST /tools/isslop/run` (enqueue `isslop` job, per-owner one-active-job cap; owner = user uid or the shared `DEVII_GUEST_COOKIE` guest identity, minted when absent), `GET /tools/isslop/list` (owner history; a signed-in request first claims any guest-cookie analyses via `store.claim_guest_analyses` - a move, never a copy), `GET /tools/isslop/{uid}` (`IsslopAnalysisOut`), `GET /tools/isslop/{uid}/events` (persisted ordered event trail, `?after=SEQ` incremental poll; live frames also publish on pub/sub `public.isslop.{uid}` - the DB trail is the source of truth, pub/sub the fast path), `GET /tools/isslop/{uid}/report` (HTML+JSON `IsslopReportOut`; live `<dp-isslop-run>` while running, server-rendered report via `render_content` when done), `GET /tools/isslop/{uid}/report.md`, `GET /tools/isslop/{uid}/badge.svg` (embeddable SVG authenticity badge). Analyses/reports/badges are permanent public capability URLs (`IsslopService.cleanup` never deletes them; only the job row is swept). Surfaced by a collapsible **Tools** header dropdown (`base.html`, visible to all) |
-| `/projects/{slug}/containers` | projects/containers/ subpackage - admin per-project container manager (`instances.py` for creation/lifecycle/exec/logs/metrics/sync plus the exec websocket, `schedules.py` for cron/interval/once schedules, shared helpers in `_shared.py`). Every instance runs the shared `ppy` image. Discoverable from the project detail page (admin-only **Containers** button) and from the admin index |
+| `/tools` | tools/ package - public developer tools surface. `index.py` (`/tools` landing) plus `seo.py` (**SEO Diagnostics**): `GET /tools/seo` page, `POST /tools/seo/run` (enqueue `seo` job, per-owner one-active-job cap), `GET /tools/seo/{uid}` (`SeoJobOut`), `GET /tools/seo/{uid}/report` (HTML+JSON `SeoReportOut`), `WS /tools/seo/{uid}/ws` (live progress, lock-owner gated, close `4013` retry), `GET /tools/seo/{uid}/screenshot/{n}`. Also `deepsearch.py` (**DeepSearch**): `GET /tools/deepsearch` page, `POST /tools/deepsearch/run` (enqueue `deepsearch` job, per-owner one-active-job cap; resolves the user `api_key` into the payload, guests use the internal key), `GET /tools/deepsearch/history` (HTML+JSON `DeepsearchHistoryOut`, owner's own past sessions newest-first with a `reopen_url` back into the session/chat routes - `database.list_deepsearch_sessions`, the `deepsearch_sessions` table already persists past runs independently of job retention), `GET /tools/deepsearch/{uid}` (`DeepsearchJobOut`), `GET /tools/deepsearch/{uid}/session` (HTML+JSON `DeepsearchSessionOut`), `WS /tools/deepsearch/{uid}/ws` (live progress, lock-owner gated, `4013` retry), `WS /tools/deepsearch/{uid}/chat` (grounded RAG chat over the session collection, reconnectable on any completed session within its job's retention window), `POST /tools/deepsearch/{uid}/{pause|resume|cancel}`, `GET /tools/deepsearch/{uid}/export.{md,json,pdf}`. The shared owner helper is `routers/tools/_shared.py` `owner_for`. Also `isslop.py` (**AI Usage Analyzer**): `GET /tools/isslop` page, `POST /tools/isslop/run` (enqueue `isslop` job, per-owner one-active-job cap; owner = user uid or the shared `DEVII_GUEST_COOKIE` guest identity, minted when absent), `GET /tools/isslop/list` (owner history; a signed-in request first claims any guest-cookie analyses via `store.claim_guest_analyses` - a move, never a copy), `GET /tools/isslop/{uid}` (`IsslopAnalysisOut`), `GET /tools/isslop/{uid}/events` (persisted ordered event trail, `?after=SEQ` incremental poll; live frames also publish on pub/sub `public.isslop.{uid}` - the DB trail is the source of truth, pub/sub the fast path), `GET /tools/isslop/{uid}/report` (HTML+JSON `IsslopReportOut`; live `<dp-isslop-run>` while running, server-rendered report via `render_content` when done), `GET /tools/isslop/{uid}/report.md`, `GET /tools/isslop/{uid}/badge.svg` (embeddable SVG authenticity badge). Analyses/reports/badges are permanent public capability URLs (`IsslopService.cleanup` never deletes them; only the job row is swept). Reachable by direct URL only; the former **Tools** header dropdown in `base.html` was removed |
+| `/projects/{slug}/containers` | projects/containers/ subpackage - admin per-project container manager (`instances.py` for creation/lifecycle/exec/logs/metrics/sync plus the exec websocket, `schedules.py` for cron/interval/once schedules, shared helpers in `_shared.py`). Every instance runs the shared `ppy` image. Reachable by direct URL and from the admin index; the former project detail page **Containers** button was removed |
 | `/admin/containers` | admin/containers.py - admin **Containers** manager: `/admin/containers` lists every instance across all projects with inline actions (start/stop/restart/terminal/edit/delete) plus a create modal (project search-select, run-as user search-select, boot language + source editor, restart policy, start-on-boot, env/ports/limits/ingress); `/admin/containers/{uid}` is the per-instance detail page (lifecycle, live logs/metrics, PTY terminal, schedules, ingress, sync, status history) and `/admin/containers/{uid}/edit` edits run-as user, boot language/script/command, restart policy, start-on-boot, and limits. `POST /admin/containers/create`, `/{uid}/edit`, `/{uid}/{start,stop,restart,pause,resume}`, `/{uid}/sync`, `/{uid}/delete` call `api.*` directly under `require_admin` (no docker/exec backend duplicated); `GET /admin/containers/projects/search` and `/users/search` back the create/edit search-selects. The lifecycle and detail views stay layered over the per-project `/projects/{slug}/containers/instances/{uid}/...` endpoints (the instance carries its `project_uid`) |
 | `/p/{slug}` | proxy.py - public ingress reverse proxy (HTTP + WebSocket) to a running container instance's published host port, opt-in per instance via `ingress_slug` |
 | `/xmlrpc` | xmlrpc.py - reverse-proxies XML-RPC calls to the forking XML-RPC bridge (`services/xmlrpc/`, supervised by `XmlrpcService` on loopback `config.XMLRPC_PORT`), which generates one XML-RPC method per documented REST endpoint from `docs_api.API_GROUPS` (`posts.create`, `feed.list`, ...). One struct of named params per call; auth via in-band `api_key`, `X-API-KEY`/`Bearer` header, or `http://user:pass@host/xmlrpc` Basic. Full introspection + `system.multicall`; REST errors become XML-RPC faults. Exempt from the rate limiter (enforced on the forwarded internal hop). See `devplacepy/services/xmlrpc/CLAUDE.md` |
@@ -48,7 +50,7 @@ Prefixes are wired in `main.py`:
 | `/quizzes` | quizzes/ package - **Quizzes**: `index.py` (hub, create, import, detail, export, edit, delete, publish, per-quiz leaderboard, cross-quiz scoreboard), `questions.py` (builder page + question CRUD + reorder), `attempts.py` (start/resume, play, answer, finish, results). The hub `GET /quizzes` is the approved three-column `.feed-layout` reused verbatim from `/feed`: filters + search left (`?filter=all|todo|done|mine|drafts`, `?search=`, `?page=`), the quiz list centre with the **New quiz** and **Create quiz with Devii** actions, and the cross-quiz **scoreboard** rail right (score per user, best attempt per quiz, 15s display cache). Publishing is terminal - every write on a published quiz is a 400 and there is no unpublish route. See `devplacepy/services/quiz/CLAUDE.md` |
 | `/battles` | battles.py - **Opinion Wars**, week-long two-faction battles attached to posts: `GET ""` (listing, quizzes-style filters active/ended/mine + search), `GET /{uid}` (JSON-only full state, runs lazy resolution), `GET /{uid}/events?after=` (durable event trail replay), `POST /{uid}/join` (join or switch faction), `POST /{uid}/fight` (spend 25 Code Farm coins, deal level-weighted damage, 24h cooldown). The battle card renders on the parent post (no battle detail HTML page). See `devplacepy/services/opinionwar/CLAUDE.md` |
 | `/game` | game/ package - the **Code Farm** idle game (`index.py` + `farm.py`): `GET /game` (page), `GET /game/state`, `GET /game/leaderboard?board=`, `POST /game/{plant,harvest,buy-plot,upgrade,fertilize,daily,grant,perk,prestige,legacy,mastery,quests/claim}`, `POST /game/{defense/upgrade,defense/downgrade,infrastructure/buy,cosmetics/buy,cosmetics/equip}`, plus social `GET /game/farm/{username}` and `POST /game/farm/{username}/{water,steal}`. See `devplacepy/services/game/CLAUDE.md` |
-| (none) | push.py - push + PWA: `GET /push.json` (VAPID public key + the providers accepting registrations), `POST /push.json` (register with any active provider; a body without `provider` is a `webpush` body, so existing clients are unchanged; an unknown, disabled or unconfigured provider is a 400), `GET /service-worker.js`, `GET /manifest.json`. Provider protocol and delivery live in `devplacepy/push/` - see `devplacepy/push/CLAUDE.md` |
+| (none) | push.py - push + PWA: `GET /push.json` (VAPID public key + the providers accepting registrations; `apns` includes `environment` when active), `POST /push.json` (register with any active provider; a body without `provider` is a `webpush` body; an APNs body is `token` plus optional `client_id` for device-stable upsert; created/revived rows are probed and the JSON may include `delivered`/`error`; an unknown, disabled or unconfigured provider is a 400), `DELETE /push.json` (unregister exactly the one registration named by `endpoint`/`token`/`client_id`, same identity priority as registration; idempotent, always 200 `{unregistered}`; wired into `PushManager.js`'s logout-link interceptor so a webpush subscription is dropped before the browser navigates to `/auth/logout`), `GET /service-worker.js`, `GET /manifest.json`. Provider protocol and delivery live in `devplacepy/push/` - see `devplacepy/push/CLAUDE.md` |
 | (none) | docs.py (docs/ package) - the documentation site (prose pages + API reference). See `routers/docs/CLAUDE.md` for the deep detail on this tree (`DOCS_PAGES`, audience tiers, prose rendering pipeline) |
 | `/pubsub` | pubsub.py - **database-free** publish/subscribe bus. `WS /pubsub/ws` (subscribe/unsubscribe/publish frames, `foo.*` wildcards), `POST /pubsub/publish` + `GET /pubsub/topics` (admin/internal). Lock-owner-gated WS (`4013` retry) so subscribers converge on one worker; topic authz in `services/pubsub/policy.py` (`user.{uid}.*` private, `public.*` shared, admin/internal anywhere, guests opt-in). In-process `services.pubsub.publish(topic, data)` for backends; frontend `app.pubsub`. See `devplacepy/services/pubsub/CLAUDE.md` |
 | (none) | seo.py - `/robots.txt`, `/sitemap.xml` |
@@ -91,6 +93,10 @@ The `comments` table uses `(target_type, target_uid)` so the same `_comment_sect
 
 Every post card on the feed has an inline comment form (`.feed-comment-form`) beneath the post actions. It posts to `/comments/create` like the detail page comment form. Tests must scope Post button clicks to `#create-post-modal button.btn-primary:has-text('Post')` to avoid matching the inline comment's Post button.
 
+### Comment permalinks
+
+Every comment's action bar (`_comment.html`) includes a "Copy link" button built on the existing `id="comment-{uid}"` anchor (already used by notifications, the profile Activity tab, and `NotificationManager.js`'s scroll-to-highlight) - no new route or URL scheme. It is a plain `data-share="#comment-{{ item.comment['uid'] }}"` button: `DomUtils.initShareButtons` resolves the relative hash against `window.location.href` at click time (so it always copies the exact page the viewer is on, canonical or not), copies it via `navigator.clipboard.writeText`, and flashes "Copied!" on the button (`Toast.flash`) - the same mechanism the gist detail page's Share button already uses.
+
 ### Comment editing
 
 A comment's owner (only the owner, never an admin) sees an inline "Edit" button (`data-action='edit'`) in `_comment.html`. `CommentManager.toggleEditForm` swaps the `.comment-text` for a textarea seeded from its `data-raw` attribute (the raw markdown, since `contentRenderer.applyTo` overwrites `textContent` on first render), posts via `Http.send` to `POST /comments/edit/{comment_uid}`, then re-renders the new body in place with `contentRenderer.applyTo`. The route (`content.edit_comment_record`) is `is_owner`-only, writes `content` + `updated_at`, records the `comment.edit` audit event, and branches on `wants_json`: JSON clients get `CommentEditOut{uid, content, url, updated_at}`, the no-JS form falls back to a redirect to the comment anchor. Edits are NOT soft-delete related (the body is overwritten in place). Devii tool: `edit_comment` (owner-only, no confirm). Scope test Edit clicks to `.comment-action-btn:has-text('Edit')`.
@@ -146,8 +152,9 @@ A dedicated `/gists` page for sharing code snippets. Uses `gists` table (auto-cr
 
 - Source code rendered in `<pre><code class="language-xxx">` block on detail page
 - Syntax highlighting handled by existing `highlight.js` loaded globally in `base.html`
-- Copy button uses `navigator.clipboard.writeText()`
+- Copy button uses `navigator.clipboard.writeText()` via the shared `data-copy` handler (`DomUtils.initClipboardCopy`)
 - Cards in listing show language badge, title, truncated description, author, star count
+- **Raw/rendered toggle for `language == "markdown"`:** unlike `"markdown_rendered"` (always rendered) and every other language (always raw), the plain `"markdown"` gist dual-renders server-side in `gist_detail.html` - the existing raw `<pre>` block plus a `render_content(gist['source_code'], ...)` block, the second one starting `hidden`. A `View rendered`/`View raw` button next to Copy uses the generic `data-view-toggle`/`data-view-toggle-alt` (+ the two `-label`/`-label-alt` pairs) attribute pair (`DomUtils.initViewToggles`, see `static/js/CLAUDE.md`) to swap the `hidden` class between the two blocks and its own label - no fetch, no new endpoint.
 
 ### Sitemap
 
@@ -250,6 +257,14 @@ All SEO features are implemented across the following locations:
 - `devplacepy/seo.py` - JSON-LD schema generators (WebSite, BreadcrumbList, DiscussionForumPosting, ProfilePage, SoftwareApplication), meta description truncation, schema combiner, sitemap XML generator
 - `routers/seo.py` - robots.txt and sitemap.xml routes
 
+### DiscussionForumPosting nested comments
+
+`discussion_forum_posting(post, author, comment_count, star_count, base_url, comments=None)` embeds up to `MAX_SCHEMA_COMMENTS` (20) of the post's comments as nested `comment: [{"@type": "Comment", "text", "author", "datePublished"}, ...]` entities - not just the aggregate `CommentAction` `InteractionCounter`, which stays for the total count. `seo.comment_schema_list(comment_tree, base_url)` flattens the already-loaded comment tree (`content.load_detail`'s `detail["comments"]`, the same nested `{comment, author, children}` shape `_comment.html` renders) depth-first up to the cap - it does not re-query the database. `posts.py::view_post` is the only call site; a future post-like discussion surface (project/gist/news comments) can reuse `comment_schema_list` the same way once/if it gets a `DiscussionForumPosting` schema of its own.
+
+### Topic category pages (`/topics`)
+
+`routers/topics.py` gives the feed's `TOPICS` filter (`constants.py`) real, independently-crawlable pages instead of only a `?topic=` query param (whose canonical collapses back to bare `/feed` - see `base_seo_context`, `canonical = f"{base}{request.url.path}"`, which drops the query string on purpose). `GET /topics` is a hub linking every topic (with a live post count); `GET /topics/{topic}` is a full post listing for that topic, built from the exact same `get_feed_posts`/`enrich_post_cards` pair `feed.py` uses (`enrich_post_cards` was extracted out of `feed_page` specifically so this page is not a second, drifting copy of the attachments/reactions/bookmarks/poll/war enrichment loop). Each topic page gets its own canonical URL, unique title/description, breadcrumbs (Home > Topics > {label}), a `rel=next` link when paginated (`list_page_seo`/`next_page_url`, same mechanism as `/feed`/`/news`), and a real crawlable `_load_more.html` link (not JS-only infinite scroll) for reaching older posts. Both `/topics` and every `/topics/{topic}` are in `sitemap.xml`.
+
 ### SEO template context
 - Every router passes `page_title`, `meta_description`, `meta_robots`, `canonical_url`, `og_title`, `og_description`, `og_image`, `og_type`, `breadcrumbs`, `page_schema` via `base_seo_context()`
 - Auth pages: `noindex,nofollow`
@@ -266,15 +281,22 @@ All SEO features are implemented across the following locations:
 - `profile.html` - username rendered as `<h1 class="profile-name">`
 - `messages.html` - `<h1 class="sr-only">Messages</h1>`
 - `projects.html` - `<h1>Projects</h1>`
-- `post.html` - post title as `<h1>`, "Related Discussions" as `<h3>`
+- `post.html` - post title as `<h1>`; "Gists from {author}", "Projects from {author}", and "Related Discussions" are `.sidebar-heading` labels in the left column (see "Post page layout" below), not `<h3>`
 
 ### Post slugs
 - Slug generated on post creation via `slugify()` and stored in `posts.slug` column
 - Posts can be looked up by slug or UUID
 - Minimum content validation: post body >= 10 chars, comment >= 3 chars
 
-### Related posts
-- `templates/post.html` - "Related Discussions" widget at bottom of post page (queried by matching topic)
+### Post page layout (three columns, mirrors `/feed`)
+
+`post.html` reuses `/feed`'s exact layout building blocks rather than inventing new ones, wrapped in `.post-page-layout` (`static/css/post.css`, `grid-template-columns: var(--sidebar-width) minmax(0, 1fr) 280px`, collapsing to one column at 1024px):
+
+- **Left column** - `<aside class="post-page-sidebar">`, sticky, holding up to three separate `.sidebar-card` blocks (never merged into one card - each is its own bordered panel): "Gists from {author}" and "Projects from {author}" (`content.get_user_sidebar_gists`/`get_user_sidebar_projects`, up to 5 each, most-recently-modified first, private projects filtered via `can_view_project`), and "Related Discussions" (same-topic posts). Every card title is a `.sidebar-heading` div (the same class `feed.html`'s left sidebar uses for "Topics"/"Resources"/"Online now" - `.sidebar-card .sidebar-heading` in `sidebar.css`, so it only styles correctly nested inside a `.sidebar-card`, never bare). Each list item reuses the plain `.related-list`/`.related-link`/`.related-title`/`.related-meta` classes (`base.css`) with `content_preview()` for the ellipsis-truncated description.
+- **Middle column** - `.post-page` (`max-width: 720px`): the post article, comments.
+- **Right column** - `<aside class="feed-right">`, the exact same class `feed.html` uses for its Daily Topic widget (sticky, hidden below 1024px via `feed.css`'s own media query - no post-page-specific override needed). Holds the "Featured" cards (`database.get_featured_topics`, up to 3, cached pool + per-request random sample): each is a `.daily-topic-card` with its own `.daily-topic-label` ("Featured") - matching the single Daily Topic card's internal label, since there is no section-level heading here (a `.sidebar-heading` div placed directly in `.feed-right` would NOT be styled, as noted above - the fix used is a per-card label instead of a bare heading).
+
+All three columns are populated by `routers/posts.py` `post_page_context()`, the single context builder shared by the real `/posts/{slug}` route and `happy404.render()` (see the root `CLAUDE.md`), so a decoy happy-404 post page renders with the identical sidebar/featured layout as a real one.
 
 ### Performance
 - `loading="lazy"` on all avatar images
@@ -301,6 +323,11 @@ All SEO features are implemented across the following locations:
 - `POST /bookmarks/{target_type}/{target_uid}` toggles a `bookmarks` row; `GET /bookmarks/saved` renders the personal list (`saved.html`). Target types: `post`, `gist`, `project`, `news`.
 - `_bookmark_button.html` takes `_type`, `_uid`, `_bookmarked`; `BookmarkManager.js` swaps the label/`bookmarked` class from the JSON `{saved}`. Batch state via `get_user_bookmarks(user_uid, target_type, uids)`.
 
+### Personal notes
+- A `notes` row is a private, per-user text annotation on a target (`post`, `gist`, `project`, `news` - the four detail pages, unlike bookmarks' listing-card coverage, since a note is read/written on the full content view, not skimmed from a card). `POST /notes/{target_type}/{target_uid}` (`NoteForm{content}`, max 4000 chars) creates or replaces the caller's own note on that target (revives a soft-deleted row rather than duplicating it, exactly like bookmarks); `POST /notes/{target_type}/{target_uid}/delete` soft-deletes it; `GET /notes/saved` renders the personal notes list (`notes.html`), mirroring `saved.html` but including each note's body.
+- There is no read/edit access for anyone but the author - the route only ever looks up `user_uid=user["uid"]`, so there is no "someone else's note" to view or moderate. `database/moderation.py` lists `notes` in `UNREPORTABLE_TABLES` ("private to the owner") for that reason.
+- `_note_button.html` takes `_type`, `_uid`, `_note` (the current content or `None`) and renders a button that opens a small inline textarea editor (not a modal - the body is short and the surrounding action bar has no room for a full dialog); `NoteManager.js` (`app.notes`) wires open/cancel/save/delete and swaps the button label/`has-note` class from the JSON `{content}` / `{deleted}` response, extending the shared `OptimisticAction` base like the other engagement controllers. Batch/single state via `get_user_notes(user_uid, target_type, uids)` (`database/engagement.py`), wired into `content.load_detail`/`detail_context` (post/gist/project) and `routers/news.py`'s detail route as `note_content` on the page context and the matching `*DetailOut` schema.
+
 ### Polls
 - A poll rides on a post (one `polls` row keyed by `post_uid`, options in `poll_options`, one-per-user votes in `poll_votes`). Created in `posts.py:create_poll` when `poll_question` plus >= 2 non-empty `poll_options` are submitted (capped at 6). Both `create_post` and `edit_post` accept the poll fields; `edit_post` only attaches a poll when the post has **none** yet (it never replaces an existing poll). The builders live in the create-post modal (`feed.html`) and the edit-post modal (`post.html`, rendered only when the post has no poll) using `data-poll-toggle` / `data-poll-add-option`.
 - `poll_options` accepts either repeated form fields (the web builders, which preserve commas inside an option label) **or** a single newline- or comma-separated string (the API/Devii path). `models.py:normalize_poll_options` splits a lone delimited element - applied as a `mode="before"` validator on `PostForm`/`PostEditForm` - so the documented "one per line or comma separated" agent format actually produces a multi-option poll instead of a single dropped option.

devplacepy/routers/admin/backups.py

@@ -1,5 +1,6 @@
 # retoor <retoor@molodetz.nl>
 
+import asyncio
 import logging
 from pathlib import Path
 from typing import Annotated
@@ -61,11 +62,12 @@ def _targets() -> list[dict]:
         for key, meta in store.BACKUP_TARGETS.items()
     ]
 
-def _dashboard(can_download: bool) -> dict:
+async def _dashboard(can_download: bool) -> dict:
     backups = [_backup_payload(row, can_download) for row in store.list_backups()]
     schedules = store.list_schedules()
+    storage = await asyncio.to_thread(store.compute_storage_stats)
     return {
-        "storage": store.compute_storage_stats(),
+        "storage": storage,
         "backups": backups,
         "schedules": schedules,
         "targets": _targets(),
@@ -77,7 +79,7 @@ def _dashboard(can_download: bool) -> dict:
 @router.get("/backups", response_class=HTMLResponse)
 async def admin_backups(request: Request):
     admin = require_admin(request)
-    data = _dashboard(is_primary_admin(admin))
+    data = await _dashboard(is_primary_admin(admin))
     base = site_url(request)
     seo_ctx = base_seo_context(
         request,
@@ -89,6 +91,7 @@ async def admin_backups(request: Request):
             {"name": "Backups", "url": "/admin/backups"},
         ],
         schemas=[website_schema(base)],
+        robots="noindex,nofollow",
     )
     return respond(
         request,
@@ -106,7 +109,7 @@ async def admin_backups(request: Request):
 @router.get("/backups/data")
 async def admin_backups_data(request: Request):
     admin = require_admin(request)
-    data = _dashboard(is_primary_admin(admin))
+    data = await _dashboard(is_primary_admin(admin))
     return JSONResponse(BackupDashboardOut.model_validate(data).model_dump(mode="json"))
 
 @router.post("/backups/run")

devplacepy/routers/admin/containers.py

@@ -409,7 +409,7 @@ async def container_instance_page(request: Request, uid: str):
             "events": store.list_events(inst["uid"]),
             "schedules": store.list_schedules(inst["uid"]),
             "stats": api.instance_stats(inst["uid"]),
-            "runtime": api.instance_runtime(inst),
+            "runtime": await api.instance_runtime(inst),
             "can_manage": can_manage,
             "admin_section": "containers",
         },

devplacepy/routers/admin/devii_tasks.py

@@ -139,6 +139,7 @@ async def admin_devii_tasks(request: Request, state: str = "active"):
             {"name": "Devii tasks", "url": "/admin/devii-tasks"},
         ],
         schemas=[website_schema(base)],
+        robots="noindex,nofollow",
     )
     return respond(
         request,

devplacepy/routers/admin/gateway_configs.py

@@ -1,21 +1,32 @@
 # retoor <retoor@molodetz.nl>
 
 import logging
+from typing import Optional
 
 from fastapi import APIRouter, Request
-from fastapi.responses import HTMLResponse, JSONResponse
+from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
 from pydantic import ValidationError
 
+from devplacepy.responses import json_error, respond, wants_json
+from devplacepy.schemas import (
+    AdminGatewayModelFormOut,
+    AdminGatewayOut,
+    AdminGatewayProviderFormOut,
+    AdminGatewayQuotaFormOut,
+)
 from devplacepy.seo import base_seo_context, site_url, website_schema
 from devplacepy.services.audit import record as audit
 from devplacepy.services.manager import service_manager
-from devplacepy.services.openai_gateway import quota, routing
+from devplacepy.services.openai_gateway import model_stats_query, quota, routing
 from devplacepy.templating import templates
-from devplacepy.utils import require_admin
+from devplacepy.utils import not_found, require_admin
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
+INDEX_URL = "/admin/gateway"
+TABS = ("models", "providers", "quota", "stats")
+
 
 def _default_provider_summary() -> dict:
     svc = service_manager.get_service("openai")
@@ -32,10 +43,81 @@ def _default_provider_summary() -> dict:
     }
 
 
+def _quota_defaults_summary() -> dict:
+    svc = service_manager.get_service("openai")
+    cfg = svc.get_config() if svc is not None else {}
+    return {
+        "user": cfg.get(quota.FIELD_DEFAULT_USER, 0.0),
+        "admin": cfg.get(quota.FIELD_DEFAULT_ADMIN, 0.0),
+        "guest": cfg.get(quota.FIELD_DEFAULT_GUEST, 0.0),
+        "internal": cfg.get(quota.FIELD_DEFAULT_INTERNAL, 0.0),
+        "key": cfg.get(quota.FIELD_DEFAULT_KEY, 0.0),
+    }
+
+
+def _rule_label(rule: dict) -> str:
+    return quota.scope_label(rule, fallback=rule.get("uid", ""))
+
+
+def _breadcrumbs(*extra: dict) -> list[dict]:
+    trail = [
+        {"name": "Home", "url": "/feed"},
+        {"name": "Admin", "url": "/admin"},
+        {"name": "Gateway", "url": INDEX_URL},
+    ]
+    return trail + list(extra)
+
+
+def _seo(request: Request, title: str, breadcrumbs: list[dict]) -> dict:
+    base = site_url(request)
+    return base_seo_context(
+        request,
+        title=title,
+        description="Manage OpenAI gateway providers, per-model routing, and quota rules.",
+        robots="noindex,nofollow",
+        breadcrumbs=breadcrumbs,
+        schemas=[website_schema(base)],
+    )
+
+
+def _first_error(exc: ValidationError) -> str:
+    return exc.errors()[0].get("msg", "Invalid input")
+
+
 def _validation_error(exc: ValidationError) -> JSONResponse:
-    first = exc.errors()[0]
-    message = first.get("msg", "Invalid input")
-    return JSONResponse({"ok": False, "error": message}, status_code=400)
+    return JSONResponse({"ok": False, "error": _first_error(exc)}, status_code=400)
+
+
+def _bool_str(value) -> str:
+    if isinstance(value, bool):
+        return "1" if value else "0"
+    return "1" if str(value).strip().lower() in ("1", "true", "on", "yes") else "0"
+
+
+def _blank_to_none(value):
+    value = "" if value is None else str(value).strip()
+    return value or None
+
+
+def _time_to_minutes(value: str) -> Optional[int]:
+    value = (value or "").strip()
+    if not value or ":" not in value:
+        return None
+    hours_str, _, minutes_str = value.partition(":")
+    try:
+        return int(hours_str) * 60 + int(minutes_str)
+    except ValueError:
+        return None
+
+
+def _minutes_to_time(value) -> str:
+    if value is None or value == "":
+        return ""
+    try:
+        total = int(value)
+    except (TypeError, ValueError):
+        return ""
+    return f"{(total // 60) % 24:02d}:{total % 60:02d}"
 
 
 async def _payload(request: Request) -> dict:
@@ -50,35 +132,54 @@ async def _payload(request: Request) -> dict:
     return {key: value for key, value in form.items()}
 
 
-@router.get("/gateway", response_class=HTMLResponse)
-async def gateway_config_page(request: Request):
-    admin = require_admin(request)
-    base = site_url(request)
-    seo_ctx = base_seo_context(
-        request,
-        title="Gateway routing - Admin",
-        description="Manage OpenAI gateway providers and per-model routing.",
-        robots="noindex,nofollow",
-        breadcrumbs=[
-            {"name": "Home", "url": "/feed"},
-            {"name": "Admin", "url": "/admin"},
-            {"name": "Gateway", "url": "/admin/gateway"},
-        ],
-        schemas=[website_schema(base)],
-    )
+def _error_response(
+    request: Request, template: str, context: dict, message: str, status_code: int = 400
+):
+    if wants_json(request):
+        return json_error(status_code, message)
     return templates.TemplateResponse(
+        request,
+        template,
+        {**context, "request": request, "error": message},
+        status_code=status_code,
+    )
+
+
+# --- Index page (tabs: models / providers / quota) --------------------------
+
+
+@router.get("/gateway", response_class=HTMLResponse)
+async def gateway_config_page(request: Request, tab: str = "models"):
+    admin = require_admin(request)
+    if tab not in TABS:
+        tab = "models"
+    quota_rules = quota.quota_rule_store.list()
+    for rule in quota_rules:
+        rule["spent_24h_usd"] = round(
+            quota.spent_24h(rule["owner_kind"], rule["owner_id"], rule["app_reference"]), 6
+        )
+    return respond(
         request,
         "admin_gateway.html",
         {
-            **seo_ctx,
-            "request": request,
+            **_seo(request, "Gateway routing - Admin", _breadcrumbs()),
             "user": admin,
             "admin_section": "gateway",
+            "tab": tab,
+            "providers": routing.provider_store.list(),
+            "models": routing.model_store.list(),
+            "quota_rules": quota_rules,
             "default_provider": _default_provider_summary(),
+            "quota_defaults": _quota_defaults_summary(),
+            "stats_ranges": list(model_stats_query.RANGE_SECONDS.keys()),
         },
+        model=AdminGatewayOut,
     )
 
 
+# --- Providers: JSON API (Devii + programmatic clients) ---------------------
+
+
 @router.get("/gateway/providers")
 async def list_providers(request: Request):
     require_admin(request)
@@ -92,13 +193,40 @@ async def list_providers(request: Request):
 
 
 @router.post("/gateway/providers")
-async def save_provider(request: Request):
+async def save_provider_json(request: Request):
     admin = require_admin(request)
     body = await _payload(request)
     try:
         payload = routing.ProviderIn(**body)
     except ValidationError as exc:
         return _validation_error(exc)
+    saved = _save_provider(request, admin, payload)
+    return JSONResponse({"ok": True, "provider": saved})
+
+
+@router.delete("/gateway/providers/{name}")
+async def delete_provider(request: Request, name: str):
+    admin = require_admin(request)
+    if not _remove_provider(request, admin, name):
+        return JSONResponse({"ok": False, "error": "Provider not found"}, status_code=404)
+    return JSONResponse({"ok": True})
+
+
+# --- Providers: backend-rendered pages ---------------------------------------
+
+
+def _provider_form_values(data: Optional[dict] = None) -> dict:
+    data = data or {}
+    return {
+        "name": str(data.get("name", "")),
+        "base_url": str(data.get("base_url", "")),
+        "api_key": str(data.get("api_key", "")),
+        "is_active": _bool_str(data.get("is_active", True)),
+        "client_profile": str(data.get("client_profile", "")),
+    }
+
+
+def _save_provider(request: Request, admin: dict, payload: "routing.ProviderIn") -> dict:
     saved = routing.provider_store.set(payload)
     audit.record(
         request,
@@ -109,25 +237,130 @@ async def save_provider(request: Request):
         target_label=payload.name,
         summary=f"admin {admin['username']} saved gateway provider {payload.name}",
     )
-    return JSONResponse({"ok": True, "provider": saved})
+    return saved
 
 
-@router.delete("/gateway/providers/{name}")
-async def delete_provider(request: Request, name: str):
-    admin = require_admin(request)
+def _remove_provider(request: Request, admin: dict, name: str) -> bool:
     existed = routing.provider_store.remove(name)
-    if not existed:
-        return JSONResponse({"ok": False, "error": "Provider not found"}, status_code=404)
-    audit.record(
+    if existed:
+        audit.record(
+            request,
+            "gateway.provider.delete",
+            user=admin,
+            target_type="gateway_provider",
+            target_uid=name,
+            target_label=name,
+            summary=f"admin {admin['username']} deleted gateway provider {name}",
+        )
+    return existed
+
+
+@router.get("/gateway/providers/new", response_class=HTMLResponse)
+async def new_provider_page(request: Request):
+    admin = require_admin(request)
+    return respond(
         request,
-        "gateway.provider.delete",
-        user=admin,
-        target_type="gateway_provider",
-        target_uid=name,
-        target_label=name,
-        summary=f"admin {admin['username']} deleted gateway provider {name}",
+        "admin_gateway_provider_form.html",
+        {
+            **_seo(request, "Add provider - Gateway - Admin", _breadcrumbs({"name": "Add provider", "url": "/admin/gateway/providers/new"})),
+            "user": admin,
+            "admin_section": "gateway",
+            "is_edit": False,
+            "form": _provider_form_values(),
+            "error": None,
+        },
+        model=AdminGatewayProviderFormOut,
     )
-    return JSONResponse({"ok": True})
+
+
+@router.post("/gateway/providers/new")
+async def create_provider_page(request: Request):
+    admin = require_admin(request)
+    data = dict(await request.form())
+    try:
+        payload = routing.ProviderIn(**data)
+    except ValidationError as exc:
+        return _error_response(
+            request,
+            "admin_gateway_provider_form.html",
+            {
+                **_seo(request, "Add provider - Gateway - Admin", _breadcrumbs({"name": "Add provider", "url": "/admin/gateway/providers/new"})),
+                "user": admin,
+                "admin_section": "gateway",
+                "is_edit": False,
+                "form": _provider_form_values(data),
+            },
+            _first_error(exc),
+        )
+    _save_provider(request, admin, payload)
+    return RedirectResponse(url=f"{INDEX_URL}?tab=providers", status_code=302)
+
+
+@router.get("/gateway/providers/{name}/edit", response_class=HTMLResponse)
+async def edit_provider_page(request: Request, name: str):
+    admin = require_admin(request)
+    provider = routing.provider_store.get(name)
+    if provider is None:
+        raise not_found("Provider not found")
+    return respond(
+        request,
+        "admin_gateway_provider_form.html",
+        {
+            **_seo(
+                request,
+                f"Edit {provider['name']} - Gateway - Admin",
+                _breadcrumbs({"name": f"Edit {provider['name']}", "url": f"/admin/gateway/providers/{provider['name']}/edit"}),
+            ),
+            "user": admin,
+            "admin_section": "gateway",
+            "is_edit": True,
+            "form": _provider_form_values(provider),
+            "error": None,
+        },
+        model=AdminGatewayProviderFormOut,
+    )
+
+
+@router.post("/gateway/providers/{name}/edit")
+async def edit_provider_save(request: Request, name: str):
+    admin = require_admin(request)
+    existing = routing.provider_store.get(name)
+    if existing is None:
+        raise not_found("Provider not found")
+    data = dict(await request.form())
+    data["name"] = existing["name"]
+    try:
+        payload = routing.ProviderIn(**data)
+    except ValidationError as exc:
+        return _error_response(
+            request,
+            "admin_gateway_provider_form.html",
+            {
+                **_seo(
+                    request,
+                    f"Edit {existing['name']} - Gateway - Admin",
+                    _breadcrumbs({"name": f"Edit {existing['name']}", "url": f"/admin/gateway/providers/{existing['name']}/edit"}),
+                ),
+                "user": admin,
+                "admin_section": "gateway",
+                "is_edit": True,
+                "form": _provider_form_values({**data, "name": existing["name"]}),
+            },
+            _first_error(exc),
+        )
+    _save_provider(request, admin, payload)
+    return RedirectResponse(url=f"{INDEX_URL}?tab=providers", status_code=302)
+
+
+@router.post("/gateway/providers/{name}/delete")
+async def delete_provider_page(request: Request, name: str):
+    admin = require_admin(request)
+    if not _remove_provider(request, admin, name):
+        raise not_found("Provider not found")
+    return RedirectResponse(url=f"{INDEX_URL}?tab=providers", status_code=302)
+
+
+# --- Model routes: JSON API (Devii + programmatic clients) ------------------
 
 
 @router.get("/gateway/models")
@@ -142,14 +375,25 @@ async def list_models(request: Request):
     )
 
 
-@router.post("/gateway/models")
-async def save_model(request: Request):
-    admin = require_admin(request)
-    body = await _payload(request)
-    try:
-        payload = routing.ModelRouteIn(**body)
-    except ValidationError as exc:
-        return _validation_error(exc)
+@router.get("/gateway/provider-models")
+async def provider_models(request: Request, provider: str = ""):
+    require_admin(request)
+    models = await routing.fetch_provider_models(provider)
+    if models is None:
+        return json_error(404, "No model list available for this provider")
+    return JSONResponse({"provider": provider, "models": models})
+
+
+def _check_fallback(payload: "routing.ModelRouteIn") -> Optional[str]:
+    if not payload.fallback_model:
+        return None
+    fallback_route = routing.model_store.get(payload.fallback_model)
+    if fallback_route is None or fallback_route.kind != payload.kind:
+        return "Fallback model must be an existing model route of the same kind"
+    return None
+
+
+def _save_model(request: Request, admin: dict, payload: "routing.ModelRouteIn") -> dict:
     saved = routing.model_store.set(payload)
     audit.record(
         request,
@@ -163,41 +407,274 @@ async def save_model(request: Request):
             f"{payload.source_model} -> {payload.target_model}"
         ),
     )
+    return saved
+
+
+def _remove_model(request: Request, admin: dict, source_model: str) -> bool:
+    existed = routing.model_store.remove(source_model)
+    if existed:
+        audit.record(
+            request,
+            "gateway.model.delete",
+            user=admin,
+            target_type="gateway_model",
+            target_uid=source_model,
+            target_label=source_model,
+            summary=f"admin {admin['username']} deleted gateway model route {source_model}",
+        )
+    return existed
+
+
+@router.post("/gateway/models")
+async def save_model_json(request: Request):
+    admin = require_admin(request)
+    body = await _payload(request)
+    try:
+        payload = routing.ModelRouteIn(**body)
+    except ValidationError as exc:
+        return _validation_error(exc)
+    fallback_error = _check_fallback(payload)
+    if fallback_error:
+        return JSONResponse({"ok": False, "error": fallback_error}, status_code=400)
+    saved = _save_model(request, admin, payload)
     return JSONResponse({"ok": True, "model": saved})
 
 
-@router.delete("/gateway/models/{source_model}")
+@router.delete("/gateway/models/{source_model:path}")
 async def delete_model(request: Request, source_model: str):
     admin = require_admin(request)
-    existed = routing.model_store.remove(source_model)
-    if not existed:
+    if not _remove_model(request, admin, source_model):
         return JSONResponse({"ok": False, "error": "Model route not found"}, status_code=404)
-    audit.record(
-        request,
-        "gateway.model.delete",
-        user=admin,
-        target_type="gateway_model",
-        target_uid=source_model,
-        target_label=source_model,
-        summary=f"admin {admin['username']} deleted gateway model route {source_model}",
-    )
     return JSONResponse({"ok": True})
 
 
-def _quota_defaults_summary() -> dict:
-    svc = service_manager.get_service("openai")
-    cfg = svc.get_config() if svc is not None else {}
+# --- Model routes: backend-rendered pages ------------------------------------
+
+MODEL_FORM_DEFAULTS = {
+    "source_model": "",
+    "provider": "",
+    "target_model": "",
+    "kind": "chat",
+    "vision_provider": "",
+    "vision_model": "",
+    "context_window": "0",
+    "price_cache_hit_per_m": "0",
+    "price_cache_miss_per_m": "0",
+    "price_output_per_m": "0",
+    "price_input_per_m": "0",
+    "context_tier_threshold_tokens": "0",
+    "price_cache_hit_per_m_tier2": "",
+    "price_cache_miss_per_m_tier2": "",
+    "price_output_per_m_tier2": "",
+    "price_input_per_m_tier2": "",
+    "off_peak_start": "",
+    "off_peak_end": "",
+    "off_peak_discount_pct": "0",
+    "fallback_model": "",
+    "is_active": "1",
+}
+
+
+def _model_form_values_from_route(route: dict) -> dict:
+    def _num(key):
+        return str(route.get(key, 0))
+
+    def _opt_num(key):
+        value = route.get(key)
+        return "" if value is None else str(value)
+
     return {
-        "user": cfg.get(quota.FIELD_DEFAULT_USER, 0.0),
-        "admin": cfg.get(quota.FIELD_DEFAULT_ADMIN, 0.0),
-        "guest": cfg.get(quota.FIELD_DEFAULT_GUEST, 0.0),
-        "internal": cfg.get(quota.FIELD_DEFAULT_INTERNAL, 0.0),
-        "key": cfg.get(quota.FIELD_DEFAULT_KEY, 0.0),
+        "source_model": route.get("source_model", ""),
+        "provider": route.get("provider", ""),
+        "target_model": route.get("target_model", ""),
+        "kind": route.get("kind", "chat"),
+        "vision_provider": route.get("vision_provider", ""),
+        "vision_model": route.get("vision_model", ""),
+        "context_window": _num("context_window"),
+        "price_cache_hit_per_m": _num("price_cache_hit_per_m"),
+        "price_cache_miss_per_m": _num("price_cache_miss_per_m"),
+        "price_output_per_m": _num("price_output_per_m"),
+        "price_input_per_m": _num("price_input_per_m"),
+        "context_tier_threshold_tokens": _num("context_tier_threshold_tokens"),
+        "price_cache_hit_per_m_tier2": _opt_num("price_cache_hit_per_m_tier2"),
+        "price_cache_miss_per_m_tier2": _opt_num("price_cache_miss_per_m_tier2"),
+        "price_output_per_m_tier2": _opt_num("price_output_per_m_tier2"),
+        "price_input_per_m_tier2": _opt_num("price_input_per_m_tier2"),
+        "off_peak_start": _minutes_to_time(route.get("off_peak_start_minute")),
+        "off_peak_end": _minutes_to_time(route.get("off_peak_end_minute")),
+        "off_peak_discount_pct": _num("off_peak_discount_pct"),
+        "fallback_model": route.get("fallback_model", ""),
+        "is_active": _bool_str(route.get("is_active", True)),
     }
 
 
-def _rule_label(rule: dict) -> str:
-    return quota.scope_label(rule, fallback=rule.get("uid", ""))
+def _model_form_values_from_submission(data: dict) -> dict:
+    values = dict(MODEL_FORM_DEFAULTS)
+    for key in values:
+        if key in data:
+            values[key] = str(data[key])
+    values["is_active"] = _bool_str(data.get("is_active", "1"))
+    return values
+
+
+def _model_payload_kwargs(data: dict) -> dict:
+    def _num(key, default="0"):
+        value = data.get(key, default)
+        return value if str(value).strip() != "" else default
+
+    def _opt_num(key):
+        value = str(data.get(key, "")).strip()
+        return value or None
+
+    return {
+        "source_model": data.get("source_model", ""),
+        "provider": data.get("provider", ""),
+        "target_model": data.get("target_model", ""),
+        "kind": data.get("kind", "chat"),
+        "vision_provider": data.get("vision_provider", ""),
+        "vision_model": data.get("vision_model", ""),
+        "context_window": _num("context_window"),
+        "price_cache_hit_per_m": _num("price_cache_hit_per_m"),
+        "price_cache_miss_per_m": _num("price_cache_miss_per_m"),
+        "price_output_per_m": _num("price_output_per_m"),
+        "price_input_per_m": _num("price_input_per_m"),
+        "context_tier_threshold_tokens": _num("context_tier_threshold_tokens"),
+        "price_cache_hit_per_m_tier2": _opt_num("price_cache_hit_per_m_tier2"),
+        "price_cache_miss_per_m_tier2": _opt_num("price_cache_miss_per_m_tier2"),
+        "price_output_per_m_tier2": _opt_num("price_output_per_m_tier2"),
+        "price_input_per_m_tier2": _opt_num("price_input_per_m_tier2"),
+        "off_peak_start_minute": _time_to_minutes(data.get("off_peak_start", "")),
+        "off_peak_end_minute": _time_to_minutes(data.get("off_peak_end", "")),
+        "off_peak_discount_pct": _num("off_peak_discount_pct"),
+        "fallback_model": data.get("fallback_model", ""),
+        "is_active": data.get("is_active", "1"),
+    }
+
+
+def _fallback_groups(exclude_source: str = "") -> list[dict]:
+    by_kind: dict[str, list[str]] = {}
+    for route in routing.model_store.list():
+        if route["source_model"] == exclude_source:
+            continue
+        by_kind.setdefault(route["kind"], []).append(route["source_model"])
+    return [{"kind": kind, "options": names} for kind, names in sorted(by_kind.items())]
+
+
+def _model_form_context(
+    request: Request,
+    admin: dict,
+    *,
+    is_edit: bool,
+    form: dict,
+    exclude_source: str = "",
+    title: str,
+    crumb_url: str,
+) -> dict:
+    return {
+        **_seo(request, title, _breadcrumbs({"name": title.split(" - ")[0], "url": crumb_url})),
+        "user": admin,
+        "admin_section": "gateway",
+        "is_edit": is_edit,
+        "form": form,
+        "providers": routing.provider_store.list(),
+        "fallback_groups": _fallback_groups(exclude_source),
+    }
+
+
+@router.get("/gateway/models/new", response_class=HTMLResponse)
+async def new_model_page(request: Request):
+    admin = require_admin(request)
+    context = _model_form_context(
+        request,
+        admin,
+        is_edit=False,
+        form=_model_form_values_from_submission({}),
+        title="Add model route - Gateway - Admin",
+        crumb_url="/admin/gateway/models/new",
+    )
+    return respond(request, "admin_gateway_model_form.html", {**context, "error": None}, model=AdminGatewayModelFormOut)
+
+
+@router.post("/gateway/models/new")
+async def create_model_page(request: Request):
+    admin = require_admin(request)
+    data = dict(await request.form())
+    context = _model_form_context(
+        request,
+        admin,
+        is_edit=False,
+        form=_model_form_values_from_submission(data),
+        exclude_source=str(data.get("source_model", "")),
+        title="Add model route - Gateway - Admin",
+        crumb_url="/admin/gateway/models/new",
+    )
+    try:
+        payload = routing.ModelRouteIn(**_model_payload_kwargs(data))
+    except ValidationError as exc:
+        return _error_response(request, "admin_gateway_model_form.html", context, _first_error(exc))
+    fallback_error = _check_fallback(payload)
+    if fallback_error:
+        return _error_response(request, "admin_gateway_model_form.html", context, fallback_error)
+    _save_model(request, admin, payload)
+    return RedirectResponse(url=f"{INDEX_URL}?tab=models", status_code=302)
+
+
+@router.get("/gateway/models/{source_model:path}/edit", response_class=HTMLResponse)
+async def edit_model_page(request: Request, source_model: str):
+    admin = require_admin(request)
+    route = routing.model_store.get(source_model)
+    if route is None:
+        raise not_found("Model route not found")
+    route_dict = route.__dict__.copy()
+    context = _model_form_context(
+        request,
+        admin,
+        is_edit=True,
+        form=_model_form_values_from_route(route_dict),
+        exclude_source=source_model,
+        title=f"Edit {source_model} - Gateway - Admin",
+        crumb_url=f"/admin/gateway/models/{source_model}/edit",
+    )
+    return respond(request, "admin_gateway_model_form.html", {**context, "error": None}, model=AdminGatewayModelFormOut)
+
+
+@router.post("/gateway/models/{source_model:path}/edit")
+async def edit_model_save(request: Request, source_model: str):
+    admin = require_admin(request)
+    existing = routing.model_store.get(source_model)
+    if existing is None:
+        raise not_found("Model route not found")
+    data = dict(await request.form())
+    data["source_model"] = source_model
+    context = _model_form_context(
+        request,
+        admin,
+        is_edit=True,
+        form=_model_form_values_from_submission(data),
+        exclude_source=source_model,
+        title=f"Edit {source_model} - Gateway - Admin",
+        crumb_url=f"/admin/gateway/models/{source_model}/edit",
+    )
+    try:
+        payload = routing.ModelRouteIn(**_model_payload_kwargs(data))
+    except ValidationError as exc:
+        return _error_response(request, "admin_gateway_model_form.html", context, _first_error(exc))
+    fallback_error = _check_fallback(payload)
+    if fallback_error:
+        return _error_response(request, "admin_gateway_model_form.html", context, fallback_error)
+    _save_model(request, admin, payload)
+    return RedirectResponse(url=f"{INDEX_URL}?tab=models", status_code=302)
+
+
+@router.post("/gateway/models/{source_model:path}/delete")
+async def delete_model_page(request: Request, source_model: str):
+    admin = require_admin(request)
+    if not _remove_model(request, admin, source_model):
+        raise not_found("Model route not found")
+    return RedirectResponse(url=f"{INDEX_URL}?tab=models", status_code=302)
+
+
+# --- Quota rules: JSON API (Devii + programmatic clients) -------------------
 
 
 @router.get("/gateway/quota-rules")
@@ -217,15 +694,9 @@ async def list_quota_rules(request: Request):
     )
 
 
-@router.post("/gateway/quota-rules")
-async def save_quota_rule(request: Request):
-    admin = require_admin(request)
-    body = await _payload(request)
-    uid = str(body.pop("uid", "") or "").strip() or None
-    try:
-        payload = quota.QuotaRuleIn(**body)
-    except ValidationError as exc:
-        return _validation_error(exc)
+def _save_quota_rule(
+    request: Request, admin: dict, payload: "quota.QuotaRuleIn", uid: Optional[str] = None
+) -> dict:
     saved = quota.quota_rule_store.set(payload, uid=uid, created_by=admin["uid"])
     audit.record(
         request,
@@ -243,6 +714,19 @@ async def save_quota_rule(request: Request):
             "is_active": saved["is_active"],
         },
     )
+    return saved
+
+
+@router.post("/gateway/quota-rules")
+async def save_quota_rule_json(request: Request):
+    admin = require_admin(request)
+    body = await _payload(request)
+    uid = str(body.pop("uid", "") or "").strip() or None
+    try:
+        payload = quota.QuotaRuleIn(**body)
+    except ValidationError as exc:
+        return _validation_error(exc)
+    saved = _save_quota_rule(request, admin, payload, uid=uid)
     return JSONResponse({"ok": True, "rule": saved})
 
 
@@ -292,3 +776,220 @@ async def delete_quota_rule(request: Request, uid: str):
         summary=f"admin {admin['username']} deleted gateway quota rule ({label})",
     )
     return JSONResponse({"ok": True})
+
+
+# --- Quota rules: backend-rendered pages -------------------------------------
+
+QUOTA_FORM_DEFAULTS = {
+    "owner_kind": "",
+    "owner_id": "",
+    "app_reference": "",
+    "limit_usd": "0",
+    "is_active": "1",
+    "label": "",
+}
+
+
+def _quota_form_values(data: Optional[dict] = None) -> dict:
+    if not data:
+        return dict(QUOTA_FORM_DEFAULTS)
+    values = dict(QUOTA_FORM_DEFAULTS)
+    for key in values:
+        if key in data and data[key] is not None:
+            values[key] = str(data[key])
+    values["is_active"] = _bool_str(data.get("is_active", "1"))
+    return values
+
+
+@router.get("/gateway/quota-rules/new", response_class=HTMLResponse)
+async def new_quota_rule_page(request: Request):
+    admin = require_admin(request)
+    return respond(
+        request,
+        "admin_gateway_quota_form.html",
+        {
+            **_seo(request, "Add quota rule - Gateway - Admin", _breadcrumbs({"name": "Add quota rule", "url": "/admin/gateway/quota-rules/new"})),
+            "user": admin,
+            "admin_section": "gateway",
+            "is_edit": False,
+            "form": _quota_form_values(),
+            "error": None,
+        },
+        model=AdminGatewayQuotaFormOut,
+    )
+
+
+@router.post("/gateway/quota-rules/new")
+async def create_quota_rule_page(request: Request):
+    admin = require_admin(request)
+    data = dict(await request.form())
+    try:
+        payload = quota.QuotaRuleIn(
+            owner_kind=_blank_to_none(data.get("owner_kind")),
+            owner_id=_blank_to_none(data.get("owner_id")),
+            app_reference=_blank_to_none(data.get("app_reference")),
+            limit_usd=data.get("limit_usd", "0") or "0",
+            is_active=data.get("is_active", "1"),
+            label=data.get("label", ""),
+        )
+    except ValidationError as exc:
+        return _error_response(
+            request,
+            "admin_gateway_quota_form.html",
+            {
+                **_seo(request, "Add quota rule - Gateway - Admin", _breadcrumbs({"name": "Add quota rule", "url": "/admin/gateway/quota-rules/new"})),
+                "user": admin,
+                "admin_section": "gateway",
+                "is_edit": False,
+                "form": _quota_form_values(data),
+            },
+            _first_error(exc),
+        )
+    _save_quota_rule(request, admin, payload)
+    return RedirectResponse(url=f"{INDEX_URL}?tab=quota", status_code=302)
+
+
+@router.get("/gateway/quota-rules/{uid}/edit", response_class=HTMLResponse)
+async def edit_quota_rule_page(request: Request, uid: str):
+    admin = require_admin(request)
+    rule = quota.quota_rule_store.get(uid)
+    if rule is None:
+        raise not_found("Quota rule not found")
+    rule_dict = rule.as_dict()
+    return respond(
+        request,
+        "admin_gateway_quota_form.html",
+        {
+            **_seo(
+                request,
+                "Edit quota rule - Gateway - Admin",
+                _breadcrumbs({"name": "Edit quota rule", "url": f"/admin/gateway/quota-rules/{uid}/edit"}),
+            ),
+            "user": admin,
+            "admin_section": "gateway",
+            "is_edit": True,
+            "uid": uid,
+            "form": _quota_form_values(rule_dict),
+            "error": None,
+        },
+        model=AdminGatewayQuotaFormOut,
+    )
+
+
+@router.post("/gateway/quota-rules/{uid}/edit")
+async def edit_quota_rule_save(request: Request, uid: str):
+    admin = require_admin(request)
+    existing = quota.quota_rule_store.get(uid)
+    if existing is None:
+        raise not_found("Quota rule not found")
+    data = dict(await request.form())
+    try:
+        payload = quota.QuotaRuleIn(
+            owner_kind=_blank_to_none(data.get("owner_kind")),
+            owner_id=_blank_to_none(data.get("owner_id")),
+            app_reference=_blank_to_none(data.get("app_reference")),
+            limit_usd=data.get("limit_usd", "0") or "0",
+            is_active=data.get("is_active", "1"),
+            label=data.get("label", ""),
+        )
+    except ValidationError as exc:
+        return _error_response(
+            request,
+            "admin_gateway_quota_form.html",
+            {
+                **_seo(
+                    request,
+                    "Edit quota rule - Gateway - Admin",
+                    _breadcrumbs({"name": "Edit quota rule", "url": f"/admin/gateway/quota-rules/{uid}/edit"}),
+                ),
+                "user": admin,
+                "admin_section": "gateway",
+                "is_edit": True,
+                "uid": uid,
+                "form": _quota_form_values(data),
+            },
+            _first_error(exc),
+        )
+    _save_quota_rule(request, admin, payload, uid=uid)
+    return RedirectResponse(url=f"{INDEX_URL}?tab=quota", status_code=302)
+
+
+@router.post("/gateway/quota-rules/{uid}/delete")
+async def delete_quota_rule_page(request: Request, uid: str):
+    admin = require_admin(request)
+    existing = quota.quota_rule_store.get(uid)
+    label = _rule_label(existing.as_dict()) if existing else uid
+    if not quota.quota_rule_store.remove(uid):
+        raise not_found("Quota rule not found")
+    audit.record(
+        request,
+        "gateway.quota_rule.delete",
+        user=admin,
+        target_type="gateway_quota_rule",
+        target_uid=uid,
+        target_label=label,
+        summary=f"admin {admin['username']} deleted gateway quota rule ({label})",
+    )
+    return RedirectResponse(url=f"{INDEX_URL}?tab=quota", status_code=302)
+
+
+@router.post("/gateway/quota-rules/{uid}/reset")
+async def reset_quota_rule_page(request: Request, uid: str):
+    admin = require_admin(request)
+    existing = quota.quota_rule_store.get(uid)
+    if existing is None:
+        raise not_found("Quota rule not found")
+    scope = quota.reset(
+        quota.QuotaResetIn(
+            owner_kind=existing.owner_kind,
+            owner_id=existing.owner_id,
+            app_reference=existing.app_reference,
+        ),
+        created_by=admin["uid"],
+    )
+    label = quota.scope_label(scope, fallback="every caller")
+    audit.record(
+        request,
+        "gateway.quota.reset",
+        user=admin,
+        target_type="gateway_quota",
+        target_uid=scope["uid"],
+        target_label=label,
+        summary=f"admin {admin['username']} reset the gateway 24h spend for {label}",
+        metadata={
+            "owner_kind": scope["owner_kind"],
+            "owner_id": scope["owner_id"],
+            "app_reference": scope["app_reference"],
+            "reset_at": scope["reset_at"],
+        },
+    )
+    return RedirectResponse(url=f"{INDEX_URL}?tab=quota", status_code=302)
+
+
+# --- Stats: JSON API for the Stats tab's charts ------------------------------
+
+
+@router.get("/gateway/stats/data")
+async def gateway_stats_data(request: Request, range: str = "24h"):
+    require_admin(request)
+    try:
+        return JSONResponse(model_stats_query.compute_summary(range))
+    except ValueError:
+        return json_error(400, f"Unknown range: {range!r}")
+
+
+@router.get("/gateway/stats/models")
+async def gateway_stats_models(request: Request):
+    require_admin(request)
+    return JSONResponse({"models": model_stats_query.list_known_models()})
+
+
+@router.get("/gateway/stats/model/{provider}/{model:path}")
+async def gateway_stats_model_detail(
+    request: Request, provider: str, model: str, range: str = "24h"
+):
+    require_admin(request)
+    try:
+        return JSONResponse(model_stats_query.compute_model_detail(provider, model, range))
+    except ValueError:
+        return json_error(400, f"Unknown range: {range!r}")

devplacepy/routers/admin/issues.py

@@ -5,11 +5,12 @@ import logging
 from fastapi import APIRouter, Request
 from fastapi.responses import HTMLResponse
 
+from devplacepy.responses import respond
+from devplacepy.schemas import AdminIssuesPlanningOut
 from devplacepy.seo import base_seo_context
 from devplacepy.services.gitea import runtime
 from devplacepy.services.gitea.config import gitea_config
 from devplacepy.services.gitea.planning import collect_open_issues
-from devplacepy.templating import templates
 from devplacepy.utils import require_admin
 
 logger = logging.getLogger(__name__)
@@ -65,4 +66,4 @@ async def admin_issues_planning(request: Request):
         "tickets": tickets,
         "tickets_error": tickets_error,
     }
-    return templates.TemplateResponse(request, "admin_issues_planning.html", context)
+    return respond(request, "admin_issues_planning.html", context, model=AdminIssuesPlanningOut)

devplacepy/routers/admin/services.py

@@ -9,6 +9,7 @@ from devplacepy.utils import require_admin, not_found
 from devplacepy.services.manager import service_manager
 from devplacepy.seo import base_seo_context, site_url, website_schema
 from devplacepy.services.audit import record as audit
+from devplacepy.services.devii import tool_prefs
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
@@ -62,7 +63,7 @@ async def service_detail(request: Request, name: str):
         request,
         title=f"{info['title']} - Services",
         description=info["description"] or f"Configure the {info['title']} service.",
-        robots="noindex",
+        robots="noindex,nofollow",
         breadcrumbs=[
             {"name": "Home", "url": "/feed"},
             {"name": "Admin", "url": "/admin"},
@@ -71,6 +72,7 @@ async def service_detail(request: Request, name: str):
         ],
         schemas=[website_schema(base)],
     )
+    tool_groups = tool_prefs.group_overview() if name == "devii" else None
     return templates.TemplateResponse(
         request,
         "service_detail.html",
@@ -79,6 +81,7 @@ async def service_detail(request: Request, name: str):
             "request": request,
             "user": admin,
             "service": info,
+            "tool_groups": tool_groups,
             "admin_section": "services",
         },
     )
@@ -184,3 +187,34 @@ async def service_config(request: Request, name: str):
     if not result["ok"]:
         return JSONResponse(result, status_code=400)
     return JSONResponse(result)
+
+
+@router.post("/devii/tools")
+async def devii_tools_config(request: Request):
+    admin = require_admin(request)
+    before = tool_prefs.disabled_tool_names()
+    form = await request.form()
+    enabled = set(form.getlist("enabled"))
+    known = set(tool_prefs.GROUPS_BY_TOOL_NAME)
+    after = known - enabled
+    tool_prefs.set_disabled_tool_names(after)
+    newly_disabled = sorted(after - before)
+    newly_enabled = sorted(before - after)
+    if newly_disabled or newly_enabled:
+        audit.record(
+            request,
+            "service.devii_tools.update",
+            user=admin,
+            target_type="service",
+            target_uid="devii",
+            target_label="devii",
+            old_value=f"{len(before)} disabled",
+            new_value=f"{len(after)} disabled",
+            summary=(
+                f"admin {admin['username']} updated the Devii tool catalog "
+                f"({len(newly_disabled)} disabled, {len(newly_enabled)} re-enabled)"
+            ),
+            metadata={"disabled": newly_disabled, "enabled": newly_enabled},
+            links=[audit.service_link("devii")],
+        )
+    return JSONResponse({"ok": True, "disabled_count": len(after), "total_count": len(known)})

devplacepy/routers/admin/trash.py

@@ -89,6 +89,7 @@ async def admin_trash(request: Request, table: str = "posts", page: int = 1):
             {"name": "Trash", "url": "/admin/trash"},
         ],
         schemas=[website_schema(base)],
+        robots="noindex,nofollow",
     )
     return respond(
         request,

devplacepy/routers/admin/workspaces.py

@@ -1,11 +1,12 @@
 # retoor <retoor@molodetz.nl>
 
+import asyncio
 from typing import Annotated
 
 from fastapi import APIRouter, Depends, Request
 from fastapi.responses import JSONResponse
 
-from devplacepy.database import db, get_table, get_users_by_uids
+from devplacepy.database import get_projects_by_uids, get_table, get_users_by_uids
 from devplacepy.dependencies import json_or_form
 from devplacepy.models import (
     EditorPrefsForm,
@@ -31,19 +32,14 @@ from devplacepy.utils import create_notification, generate_uid, not_found, requi
 router = APIRouter()
 
 
-def _decorate(rows: list[dict]) -> list[dict]:
+async def _decorate(rows: list[dict]) -> list[dict]:
     owner_uids = {row.get("workspace_owner_uid") for row in rows if row.get("workspace_owner_uid")}
     owners = get_users_by_uids(list(owner_uids)) if owner_uids else {}
-    projects = {}
-    if "projects" in db.tables:
-        project_uids = {row.get("project_uid") for row in rows if row.get("project_uid")}
-        for uid in project_uids:
-            found = get_table("projects").find_one(uid=uid)
-            if found:
-                projects[uid] = found
+    project_uids = {row.get("project_uid") for row in rows if row.get("project_uid")}
+    projects = get_projects_by_uids(list(project_uids)) if project_uids else {}
+    views = await asyncio.gather(*(provision.view(row) for row in rows))
     decorated = []
-    for row in rows:
-        view = provision.view(row)
+    for row, view in zip(rows, views):
         owner = owners.get(row.get("workspace_owner_uid", "")) or {}
         project = projects.get(row.get("project_uid", "")) or {}
         view["owner_username"] = owner.get("username", "")
@@ -84,7 +80,7 @@ async def admin_workspaces(request: Request):
     if not isinstance(admin, dict):
         return admin
     context = {
-        "workspaces": _decorate(_all_workspaces()),
+        "workspaces": await _decorate(_all_workspaces()),
         "flags": flags.list_flags(),
         "admin_section": "workspaces",
         "user": admin,
@@ -109,7 +105,7 @@ async def admin_workspaces_data(request: Request):
     if not isinstance(admin, dict):
         return admin
     return JSONResponse(
-        {"workspaces": _decorate(_all_workspaces()), "flags": flags.list_flags()}
+        {"workspaces": await _decorate(_all_workspaces()), "flags": flags.list_flags()}
     )
 
 

devplacepy/routers/auth/token.py

@@ -22,12 +22,6 @@ async def token(
     request: Request,
     data: Annotated[LoginForm, Depends(json_or_form(LoginForm))],
 ):
-    """Issue a DevPlace access token.
-
-    Accepts ``email`` + ``password`` (JSON or form-encoded).  Returns a JSON
-    object with ``access_token``, ``token_type``, and ``expires_in`` on success,
-    or a ``401`` error on bad credentials.
-    """
     identifier = data.email.strip().lower()
     password = data.password
 

devplacepy/routers/battles.py

@@ -11,7 +11,7 @@ from devplacepy.dependencies import json_or_form
 from devplacepy.models import WarJoinForm
 from devplacepy.responses import action_result, json_error, respond, wants_json
 from devplacepy.schemas import BattlesOut, WarEventsOut, WarOut
-from devplacepy.seo import base_seo_context
+from devplacepy.seo import base_seo_context, site_url, website_schema
 from devplacepy.services.opinionwar import WarError, rules, store
 from devplacepy.utils import get_current_user, not_found, require_user
 
@@ -43,6 +43,7 @@ async def battles_page(
         search=search,
         page=max(1, page),
     )
+    base = site_url(request)
     seo_ctx = base_seo_context(
         request,
         title="Opinion Wars",
@@ -54,6 +55,7 @@ async def battles_page(
             {"name": "Home", "url": "/feed"},
             {"name": "Battles", "url": "/battles"},
         ],
+        schemas=[website_schema(base)],
     )
     return respond(
         request,

devplacepy/routers/dbapi/query.py

@@ -1,5 +1,6 @@
 # retoor <retoor@molodetz.nl>
 
+import asyncio
 import json
 import logging
 
@@ -141,7 +142,8 @@ async def dbapi_query_result(request: Request, uid: str):
     if not path.is_relative_to(DBAPI_DIR.resolve()) or not path.is_file():
         return error(404, "Result not available")
     queue.touch_job(uid, get_int_setting("dbquery_retention_seconds", 604800))
-    return JSONResponse(json.loads(path.read_text(encoding="utf-8")))
+    text = await asyncio.to_thread(path.read_text, encoding="utf-8")
+    return JSONResponse(json.loads(text))
 
 
 @router.websocket("/query/{uid}/ws")

devplacepy/routers/devii.py

@@ -137,8 +137,6 @@ def _validate_target(value: str) -> str:
 
 @router.get("/adopt")
 async def devii_adopt(request: Request):
-    # The terminal's agent logged in; adopt the real session it minted into the browser so both
-    # share one session, then return to where the user was.
     target = _validate_target(request.query_params.get("next", "/"))
     response = RedirectResponse(target, status_code=303)
     svc = _service()
@@ -288,6 +286,7 @@ async def devii_ws(websocket: WebSocket):
                         }
                     )
                     continue
+                svc.maybe_warn_quota_threshold(owner_kind, owner_id, owner_is_admin)
                 session.spawn_turn(text)
             elif kind == "reset":
                 await session.reset()

devplacepy/routers/docs/CLAUDE.md

@@ -66,7 +66,7 @@ The `endpoint()` factory derives `min_role` from `auth` (`public` -> Public, `us
 
 ## Admin-only pages
 
-A group with `"admin": True` (currently `services`, `admin`) is hidden from the sidebar (`docs.py` filters `visible_pages`) and returns 404 for non-admins (`page.get("admin") and not is_admin -> not_found`, same message as unknown slugs). `index.html` wraps the operator links in `{% if user and user.get('role') == 'Admin' %}`.
+A group with `"admin": True` (currently `services`, `admin`, `containers`) is hidden from the sidebar (`docs.py` filters `visible_pages`) and returns 404 for non-admins (`page.get("admin") and not is_admin -> not_found`, same message as unknown slugs). `index.html` wraps the operator links in `{% if user and user.get('role') == 'Admin' %}`.
 
 ## Dynamic Background Services page
 

devplacepy/routers/docs/pages.py

@@ -44,7 +44,6 @@ AUDIENCES = [
 ]
 
 DOCS_PAGES = [
-    # General - how to use the site and Devii (everyone)
     {"slug": "index", "title": "Overview", "kind": "prose", "section": SECTION_GENERAL},
     {
         "slug": "getting-started",
@@ -160,7 +159,6 @@ DOCS_PAGES = [
         "kind": "prose",
         "section": SECTION_GENERAL,
     },
-    # Legal - the policies the platform is operated under (everyone)
     {
         "slug": "terms",
         "title": "Terms of Service",
@@ -204,7 +202,6 @@ DOCS_PAGES = [
         "section": SECTION_LEGAL,
         "admin": True,
     },
-    # Tools - public developer tools (everyone)
     {
         "slug": "tools-seo",
         "title": "SEO Diagnostics",
@@ -229,7 +226,6 @@ DOCS_PAGES = [
         "kind": "prose",
         "section": SECTION_TOOLS,
     },
-    # Claude Code - the native subagent, command, and workflow setup under .claude/
     {
         "slug": "claude",
         "title": "Claude Code setup",
@@ -260,7 +256,6 @@ DOCS_PAGES = [
         "kind": "prose",
         "section": SECTION_CLAUDE,
     },
-    # Components - custom HTML web components with live examples (everyone)
     {
         "slug": "components",
         "title": "Components overview",
@@ -339,7 +334,6 @@ DOCS_PAGES = [
         "kind": "prose",
         "section": SECTION_COMPONENTS,
     },
-    # Styles - the design system: colors, layout, responsiveness, and hard structural rules (everyone)
     {
         "slug": "styles",
         "title": "Design system overview",
@@ -370,7 +364,6 @@ DOCS_PAGES = [
         "kind": "prose",
         "section": SECTION_STYLES,
     },
-    # API - developer reference (everyone); admin-only groups are routed to Administration below
     {
         "slug": "authentication",
         "title": "Authentication",
@@ -383,7 +376,6 @@ DOCS_PAGES = [
         "kind": "prose",
         "section": SECTION_API,
     },
-    # devRant API - legacy-compatible protocol, spread over focused pages
     {
         "slug": "devrant",
         "title": "Overview",
@@ -430,7 +422,6 @@ DOCS_PAGES = [
         {**page, "section": (SECTION_ADMIN if page.get("admin") else SECTION_API)}
         for page in api_doc_pages()
     ],
-    # Administration - operational guides (admins only)
     {
         "slug": "devii-admin",
         "title": "Devii for admins",
@@ -473,7 +464,6 @@ DOCS_PAGES = [
         "admin": True,
         "section": SECTION_ADMIN,
     },
-    # Devii internals - technical reference for the Devii assistant (admins only)
     {
         "slug": "audit-log",
         "title": "Audit Log",
@@ -523,7 +513,6 @@ DOCS_PAGES = [
         "admin": True,
         "section": SECTION_DEVII,
     },
-    # Bots internals - deep technical reference for the autonomous bot fleet (admins only)
     {
         "slug": "bots-internals",
         "title": "Bots internals",
@@ -573,7 +562,6 @@ DOCS_PAGES = [
         "admin": True,
         "section": SECTION_BOTS,
     },
-    # Services - the background service framework and every service in detail (admins only)
     {
         "slug": "services-overview",
         "title": "Services overview",
@@ -651,7 +639,6 @@ DOCS_PAGES = [
         "admin": True,
         "section": SECTION_SERVICES,
     },
-    # Architecture - platform design, structure, and development process (admins only)
     {
         "slug": "architecture",
         "title": "Architecture overview",
@@ -701,7 +688,6 @@ DOCS_PAGES = [
         "admin": True,
         "section": SECTION_ARCH,
     },
-    # Testing - test framework, load testing, and make targets (admins only)
     {
         "slug": "testing",
         "title": "Testing overview",
@@ -737,7 +723,6 @@ DOCS_PAGES = [
         "admin": True,
         "section": SECTION_TESTING,
     },
-    # Production - deployment and operations reference (admins only)
     {
         "slug": "production",
         "title": "Production overview",

devplacepy/routers/feed.py

@@ -81,21 +81,7 @@ def get_feed_posts(
     return result, next_cursor
 
 
-@router.get("", response_class=HTMLResponse)
-async def feed_page(
-    request: Request,
-    tab: str = "all",
-    topic: str = None,
-    search: str = "",
-    before: str = None,
-):
-    user = get_current_user(request)
-    posts, next_cursor = get_feed_posts(user, tab, topic, search, before)
-    stats = get_site_stats()
-    top_authors = get_top_authors(5)
-    daily_topic = get_daily_topic()
-    online_users = presence.online_users()
-
+def enrich_post_cards(posts, user):
     post_uids_list = [item["post"]["uid"] for item in posts]
     attachments_map = get_attachments_batch("post", post_uids_list)
     recent_comments = get_recent_comments_by_post_uids(post_uids_list, 3, user)
@@ -113,6 +99,25 @@ async def feed_page(
         item["bookmarked"] = uid in bookmark_set
         item["poll"] = polls_map.get(uid)
         item["war"] = wars_map.get(uid)
+    return posts
+
+
+@router.get("", response_class=HTMLResponse)
+async def feed_page(
+    request: Request,
+    tab: str = "all",
+    topic: str = None,
+    search: str = "",
+    before: str = None,
+):
+    user = get_current_user(request)
+    posts, next_cursor = get_feed_posts(user, tab, topic, search, before)
+    stats = get_site_stats()
+    top_authors = get_top_authors(5)
+    daily_topic = get_daily_topic()
+    online_users = presence.online_users()
+
+    posts = enrich_post_cards(posts, user)
 
     seo_ctx = list_page_seo(
         request,

devplacepy/routers/game/farm.py

@@ -9,6 +9,7 @@ from devplacepy.models import GameSlotForm
 from devplacepy.responses import respond, wants_json
 from devplacepy.schemas import GameFarmViewOut
 from devplacepy.services.game import GameError, store
+from devplacepy.services.audit import record as audit
 from devplacepy.utils import (
     create_notification,
     get_current_user,
@@ -58,10 +59,24 @@ async def water_farm(
     if not owner:
         raise HTTPException(status_code=404, detail="Farm not found")
     try:
-        store.water(viewer, owner, data.slot)
+        result = store.water(viewer, owner, data.slot)
     except GameError as exc:
         return action_error(request, str(exc), f"/game/farm/{username}")
     track_action(viewer["uid"], "water")
+    audit.record(
+        request,
+        "game.water",
+        user=viewer,
+        target_type="user",
+        target_uid=owner["uid"],
+        target_label=owner["username"],
+        metadata=result,
+        summary=(
+            f"{viewer['username']} watered {owner['username']}'s build on plot "
+            f"{result['slot']} and earned {result['reward_coins']} coins"
+        ),
+        links=[audit.target("user", owner["uid"], owner["username"])],
+    )
     await notify_farm(owner["username"])
     if wants_json(request):
         payload = store.serialize_farm(
@@ -87,6 +102,31 @@ async def steal_farm(
     track_action(owner["uid"], "got_stolen_from")
     if result.get("underdog_triggered"):
         track_action(viewer["uid"], "underdog_raid")
+    audit.record(
+        request,
+        "game.steal",
+        user=viewer,
+        target_type="user",
+        target_uid=owner["uid"],
+        target_label=owner["username"],
+        metadata={
+            "thief_uid": viewer["uid"],
+            "thief_username": viewer["username"],
+            "owner_uid": owner["uid"],
+            "owner_username": owner["username"],
+            "slot": result["slot"],
+            "crop": result["crop"],
+            "coins": result["coins"],
+            "share": result["share"],
+            "underdog_triggered": result.get("underdog_triggered", False),
+        },
+        summary=(
+            f"{viewer['username']} raided {owner['username']}'s Code Farm and took "
+            f"{result['coins']} coins ({round(result['share'] * 100)}%) from their "
+            f"{result['crop_name']} build"
+        ),
+        links=[audit.target("user", owner["uid"], owner["username"])],
+    )
     create_notification(
         owner["uid"],
         "harvest_stolen",

devplacepy/routers/game/index.py

@@ -80,8 +80,21 @@ async def _respond_action(request: Request, user: dict, fn, on_success=None):
 @router.post("/plant")
 async def game_plant(request: Request, data: Annotated[GamePlantForm, Form()]):
     user = require_user(request)
+
+    def recorded(result):
+        audit.record(
+            request,
+            "game.plant",
+            user=user,
+            metadata=result,
+            summary=(
+                f"{user['username']} planted {result['crop']} on plot "
+                f"{result['slot']} for {result['spent']} coins"
+            ),
+        )
+
     return await _respond_action(
-        request, user, lambda: store.plant(user, data.slot, data.crop)
+        request, user, lambda: store.plant(user, data.slot, data.crop), recorded
     )
 
 
@@ -92,6 +105,16 @@ async def game_harvest(request: Request, data: Annotated[GameSlotForm, Form()]):
     def reward(result):
         track_action(user["uid"], "harvest")
         award_rewards(user["uid"], economy.site_xp_for(result.get("xp", 0)))
+        audit.record(
+            request,
+            "game.harvest",
+            user=user,
+            metadata=result,
+            summary=(
+                f"{user['username']} harvested {result['crop']} on plot "
+                f"{result['slot']} for {result['coins']} coins"
+            ),
+        )
 
     return await _respond_action(
         request, user, lambda: store.harvest(user, data.slot), reward
@@ -101,25 +124,79 @@ async def game_harvest(request: Request, data: Annotated[GameSlotForm, Form()]):
 @router.post("/buy-plot")
 async def game_buy_plot(request: Request):
     user = require_user(request)
-    return await _respond_action(request, user, lambda: store.buy_plot(user))
+
+    def recorded(result):
+        audit.record(
+            request,
+            "game.plot.buy",
+            user=user,
+            metadata=result,
+            summary=(
+                f"{user['username']} bought Code Farm plot "
+                f"{result['plot_count']} for {result['spent']} coins"
+            ),
+        )
+
+    return await _respond_action(request, user, lambda: store.buy_plot(user), recorded)
 
 
 @router.post("/upgrade")
 async def game_upgrade(request: Request):
     user = require_user(request)
-    return await _respond_action(request, user, lambda: store.upgrade_ci(user))
+
+    def recorded(result):
+        audit.record(
+            request,
+            "game.ci.upgrade",
+            user=user,
+            metadata=result,
+            summary=(
+                f"{user['username']} upgraded Code Farm CI to tier "
+                f"{result['ci_tier']} for {result['spent']} coins"
+            ),
+        )
+
+    return await _respond_action(request, user, lambda: store.upgrade_ci(user), recorded)
 
 
 @router.post("/fertilize")
 async def game_fertilize(request: Request, data: Annotated[GameSlotForm, Form()]):
     user = require_user(request)
-    return await _respond_action(request, user, lambda: store.fertilize(user, data.slot))
+
+    def recorded(result):
+        audit.record(
+            request,
+            "game.fertilize",
+            user=user,
+            metadata=result,
+            summary=(
+                f"{user['username']} fertilized plot {result['slot']} "
+                f"for {result['spent']} coins"
+            ),
+        )
+
+    return await _respond_action(
+        request, user, lambda: store.fertilize(user, data.slot), recorded
+    )
 
 
 @router.post("/daily")
 async def game_daily(request: Request):
     user = require_user(request)
-    return await _respond_action(request, user, lambda: store.claim_daily(user))
+
+    def recorded(result):
+        audit.record(
+            request,
+            "game.daily.claim",
+            user=user,
+            metadata=result,
+            summary=(
+                f"{user['username']} claimed the Code Farm daily bonus of "
+                f"{result['reward']} coins (streak {result['streak']})"
+            ),
+        )
+
+    return await _respond_action(request, user, lambda: store.claim_daily(user), recorded)
 
 
 @router.post("/grant")
@@ -141,8 +218,21 @@ async def game_claim_grant(request: Request):
 @router.post("/perk")
 async def game_perk(request: Request, data: Annotated[GamePerkForm, Form()]):
     user = require_user(request)
+
+    def recorded(result):
+        audit.record(
+            request,
+            "game.perk.upgrade",
+            user=user,
+            metadata=result,
+            summary=(
+                f"{user['username']} upgraded perk {result['perk']} to level "
+                f"{result['level']} for {result['spent']} coins"
+            ),
+        )
+
     return await _respond_action(
-        request, user, lambda: store.upgrade_perk(user, data.perk)
+        request, user, lambda: store.upgrade_perk(user, data.perk), recorded
     )
 
 
@@ -168,8 +258,21 @@ async def game_prestige(request: Request):
 @router.post("/legacy")
 async def game_legacy(request: Request, data: Annotated[GameLegacyForm, Form()]):
     user = require_user(request)
+
+    def recorded(result):
+        audit.record(
+            request,
+            "game.legacy.upgrade",
+            user=user,
+            metadata=result,
+            summary=(
+                f"{user['username']} upgraded Legacy {result['key']} to level "
+                f"{result['level']} for {result['spent']} stars"
+            ),
+        )
+
     return await _respond_action(
-        request, user, lambda: store.upgrade_legacy(user, data.key)
+        request, user, lambda: store.upgrade_legacy(user, data.key), recorded
     )
 
 
@@ -179,6 +282,16 @@ async def game_claim_quest(request: Request, data: Annotated[GameQuestForm, Form
 
     def reward(result):
         award_rewards(user["uid"], economy.site_xp_for(result.get("reward_xp", 0)))
+        audit.record(
+            request,
+            "game.quest.claim",
+            user=user,
+            metadata=result,
+            summary=(
+                f"{user['username']} claimed quest {result['kind']} for "
+                f"{result['reward_coins']} coins"
+            ),
+        )
 
     return await _respond_action(
         request, user, lambda: store.claim_quest(user, data.quest, data.scope), reward
@@ -251,8 +364,21 @@ async def game_buy_infrastructure(request: Request, data: Annotated[GameInfraFor
 @router.post("/mastery")
 async def game_upgrade_mastery(request: Request, data: Annotated[GameMasteryForm, Form()]):
     user = require_user(request)
+
+    def recorded(result):
+        audit.record(
+            request,
+            "game.mastery.upgrade",
+            user=user,
+            metadata=result,
+            summary=(
+                f"{user['username']} upgraded Mastery {result['key']} to level "
+                f"{result['level']} for {result['spent']} mastery points"
+            ),
+        )
+
     return await _respond_action(
-        request, user, lambda: store.upgrade_mastery(user, data.key)
+        request, user, lambda: store.upgrade_mastery(user, data.key), recorded
     )
 
 
@@ -281,6 +407,16 @@ async def game_buy_cosmetic(request: Request, data: Annotated[GameCosmeticForm,
 @router.post("/cosmetics/equip")
 async def game_equip_cosmetic(request: Request, data: Annotated[GameCosmeticForm, Form()]):
     user = require_user(request)
+
+    def recorded(result):
+        audit.record(
+            request,
+            "game.cosmetic.equip",
+            user=user,
+            metadata=result,
+            summary=f"{user['username']} equipped Code Farm title {result['active_title']}",
+        )
+
     return await _respond_action(
-        request, user, lambda: store.equip_title(user, data.key)
+        request, user, lambda: store.equip_title(user, data.key), recorded
     )

devplacepy/routers/issues/attachments.py

@@ -8,10 +8,12 @@ from fastapi.responses import JSONResponse
 
 from devplacepy.attachments import (
     get_attachments,
+    get_orphan_attachments_batch,
     link_attachments,
-    mirror_attachment_to_gitea,
-    remove_gitea_asset,
+    schedule_gitea_mirror,
+    schedule_gitea_removal,
     soft_delete_attachment,
+    split_attachment_uids,
 )
 from devplacepy.database import get_table
 from devplacepy.dependencies import json_or_form
@@ -28,10 +30,6 @@ logger = logging.getLogger(__name__)
 router = APIRouter()
 
 
-def _split_uids(raw) -> list[str]:
-    return [u.strip() for item in raw or [] for u in str(item).split(",") if u.strip()]
-
-
 def _can_modify(att: dict, user: dict | None, is_open: bool) -> bool:
     if not user or not is_open:
         return False
@@ -47,21 +45,6 @@ def _payload(rows: list[dict], user: dict | None, is_open: bool) -> list[dict]:
     return items
 
 
-def _claim_orphans(uids: list[str], user: dict) -> list[str]:
-    admin = is_admin(user)
-    owned = []
-    for uid in uids:
-        row = get_table("attachments").find_one(uid=uid, deleted_at=None)
-        if not row:
-            continue
-        if row.get("user_uid") and row["user_uid"] != user["uid"] and not admin:
-            continue
-        if row.get("target_uid"):
-            continue
-        owned.append(uid)
-    return owned
-
-
 async def _load_issue(number: int) -> dict:
     try:
         return await runtime.get_client().get_issue(number)
@@ -106,12 +89,14 @@ async def add_issue_attachment(
             summary=f"{user['username']} tried to attach to closed issue #{number}",
         )
         return json_error(409, "Attachments cannot be changed on a closed issue")
-    owned = _claim_orphans(_split_uids(data.attachment_uids), user)
+    owned = get_orphan_attachments_batch(
+        split_attachment_uids(data.attachment_uids), user, admin=is_admin(user)
+    )
     if not owned:
         return json_error(400, "No valid attachments to add")
     link_attachments(owned, "issue", str(number))
     for uid in owned:
-        await mirror_attachment_to_gitea(uid)
+        schedule_gitea_mirror(uid)
     audit.record(
         request,
         "issue.attachment.add",
@@ -161,7 +146,7 @@ async def delete_issue_attachment(request: Request, number: int, uid: str):
         )
         return json_error(409, "Attachments cannot be changed on a closed issue")
     soft_delete_attachment(uid, deleted_by=user["uid"])
-    await remove_gitea_asset(att)
+    schedule_gitea_removal(att)
     audit.record(
         request,
         "issue.attachment.delete",
@@ -202,12 +187,14 @@ async def add_comment_attachment(
     issue = await _load_issue(number)
     if issue.get("state") != STATE_OPEN:
         return json_error(409, "Attachments cannot be changed on a closed issue")
-    owned = _claim_orphans(_split_uids(data.attachment_uids), user)
+    owned = get_orphan_attachments_batch(
+        split_attachment_uids(data.attachment_uids), user, admin=is_admin(user)
+    )
     if not owned:
         return json_error(400, "No valid attachments to add")
     link_attachments(owned, "issue_comment", str(cid))
     for uid in owned:
-        await mirror_attachment_to_gitea(uid)
+        schedule_gitea_mirror(uid)
     audit.record(
         request,
         "issue.attachment.add",
@@ -250,7 +237,7 @@ async def delete_comment_attachment(
     if issue.get("state") != STATE_OPEN:
         return json_error(409, "Attachments cannot be changed on a closed issue")
     soft_delete_attachment(uid, deleted_by=user["uid"])
-    await remove_gitea_asset(att)
+    schedule_gitea_removal(att)
     audit.record(
         request,
         "issue.attachment.delete",

devplacepy/routers/issues/comment.py

@@ -5,7 +5,12 @@ from typing import Annotated
 
 from fastapi import Depends, APIRouter, Request
 
-from devplacepy.attachments import link_attachments, mirror_attachment_to_gitea
+from devplacepy.attachments import (
+    get_orphan_attachments_batch,
+    link_attachments,
+    schedule_gitea_mirror,
+    split_attachment_uids,
+)
 from devplacepy.database import get_table
 from devplacepy.models import IssueCommentForm
 from devplacepy.responses import action_result, json_error
@@ -21,18 +26,6 @@ logger = logging.getLogger(__name__)
 router = APIRouter()
 
 
-def _owned_orphan_uids(raw: list[str], user: dict) -> list[str]:
-    flat = [u.strip() for item in raw or [] for u in str(item).split(",") if u.strip()]
-    owned = []
-    for uid in flat:
-        row = get_table("attachments").find_one(uid=uid, deleted_at=None)
-        if not row or row.get("target_uid"):
-            continue
-        if row.get("user_uid") and row["user_uid"] != user["uid"]:
-            continue
-        owned.append(uid)
-    return owned
-
 def _notify_admins(actor: dict, number: int) -> None:
     for admin in get_table("users").find(role="Admin"):
         if admin["uid"] == actor["uid"]:
@@ -74,11 +67,13 @@ async def comment_issue(
 
     comment_id = int(comment.get("id", 0))
     store.record_comment_author(comment_id, number, user["uid"])
-    owned = _owned_orphan_uids(data.attachment_uids, user)
+    owned = get_orphan_attachments_batch(
+        split_attachment_uids(data.attachment_uids), user
+    )
     if owned:
         link_attachments(owned, "issue_comment", str(comment_id))
         for uid in owned:
-            await mirror_attachment_to_gitea(uid)
+            schedule_gitea_mirror(uid)
     background.submit(_notify_admins, user, number)
     audit.record(
         request,

devplacepy/routers/issues/create.py

@@ -6,7 +6,7 @@ from typing import Annotated
 from fastapi import Depends, APIRouter, Request
 from fastapi.responses import JSONResponse
 
-from devplacepy.database import get_table
+from devplacepy.attachments import get_orphan_attachments_batch, split_attachment_uids
 from devplacepy.models import IssueForm
 from devplacepy.responses import json_error
 from devplacepy.schemas import IssueJobOut
@@ -20,19 +20,6 @@ logger = logging.getLogger(__name__)
 router = APIRouter()
 
 
-def _owned_orphan_uids(raw: list[str], user: dict) -> list[str]:
-    flat = [u.strip() for item in raw or [] for u in str(item).split(",") if u.strip()]
-    owned = []
-    for uid in flat:
-        row = get_table("attachments").find_one(uid=uid, deleted_at=None)
-        if not row or row.get("target_uid"):
-            continue
-        if row.get("user_uid") and row["user_uid"] != user["uid"]:
-            continue
-        owned.append(uid)
-    return owned
-
-
 @router.post("/create")
 async def create_issue(request: Request, data: Annotated[IssueForm, Depends(json_or_form(IssueForm))]):
     user = require_user(request)
@@ -45,7 +32,9 @@ async def create_issue(request: Request, data: Annotated[IssueForm, Depends(json
             "author_uid": user["uid"],
             "title": title,
             "description": data.description.strip(),
-            "attachment_uids": _owned_orphan_uids(data.attachment_uids, user),
+            "attachment_uids": get_orphan_attachments_batch(
+                split_attachment_uids(data.attachment_uids), user
+            ),
         },
         "user",
         user["uid"],

devplacepy/routers/messages.py

@@ -20,7 +20,6 @@ from devplacepy.templating import clear_messages_cache
 from devplacepy.utils import (
     require_user,
     time_ago,
-    is_admin,
     _user_from_session,
     _user_from_api_key,
 )
@@ -31,6 +30,7 @@ from devplacepy.services import presence
 from devplacepy.services.audit import record as audit
 from devplacepy.services.correction import PENDING_SCOPE_KEY
 from devplacepy.dependencies import json_or_form
+from devplacepy.services.moderation.screening import ContentRefused
 from devplacepy.services.messaging import (
     issue_ticket,
     message_frame,
@@ -38,6 +38,8 @@ from devplacepy.services.messaging import (
     message_relay,
     persist_message,
     redeem_ticket,
+    stamp_content_revision,
+    touch_active_conversation,
 )
 
 logger = logging.getLogger(__name__)
@@ -113,22 +115,38 @@ def get_conversations(user_uid: str):
         conv.pop("other_uid", None)
     return conversations
 
-def get_conversation_messages(user_uid: str, other_uid: str):
+def get_conversation_messages(user_uid: str, other_uid: str, before: str = ""):
     if other_uid in get_blocked_uids(user_uid):
         return [], None
     if "messages" not in db.tables:
         return [], get_users_by_uids([other_uid]).get(other_uid)
-    msgs = list(
-        db.query(
-            "SELECT * FROM messages"
-            " WHERE (sender_uid = :me AND receiver_uid = :other)"
-            "    OR (sender_uid = :other AND receiver_uid = :me)"
-            " ORDER BY created_at DESC, id DESC LIMIT :lim",
-            me=user_uid,
-            other=other_uid,
-            lim=CONVERSATION_MESSAGE_LIMIT,
+    before = (before or "").strip()
+    if before:
+        msgs = list(
+            db.query(
+                "SELECT * FROM messages"
+                " WHERE ((sender_uid = :me AND receiver_uid = :other)"
+                "    OR (sender_uid = :other AND receiver_uid = :me))"
+                "   AND created_at < :before"
+                " ORDER BY created_at DESC, id DESC LIMIT :lim",
+                me=user_uid,
+                other=other_uid,
+                before=before,
+                lim=CONVERSATION_MESSAGE_LIMIT,
+            )
+        )
+    else:
+        msgs = list(
+            db.query(
+                "SELECT * FROM messages"
+                " WHERE (sender_uid = :me AND receiver_uid = :other)"
+                "    OR (sender_uid = :other AND receiver_uid = :me)"
+                " ORDER BY created_at DESC, id DESC LIMIT :lim",
+                me=user_uid,
+                other=other_uid,
+                lim=CONVERSATION_MESSAGE_LIMIT,
+            )
         )
-    )
     msgs.reverse()
 
     user_ids = list({m["sender_uid"] for m in msgs} | {other_uid})
@@ -158,7 +176,9 @@ def get_conversation_messages(user_uid: str, other_uid: str):
     return result, other_user
 
 @router.get("", response_class=HTMLResponse)
-async def messages_page(request: Request, with_uid: str = None, search: str = ""):
+async def messages_page(
+    request: Request, with_uid: str = None, search: str = "", before: str = ""
+):
     user = require_user(request)
     conversations = get_conversations(user["uid"])
 
@@ -174,25 +194,28 @@ async def messages_page(request: Request, with_uid: str = None, search: str = ""
     other_online = False
     other_last_seen = None
     if with_uid:
-        messages, other_user = get_conversation_messages(user["uid"], with_uid)
-        mark_conversation_read(user["uid"], with_uid)
-        mark_notifications_read_by_target(
-            user["uid"], f"/messages?with_uid={with_uid}"
+        messages, other_user = get_conversation_messages(
+            user["uid"], with_uid, before=before
         )
         current_conversation = with_uid
         other_online = presence.is_online(other_user)
         other_last_seen = other_user.get("last_seen") if other_user else None
-        audit.record(
-            request,
-            "message.read_on_view",
-            user=user,
-            target_type="user",
-            target_uid=with_uid,
-            target_label=other_user.get("username") if other_user else None,
-            metadata={"message_count": len(messages)},
-            summary=f"{user['username']} read messages from {other_user.get('username') if other_user else with_uid}",
-            links=[audit.target("user", with_uid, other_user.get("username") if other_user else None)],
-        )
+        if not before:
+            mark_conversation_read(user["uid"], with_uid)
+            mark_notifications_read_by_target(
+                user["uid"], f"/messages?with_uid={with_uid}"
+            )
+            audit.record(
+                request,
+                "message.read_on_view",
+                user=user,
+                target_type="user",
+                target_uid=with_uid,
+                target_label=other_user.get("username") if other_user else None,
+                metadata={"message_count": len(messages)},
+                summary=f"{user['username']} read messages from {other_user.get('username') if other_user else with_uid}",
+                links=[audit.target("user", with_uid, other_user.get("username") if other_user else None)],
+            )
 
     seo_ctx = base_seo_context(
         request,
@@ -254,12 +277,13 @@ async def send_message(request: Request, data: Annotated[MessageForm, Depends(js
         data.attachment_uids,
         request=request,
         origin="web",
+        client_id=data.client_id,
     )
     if message is None:
         return action_result(request, "/messages")
 
     ai_processed = await _finalize_and_broadcast(
-        user, message, request, client_id=data.client_id
+        user, message, request, client_id=data.client_id, wait_ai=True
     )
     frame = message_frame(
         message, user.get("username", ""), data.client_id,
@@ -271,31 +295,107 @@ async def send_message(request: Request, data: Annotated[MessageForm, Depends(js
 
 async def broadcast_message(
     sender: dict, message: dict, client_id: Optional[str] = None,
-    ai_processed: bool = False,
+    ai_processed: bool = False, ai_pending: bool = False,
 ) -> None:
     frame = message_frame(
         message, sender.get("username", ""), client_id,
         sender_role=sender.get("role"), ai_processed=ai_processed,
     )
-    message_hub.mark_delivered(message["uid"])
+    frame["ai_pending"] = ai_pending
+    if not ai_processed:
+        message_hub.mark_delivered(message["uid"])
     targets = [message["sender_uid"], message["receiver_uid"]]
     await message_hub.send_to_users(targets, frame)
 
+async def _await_ai_and_push(
+    sender: dict, message: dict, pending: list, client_id: Optional[str]
+) -> bool:
+    await asyncio.gather(*pending, return_exceptions=True)
+    pending.clear()
+    row = get_table("messages").find_one(uid=message["uid"])
+    if not row:
+        return False
+    changed = row.get("content") != message.get("content")
+    if changed:
+        message["content"] = row["content"]
+        stamp_content_revision(message["uid"])
+    await broadcast_message(sender, message, client_id, ai_processed=True)
+    return changed
+
 async def _finalize_and_broadcast(
-    sender: dict, message: dict, request: object, client_id: Optional[str] = None
+    sender: dict,
+    message: dict,
+    request: object,
+    client_id: Optional[str] = None,
+    wait_ai: bool = True,
 ) -> bool:
-    message_hub.mark_delivered(message["uid"])
     scope = getattr(request, "scope", None)
     pending = scope.get(PENDING_SCOPE_KEY) if scope is not None else None
-    ai_processed = bool(pending)
-    if pending:
-        await asyncio.gather(*pending, return_exceptions=True)
-        pending.clear()
-        row = get_table("messages").find_one(uid=message["uid"])
-        if row:
-            message["content"] = row["content"]
-    await broadcast_message(sender, message, client_id, ai_processed=ai_processed)
-    return ai_processed
+    has_pending = bool(pending)
+    await broadcast_message(
+        sender, message, client_id, ai_processed=False, ai_pending=has_pending
+    )
+    if not pending:
+        return False
+    if wait_ai:
+        return await _await_ai_and_push(sender, message, pending, client_id)
+    asyncio.create_task(_await_ai_and_push(sender, message, pending, client_id))
+    return False
+
+SYNC_LIMIT = 200
+
+
+async def _sync_missed(user_uid: str, data: dict, websocket: WebSocket) -> None:
+    since = str(data.get("since") or "").strip()
+    with_uid = str(data.get("with_uid") or "").strip()
+    if not since or "messages" not in db.tables:
+        return
+    if with_uid:
+        rows = list(
+            db.query(
+                "SELECT * FROM messages"
+                " WHERE ((sender_uid = :me AND receiver_uid = :other)"
+                "    OR (sender_uid = :other AND receiver_uid = :me))"
+                "   AND (created_at > :since"
+                "        OR (updated_at IS NOT NULL AND updated_at > :since))"
+                " ORDER BY created_at ASC, id ASC LIMIT :lim",
+                me=user_uid,
+                other=with_uid,
+                since=since,
+                lim=SYNC_LIMIT,
+            )
+        )
+    else:
+        rows = list(
+            db.query(
+                "SELECT * FROM messages"
+                " WHERE (sender_uid = :me OR receiver_uid = :me)"
+                "   AND (created_at > :since"
+                "        OR (updated_at IS NOT NULL AND updated_at > :since))"
+                " ORDER BY created_at ASC, id ASC LIMIT :lim",
+                me=user_uid,
+                since=since,
+                lim=SYNC_LIMIT,
+            )
+        )
+    if not rows:
+        return
+    sender_uids = {row["sender_uid"] for row in rows}
+    senders = get_users_by_uids(list(sender_uids)) if sender_uids else {}
+    for row in rows:
+        sender = senders.get(row["sender_uid"]) or {}
+        frame = message_frame(
+            dict(row),
+            sender.get("username", ""),
+            sender_role=sender.get("role"),
+            ai_processed=bool(row.get("updated_at")),
+        )
+        try:
+            await websocket.send_json(frame)
+        except Exception:  # noqa: BLE001
+            logger.debug("sync frame dropped for %s", user_uid)
+            return
+
 
 def _resolve_ws_user(websocket: WebSocket):
     user = _user_from_session(websocket)
@@ -351,26 +451,45 @@ async def messages_ws(websocket: WebSocket):
                 attachment_uids = [str(a) for a in raw_attachments][:MAX_WS_ATTACHMENTS]
                 if not receiver_uid:
                     continue
-                message = persist_message(
-                    user,
-                    receiver_uid,
-                    content,
-                    attachment_uids,
-                    request=websocket,
-                    origin="websocket",
-                )
+                try:
+                    message = persist_message(
+                        user,
+                        receiver_uid,
+                        content,
+                        attachment_uids,
+                        request=websocket,
+                        origin="websocket",
+                        client_id=client_id,
+                    )
+                except ContentRefused as exc:
+                    await websocket.send_json(
+                        {
+                            "type": "error",
+                            "client_id": client_id,
+                            "text": exc.message,
+                        }
+                    )
+                    continue
                 if message is None:
                     await websocket.send_json(
                         {"type": "error", "client_id": client_id, "text": "Message not sent."}
                     )
                     continue
-                await _finalize_and_broadcast(user, message, websocket, client_id)
+                await _finalize_and_broadcast(
+                    user, message, websocket, client_id, wait_ai=False
+                )
+            elif kind == "sync":
+                await _sync_missed(user_uid, data, websocket)
             elif kind == "typing":
                 receiver_uid = str(data.get("receiver_uid", "")).strip()
                 if receiver_uid:
                     await message_hub.send_to_user(
                         receiver_uid, {"type": "typing", "from_uid": user_uid}
                     )
+            elif kind == "active":
+                with_uid = str(data.get("with_uid", "")).strip()
+                if with_uid:
+                    touch_active_conversation(user_uid, with_uid)
             elif kind == "read":
                 with_uid = str(data.get("with_uid", "")).strip()
                 if with_uid:

devplacepy/routers/news.py

@@ -13,6 +13,7 @@ from devplacepy.database import (
     get_news_images_by_uids,
     get_recent_comments_by_target_uids,
     get_user_bookmarks,
+    get_user_notes,
     paginate,
     resolve_object_url,
     mark_notifications_read_by_target,
@@ -125,6 +126,11 @@ async def news_detail_page(request: Request, news_slug: str):
     bookmarked = bool(user) and article["uid"] in get_user_bookmarks(
         user["uid"], "news", [article["uid"]]
     )
+    note_content = (
+        get_user_notes(user["uid"], "news", [article["uid"]]).get(article["uid"])
+        if user
+        else None
+    )
 
     base = site_url(request)
     page_url = f"{base}/news/{canonical_slug}"
@@ -157,6 +163,7 @@ async def news_detail_page(request: Request, news_slug: str):
             "time_ago": time_ago(article["synced_at"]),
             "comments": comments,
             "bookmarked": bookmarked,
+            "note_content": note_content,
             "maturity": get_maturity("news", article["uid"])["level"],
         },
         model=NewsDetailOut,

devplacepy/routers/notes.py

@@ -0,0 +1,188 @@
+# retoor <retoor@molodetz.nl>
+import logging
+from datetime import datetime, timezone
+from typing import Annotated
+from fastapi import APIRouter, Form, Request
+from fastapi.responses import RedirectResponse, JSONResponse, HTMLResponse
+from devplacepy.database import get_table, db, paginate, resolve_object_url, _now_iso
+from devplacepy.models import NoteForm
+from devplacepy.utils import generate_uid, require_user, time_ago, redirect_back
+from devplacepy.seo import base_seo_context
+from devplacepy.responses import respond
+from devplacepy.schemas import NotesOut
+from devplacepy.services.audit import record as audit
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+NOTABLE: set[str] = {"post", "gist", "project", "news"}
+
+TABLE_BY_TYPE: dict[str, str] = {
+    "post": "posts",
+    "gist": "gists",
+    "project": "projects",
+    "news": "news",
+}
+
+LABEL_BY_TYPE: dict[str, str] = {
+    "post": "Post",
+    "gist": "Gist",
+    "project": "Project",
+    "news": "Article",
+}
+
+
+@router.get("/saved", response_class=HTMLResponse)
+async def notes_page(request: Request, before: str = None):
+    user = require_user(request)
+    notes = get_table("notes")
+    rows, next_cursor = paginate(notes, before=before, user_uid=user["uid"])
+
+    uids_by_type: dict[str, list] = {}
+    for row in rows:
+        uids_by_type.setdefault(row["target_type"], []).append(row["target_uid"])
+
+    resolved: dict[tuple, dict] = {}
+    for target_type, uids in uids_by_type.items():
+        table_name = TABLE_BY_TYPE.get(target_type)
+        if not table_name or table_name not in db.tables:
+            continue
+        table = get_table(table_name)
+        clauses = [table.table.columns.uid.in_(uids)]
+        if table.has_column("deleted_at"):
+            clauses.append(table.table.columns.deleted_at.is_(None))
+        for obj in table.find(*clauses):
+            resolved[(target_type, obj["uid"])] = obj
+
+    items = []
+    for row in rows:
+        obj = resolved.get((row["target_type"], row["target_uid"]))
+        if not obj:
+            continue
+        title = obj.get("title") or (obj.get("content", "") or "")[:80] or "Untitled"
+        items.append(
+            {
+                "target_type": row["target_type"],
+                "type_label": LABEL_BY_TYPE.get(
+                    row["target_type"], row["target_type"].title()
+                ),
+                "title": title,
+                "url": resolve_object_url(row["target_type"], row["target_uid"]),
+                "content": row["content"],
+                "time_ago": time_ago(row.get("updated_at") or row["created_at"]),
+                "updated_at": row.get("updated_at"),
+            }
+        )
+
+    seo_ctx = base_seo_context(
+        request,
+        title="Notes",
+        description="Your personal notes on DevPlace.",
+        robots="noindex,nofollow",
+    )
+    return respond(
+        request,
+        "notes.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "user": user,
+            "items": items,
+            "next_cursor": next_cursor,
+        },
+        model=NotesOut,
+    )
+
+
+@router.post("/{target_type}/{target_uid}")
+async def set_note(
+    request: Request,
+    target_type: str,
+    target_uid: str,
+    data: Annotated[NoteForm, Form()],
+):
+    user = require_user(request)
+    if target_type not in NOTABLE:
+        return JSONResponse({"error": "Invalid target"}, status_code=400)
+
+    notes = get_table("notes")
+    existing = notes.find_one(
+        user_uid=user["uid"], target_uid=target_uid, target_type=target_type
+    )
+    now = datetime.now(timezone.utc).isoformat()
+    content = data.content.strip()
+    if existing:
+        uid = existing["uid"]
+        notes.update(
+            {
+                "id": existing["id"],
+                "content": content,
+                "updated_at": now,
+                "deleted_at": None,
+                "deleted_by": None,
+            },
+            ["id"],
+        )
+    else:
+        uid = generate_uid()
+        notes.insert(
+            {
+                "uid": uid,
+                "user_uid": user["uid"],
+                "target_uid": target_uid,
+                "target_type": target_type,
+                "content": content,
+                "created_at": now,
+                "updated_at": now,
+                "deleted_at": None,
+                "deleted_by": None,
+            }
+        )
+
+    audit.record(
+        request,
+        "note.set",
+        user=user,
+        target_type=target_type,
+        target_uid=target_uid,
+        summary=f"{user['username']} saved a note on {target_type} {target_uid}",
+        links=[audit.target(target_type, target_uid)],
+    )
+
+    if request.headers.get("x-requested-with") == "fetch":
+        return JSONResponse({"uid": uid, "content": content})
+    return RedirectResponse(url=redirect_back(request), status_code=302)
+
+
+@router.post("/{target_type}/{target_uid}/delete")
+async def delete_note(request: Request, target_type: str, target_uid: str):
+    user = require_user(request)
+    if target_type not in NOTABLE:
+        return JSONResponse({"error": "Invalid target"}, status_code=400)
+
+    notes = get_table("notes")
+    existing = notes.find_one(
+        user_uid=user["uid"], target_uid=target_uid, target_type=target_type
+    )
+    if existing and not existing.get("deleted_at"):
+        notes.update(
+            {
+                "id": existing["id"],
+                "deleted_at": _now_iso(),
+                "deleted_by": user["uid"],
+            },
+            ["id"],
+        )
+        audit.record(
+            request,
+            "note.delete",
+            user=user,
+            target_type=target_type,
+            target_uid=target_uid,
+            summary=f"{user['username']} deleted a note on {target_type} {target_uid}",
+            links=[audit.target(target_type, target_uid)],
+        )
+
+    if request.headers.get("x-requested-with") == "fetch":
+        return JSONResponse({"deleted": True})
+    return RedirectResponse(url=redirect_back(request), status_code=302)

devplacepy/routers/posts.py

@@ -12,6 +12,8 @@ from devplacepy.database import (
     resolve_by_slug,
     resolve_object_url,
     mark_notifications_read_by_target,
+    get_featured_topics,
+    get_blocked_uids,
 )
 from devplacepy.utils import (
     get_current_user,
@@ -29,6 +31,8 @@ from devplacepy.content import (
     detail_context,
     canonical_redirect,
     first_image_url,
+    get_user_sidebar_gists,
+    get_user_sidebar_projects,
 )
 from devplacepy.responses import respond, action_result
 from devplacepy.schemas import PostDetailOut
@@ -37,6 +41,7 @@ from devplacepy.seo import (
     site_url,
     website_schema,
     discussion_forum_posting,
+    comment_schema_list,
 )
 from devplacepy.attachments import save_inline_image
 from devplacepy.models import PostForm, PostEditForm
@@ -136,20 +141,26 @@ def create_poll(
         links=[audit.poll(poll_uid, question), audit.parent("post", post_uid)],
     )
 
-@router.get("/{post_slug}", response_class=HTMLResponse)
-async def view_post(request: Request, post_slug: str):
-    user = get_current_user(request)
-    detail = load_detail("posts", "post", post_slug, user)
-    if not detail:
-        raise not_found("Post not found")
+def _next_post_url(post: dict, viewer: dict | None) -> str | None:
+    if "posts" not in db.tables:
+        return None
+    blocked = get_blocked_uids(viewer["uid"]) if viewer else frozenset()
+    rows = db.query(
+        "SELECT slug, uid, user_uid FROM posts WHERE created_at < :created_at "
+        "AND deleted_at IS NULL ORDER BY created_at DESC LIMIT :limit",
+        created_at=post["created_at"],
+        limit=10,
+    )
+    for row in rows:
+        if row["user_uid"] not in blocked:
+            return f"/posts/{row['slug'] or row['uid']}"
+    return None
+
+
+def post_page_context(
+    request: Request, user: dict | None, detail: dict, *, robots: str | None = None
+) -> dict:
     post = detail["item"]
-    redirect = canonical_redirect("posts", post, post_slug)
-    if redirect:
-        return redirect
-    if user:
-        mark_notifications_read_by_target(
-            user["uid"], resolve_object_url("post", post["uid"])
-        )
     author = detail["author"]
     top_level = detail["comments"]
 
@@ -161,10 +172,12 @@ async def view_post(request: Request, post_slug: str):
 
     comment_count = count_all(top_level)
     base = site_url(request)
+    next_post_url = _next_post_url(post, user)
     seo_ctx = base_seo_context(
         request,
         title=post.get("title") or "Post",
         description=post.get("content", ""),
+        robots=robots or "index,follow",
         seo_target=("post", post["uid"]),
         breadcrumbs=[
             {"name": "Home", "url": "/feed"},
@@ -176,10 +189,16 @@ async def view_post(request: Request, post_slug: str):
         ],
         og_type="article",
         og_image=first_image_url(post, detail["attachments"]),
+        next_url=next_post_url,
         schemas=[
             website_schema(base),
             discussion_forum_posting(
-                post, author, comment_count, detail["star_count"], base
+                post,
+                author,
+                comment_count,
+                detail["star_count"],
+                base,
+                comments=comment_schema_list(top_level, base),
             ),
         ],
     )
@@ -202,21 +221,43 @@ async def view_post(request: Request, post_slug: str):
                 }
             )
 
+    author_uid = post["user_uid"]
+    return detail_context(
+        request,
+        user,
+        detail,
+        "post",
+        seo_ctx,
+        {
+            "comment_count": comment_count,
+            "related_posts": related_posts,
+            "topics": list(TOPICS),
+            "featured_topics": get_featured_topics(3),
+            "author_gists": get_user_sidebar_gists(author_uid, 5),
+            "author_projects": get_user_sidebar_projects(author_uid, user, 5),
+            "next_post_url": next_post_url,
+        },
+    )
+
+
+@router.get("/{post_slug}", response_class=HTMLResponse)
+async def view_post(request: Request, post_slug: str):
+    user = get_current_user(request)
+    detail = load_detail("posts", "post", post_slug, user)
+    if not detail:
+        raise not_found("Post not found")
+    post = detail["item"]
+    redirect = canonical_redirect("posts", post, post_slug)
+    if redirect:
+        return redirect
+    if user:
+        mark_notifications_read_by_target(
+            user["uid"], resolve_object_url("post", post["uid"])
+        )
     return respond(
         request,
         "post.html",
-        detail_context(
-            request,
-            user,
-            detail,
-            "post",
-            seo_ctx,
-            {
-                "comment_count": comment_count,
-                "related_posts": related_posts,
-                "topics": list(TOPICS),
-            },
-        ),
+        post_page_context(request, user, detail),
         model=PostDetailOut,
     )
 

devplacepy/routers/projects/CLAUDE.md

@@ -11,7 +11,7 @@ Each project card links to `/projects/{project_uid}` showing full project detail
 
 **Project overview page.** The detail page is a dedicated project showcase: one encompassing dark card (`.project-shell`, the site `--bg-card` surface with clipped corners) wraps the hero, the section tab bar and the two-column body, and every inner panel (tab bar, sidebar cards, devlog post cards, empty state, comments section) sits one elevation lighter on `--bg-secondary`. The hero's cover banner is the attachment referenced by `projects.cover_attachment_uid`, falling back to the first image attachment (brand-gradient band when neither exists); the title block, type/platform chips and author row render OVERLAID on the banner behind a bottom scrim (dark text-shadow for readability) beside the optional `projects.logo_attachment_uid` tile, with an owner-set **Visit Website** CTA (`projects.website_url`). Cover and logo ride the ONE existing upload pipeline: `dp-upload` widgets (`name="cover_attachment_uid"`/`"logo_attachment_uid"`, `max-files="1"`) in the create/edit modals upload to `/uploads/upload`, the route validates each uid via `database.get_user_attachment` (must exist, belong to the actor, be an image - `_hero_attachment_uid`) and links it to the project through `attachments.link_attachments`; an empty value on edit keeps the current image (no removal control). `website_url`/`repo_url` are normalized by `models.normalize_website_url` (scheme-less input gets `https://`, non-http(s) rejected) and render with `rel="noopener nofollow"`. Below the hero an anchor **section tab bar** (`.project-tabs`, underline style, Overview `.active`) links `#about` / `#devlog` / `#screenshots` (only when gallery images exist) / `#comments` / the Files page - server-rendered anchors, no JS tab state. The main column holds **About** (description + non-image attachments), the **Devlog** (every post whose `project_uid` points at the project via `_post_card.html` - the template loads `feed.css` for the card styles alongside `post.css`, the same rule as `news.html`) with `devlog_count` (`content.count_project_devlog`) and an owner **Post update** button (`.project-devlog-post-btn`) opening the shared composer preset to `topic=devlog` + this project (the form lives ONCE in `templates/_post_composer_form.html`, locals `_composer_topic`/`_composer_project`, included by `feed.html` and `project_detail.html` - never fork a second copy), a **Screenshots** gallery (image attachments minus the cover/logo, thumbnails, `data-lightbox`, capped at 12 rendered), and the comment thread; the sidebar holds Links (website/repository/files/fork source), the Stats card (5 `.project-stat` entries + a last-update line) and the Author card. Owners add gallery images via the More-menu **Add screenshots** modal: `_attachment_form.html` uploads, then `POST /projects/{slug}/screenshots` (`ProjectScreenshotsForm`, owner-only, audit `project.screenshots.add`) links the uids through the same `link_attachments` choke point; Devii action `project_add_screenshots`, docs id `projects-screenshots`. `comment_count`/`devlog_count` ride `ProjectDetailOut`; the new project fields ride `ProjectOut`; the page og:image prefers the cover attachment. **Locator discipline:** the page has several `Files` anchors (action row, tab bar, sidebar) and, for owners, a second hidden `textarea[name='content']`/Post button inside the composer modal - tests MUST scope (`.project-detail-actions a:has-text('Files')`, `.comment-form textarea[name='content']`).
 
-**Action row overflow.** The detail page has more actions than fit one line, so `project_detail.html` keeps the engagement actions inline (Files, Share, star vote, bookmark, reactions) and collapses the rest behind a single **More** button (`.project-actions-more`) that opens the shared `app.contextMenu`. The secondary actions (Workspace, Containers, Download zip, Fork, the owner Edit/Private/Read-only/Delete controls) live as real elements inside a hidden `.project-actions-overflow` container, each tagged `data-menu-action` plus `data-menu-icon`/`data-menu-label`. `static/js/ProjectActionsMenu.js` builds the menu from those elements and each item's `onSelect` simply `.click()`s the real element, so all existing wiring is reused unchanged - `app.zipDownloader` (`data-zip-download`), `app.projectForker` (`data-fork-project`), `data-share`, the `data-modal` Edit trigger, and the delegated `data-confirm`/`data-confirm-danger` dialog on the owner forms. The open handler must `stopPropagation()` because `app.contextMenu`'s document-level close listener would otherwise dismiss it on the same click (every other caller opens it from a right-click `attach`, not a left-click). Reuse this pattern - a `More` trigger over `[data-menu-action]` real elements - for any future action row that overflows; do not duplicate controller logic into menu callbacks.
+**Action row overflow.** The detail page has more actions than fit one line, so `project_detail.html` keeps the engagement actions inline (Files, Share, star vote, bookmark, reactions) and collapses the rest behind a single **More** button (`.project-actions-more`) that opens the shared `app.contextMenu`. The secondary actions (Download zip, Fork, the owner Edit/Private/Read-only/Delete controls) live as real elements inside a hidden `.project-actions-overflow` container, each tagged `data-menu-action` plus `data-menu-icon`/`data-menu-label`. `static/js/ProjectActionsMenu.js` builds the menu from those elements and each item's `onSelect` simply `.click()`s the real element, so all existing wiring is reused unchanged - `app.zipDownloader` (`data-zip-download`), `app.projectForker` (`data-fork-project`), `data-share`, the `data-modal` Edit trigger, and the delegated `data-confirm`/`data-confirm-danger` dialog on the owner forms. The open handler must `stopPropagation()` because `app.contextMenu`'s document-level close listener would otherwise dismiss it on the same click (every other caller opens it from a right-click `attach`, not a left-click). Reuse this pattern - a `More` trigger over `[data-menu-action]` real elements - for any future action row that overflows; do not duplicate controller logic into menu callbacks.
 
 **Owner editing.** Mirrors post editing exactly: an owner-only **Edit** menu item (`data-modal="edit-project-modal"`) opens the `modal()` macro's `edit-project-modal`, a plain `POST` form to `/projects/edit/{slug}` (route `edit_project` in `routers/projects/index.py`, body `ProjectEditForm`, owner-gated through the shared `content.edit_content_item` which returns 403 JSON / redirect for non-owners and stamps `updated_at`). The modal is the create modal pre-filled from the `project` row (title, description, type/status radios pre-checked, dates via `format_date()` back to DD/MM/YYYY). `is_private`/`read_only` are NOT edited here - they stay on their dedicated toggles. The platforms tag widget reuses the create modal's `platforms-input`/`platforms`/`platforms-tags` ids; `ProfileEditor.initPlatformTags` now **seeds existing tags** from the hidden `#platforms` value on load, so both the empty create form and the pre-filled edit form work from the same code. Devii tool `edit_project`; documented in `docs_api.py` (`projects-edit`).
 

devplacepy/routers/projects/containers/instances.py

@@ -83,8 +83,6 @@ async def containers_json(request: Request, project_slug: str):
         }
     )
 
-# ---------------- instances ----------------
-
 @router.post("/{project_slug}/containers/instances")
 async def create_instance(
     request: Request, project_slug: str, data: Annotated[ContainerInstanceForm, Depends(json_or_form(ContainerInstanceForm))]
@@ -133,7 +131,7 @@ async def instance_detail(request: Request, project_slug: str, uid: str):
             "events": store.list_events(uid),
             "schedules": store.list_schedules(uid),
             "stats": api.instance_stats(uid),
-            "runtime": api.instance_runtime(inst),
+            "runtime": await api.instance_runtime(inst),
         }
     )
 

devplacepy/routers/projects/containers/workspace.py

@@ -74,11 +74,12 @@ async def workspace_page(request: Request, slug: str):
     profile = editor.resolve(user["uid"], instance)
     context = {
         "project": project,
-        "workspace": provision.view(instance) if instance else None,
+        "workspace": await provision.view(instance) if instance else None,
         "has_workspace": bool(instance),
         "viewer_can_workspace": True,
         "workspace_count": provision.count_for_owner(user["uid"]),
         "max_workspaces": limits.max_workspaces,
+        "unlimited_workspaces": limits.unlimited,
         "editor_url": (
             f"/projects/{slug}/containers/instances/{instance['uid']}/code/"
             if instance
@@ -129,7 +130,7 @@ async def workspace_open(request: Request, slug: str):
     )
     provision.write_manifest(instance)
     return action_result(
-        request, f"/projects/{slug}/workspace", data=provision.view(instance)
+        request, f"/projects/{slug}/workspace", data=await provision.view(instance)
     )
 
 
@@ -310,12 +311,18 @@ async def editor_proxy(request: Request, slug: str, uid: str, path: str = ""):
     if denial is not None:
         return denial
     if instance.get("suspended_at"):
-        return Response("this workspace is suspended", status_code=403)
+        return Response(
+            "this workspace is suspended", status_code=403, media_type="text/plain"
+        )
     if instance.get("status") != store.ST_RUNNING:
-        return Response("this workspace is not running", status_code=409)
+        return Response(
+            "this workspace is not running", status_code=409, media_type="text/plain"
+        )
     host, port = provision.editor_target(instance)
     if not host or not port:
-        return Response("the editor has no reachable port", status_code=502)
+        return Response(
+            "the editor has no reachable port", status_code=502, media_type="text/plain"
+        )
     activity.touch(instance["uid"])
     prefix = f"/projects/{slug}/containers/instances/{uid}/code"
     return await forward.proxy_http(request, host, port, path, prefix=prefix)

devplacepy/routers/projects/index.py

@@ -187,15 +187,12 @@ async def projects_page(
         model=ProjectsOut,
     )
 
-def _editor_launch(project: dict, user: dict) -> dict:
-    from devplacepy.services.containers import store
+async def _editor_launch(project: dict, user: dict) -> dict:
     from devplacepy.services.containers.workspace import editor, provision
 
     blank = {"url": "", "mode": "tab", "width": 0, "height": 0}
     instance = provision.find_for_project(project["uid"], user["uid"])
-    if not instance or instance.get("suspended_at"):
-        return blank
-    if instance.get("status") != store.ST_RUNNING:
+    if not instance or not await provision.editor_ready(instance):
         return blank
     slug = project["slug"] or project["uid"]
     profile = editor.resolve(user["uid"], instance)
@@ -265,7 +262,7 @@ async def project_detail(request: Request, project_slug: str, before: str = None
     )
     viewer_can_workspace = can_open_workspace(project, user)
     editor_launch = (
-        _editor_launch(project, user)
+        await _editor_launch(project, user)
         if viewer_can_workspace
         else {"url": "", "mode": "tab", "width": 0, "height": 0}
     )

devplacepy/routers/push.py

@@ -49,17 +49,24 @@ async def push_register(request: Request) -> JSONResponse:
     if fields is None:
         return JSONResponse({"error": "Invalid request"}, status_code=400)
 
-    _, created = push.register(user["uid"], provider.name, fields)
+    fields = provider.stamp_registration(fields)
+    write = push.register(user["uid"], provider.name, fields)
 
-    if created:
+    delivered = None
+    detail = ""
+    if write.probe:
         try:
-            await push.notify_user(user["uid"], WELCOME_PAYLOAD)
+            outcome = await push.notify_registration(write.record, WELCOME_PAYLOAD)
+            delivered = outcome.status == providers.ACCEPTED
+            detail = outcome.detail
         except Exception as exc:
             logger.warning("Welcome push failed for %s: %s", user["uid"], exc)
+            delivered = False
+            detail = str(exc)
 
     audit.record(
         request,
-        "push.subscribe" if created else "push.update",
+        "push.subscribe" if write.created else "push.update",
         user=user,
         target_type="user",
         target_uid=user["uid"],
@@ -69,12 +76,62 @@ async def push_register(request: Request) -> JSONResponse:
             "endpoint_host": urlparse(fields["endpoint"]).hostname
             if fields.get("endpoint")
             else None,
-            "created": created,
+            "created": write.created,
+            "revived": write.revived,
+            "has_client_id": bool(fields.get("client_id")),
         },
-        summary=f"{user.get('username')} {'registered' if created else 'updated'} a push subscription",
+        summary=f"{user.get('username')} {'registered' if write.created else 'updated'} a push subscription",
         links=[audit.target("user", user["uid"], user.get("username"))],
     )
-    return JSONResponse({"registered": True})
+    payload: dict = {"registered": True}
+    if delivered is not None:
+        payload["delivered"] = delivered
+        if detail and not delivered:
+            payload["error"] = detail
+    return JSONResponse(payload)
+
+
+@router.delete("/push.json")
+async def push_unregister(request: Request) -> JSONResponse:
+    user = require_user_api(request)
+    try:
+        body = await request.json()
+    except ValueError:
+        return JSONResponse({"error": "Invalid JSON"}, status_code=400)
+
+    if not isinstance(body, dict):
+        return JSONResponse({"error": "Invalid request"}, status_code=400)
+
+    provider = providers.get(body.get("provider"))
+    if provider is None:
+        return JSONResponse({"error": "Unknown provider"}, status_code=400)
+
+    identity = {key: body.get(key) for key in ("client_id", "token", "endpoint")}
+    if not any(isinstance(value, str) and value.strip() for value in identity.values()):
+        return JSONResponse({"error": "Invalid request"}, status_code=400)
+
+    removed = push.unregister(user["uid"], provider.name, identity)
+
+    if removed:
+        endpoint = identity.get("endpoint")
+        audit.record(
+            request,
+            "push.unsubscribe",
+            user=user,
+            target_type="user",
+            target_uid=user["uid"],
+            target_label=user.get("username"),
+            metadata={
+                "provider": provider.name,
+                "endpoint_host": urlparse(endpoint).hostname
+                if isinstance(endpoint, str) and endpoint
+                else None,
+                "has_client_id": bool(identity.get("client_id")),
+            },
+            summary=f"{user.get('username')} unregistered a push subscription",
+            links=[audit.target("user", user["uid"], user.get("username"))],
+        )
+    return JSONResponse({"unregistered": removed})
 
 
 @router.get("/service-worker.js")

devplacepy/routers/seo.py

@@ -15,8 +15,8 @@ async def robots_txt(request: Request):
     return PlainTextResponse(
         f"""User-agent: *
 Disallow: /auth/
-Disallow: /messages/
-Disallow: /notifications/
+Disallow: /messages
+Disallow: /notifications
 Disallow: /votes/
 Disallow: /avatar/
 Disallow: /follow/
@@ -24,6 +24,7 @@ Disallow: /admin/
 Disallow: /uploads/
 Disallow: /reports/mine
 Disallow: /profile/*/delete
+Disallow: /game
 Disallow: /*?tab=
 Disallow: /*?sort=
 Allow: /static/

devplacepy/routers/tools/deepsearch.py

@@ -11,7 +11,7 @@ from devplacepy import database
 from devplacepy.config import DEEPSEARCH_DIR
 from devplacepy.models import DeepsearchChatForm, DeepsearchRunForm
 from devplacepy.responses import respond
-from devplacepy.schemas import DeepsearchJobOut, DeepsearchSessionOut
+from devplacepy.schemas import DeepsearchHistoryOut, DeepsearchJobOut, DeepsearchSessionOut
 from devplacepy.seo import base_seo_context, site_url, web_application_schema, website_schema
 from devplacepy.services.deepsearch.chat import DeepsearchChat
 from devplacepy.services.deepsearch.export import to_json, to_markdown, to_pdf
@@ -197,6 +197,52 @@ def _enqueue(uid: str, payload: dict, owner_kind: str, owner_id: str, query: str
         }
     )
 
+def _history_item(row: dict) -> dict:
+    uid = row.get("uid", "")
+    status = row.get("status", "")
+    job = queue.get_job(uid)
+    return {
+        "uid": uid,
+        "query": row.get("query"),
+        "status": status,
+        "score": row.get("score"),
+        "confidence": row.get("confidence"),
+        "source_diversity": row.get("source_diversity"),
+        "page_count": int(row.get("page_count") or 0),
+        "chunk_count": int(row.get("chunk_count") or 0),
+        "summary": row.get("summary") or None,
+        "reopen_url": f"/tools/deepsearch/{uid}/session",
+        "chat_available": status == "done" and job is not None,
+        "available": job is not None,
+        "created_at": row.get("created_at"),
+        "completed_at": row.get("completed_at") or None,
+    }
+
+@router.get("/history")
+async def deepsearch_history(request: Request, limit: int = 20):
+    owner_kind, owner_id = owner_for(request)
+    user = get_current_user(request)
+    rows = database.list_deepsearch_sessions(owner_kind, owner_id, min(max(1, limit), 100))
+    sessions = [_history_item(row) for row in rows]
+    seo_ctx = base_seo_context(
+        request,
+        title="DeepSearch History",
+        description="Past DeepSearch research runs, with links back to each report and its grounded chat.",
+        robots="noindex,nofollow",
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {"name": "Tools", "url": "/tools"},
+            {"name": "DeepSearch", "url": "/tools/deepsearch"},
+            {"name": "History", "url": "/tools/deepsearch/history"},
+        ],
+    )
+    return respond(
+        request,
+        "tools/deepsearch_history.html",
+        {**seo_ctx, "request": request, "user": user, "sessions": sessions},
+        model=DeepsearchHistoryOut,
+    )
+
 @router.get("/{uid}")
 async def deepsearch_status(request: Request, uid: str):
     job = queue.get_job(uid)
@@ -243,6 +289,7 @@ def _session_context(request: Request, uid: str, job: dict, session: dict) -> di
         "sources": report.get("sources", []),
         "findings": report.get("findings", []),
         "timeline": report.get("timeline", []),
+        "follow_up_questions": report.get("follow_up_questions", []),
         "chat_ws_url": f"/tools/deepsearch/{uid}/chat" if done else None,
         "export_md_url": f"/tools/deepsearch/{uid}/export.md" if done else None,
         "export_json_url": f"/tools/deepsearch/{uid}/export.json" if done else None,

devplacepy/routers/tools/isslop.py

@@ -1,5 +1,6 @@
 # retoor <retoor@molodetz.nl>
 
+import asyncio
 import logging
 import re
 from urllib.parse import quote
@@ -435,7 +436,7 @@ async def isslop_source(request: Request, uid: str, path: str, line: int = 0):
     source_path = (store.media_dir_for(uid) / source_name).resolve()
     if not source_path.is_relative_to(ISSLOP_MEDIA_DIR.resolve()) or not source_path.is_file():
         raise not_found("Source not available for this file")
-    text = source_path.read_text(encoding="utf-8", errors="replace")
+    text = await asyncio.to_thread(source_path.read_text, encoding="utf-8", errors="replace")
     signals = store.decode_json(result.get("signals"), [])
     marked: dict[int, list] = {}
     for signal in signals:

devplacepy/routers/topics.py

@@ -0,0 +1,84 @@
+# retoor <retoor@molodetz.nl>
+
+from fastapi import APIRouter, Request
+from fastapi.responses import HTMLResponse
+from devplacepy.constants import TOPICS, TOPIC_LABELS
+from devplacepy.database import get_table
+from devplacepy.routers.feed import get_feed_posts, enrich_post_cards
+from devplacepy.utils import get_current_user, not_found
+from devplacepy.seo import list_page_seo, next_page_url
+from devplacepy.responses import respond
+from devplacepy.schemas import TopicOut, TopicsHubOut
+
+router = APIRouter()
+
+
+@router.get("", response_class=HTMLResponse)
+async def topics_hub(request: Request):
+    user = get_current_user(request)
+    posts_table = get_table("posts")
+    topics = [
+        {
+            "key": topic,
+            "label": TOPIC_LABELS.get(topic, topic.title()),
+            "post_count": posts_table.count(topic=topic, deleted_at=None),
+        }
+        for topic in TOPICS
+    ]
+
+    seo_ctx = list_page_seo(
+        request,
+        title="Topics",
+        description="Browse DevPlace posts by topic: devlog, showcase, questions, rants, fun, and more.",
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {"name": "Topics", "url": "/topics"},
+        ],
+    )
+    return respond(
+        request,
+        "topics.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "user": user,
+            "topics": topics,
+        },
+        model=TopicsHubOut,
+    )
+
+
+@router.get("/{topic}", response_class=HTMLResponse)
+async def topic_page(request: Request, topic: str, before: str = None):
+    if topic not in TOPICS:
+        raise not_found("Topic not found")
+    user = get_current_user(request)
+    posts, next_cursor = get_feed_posts(user, "all", topic, "", before)
+    posts = enrich_post_cards(posts, user)
+    label = TOPIC_LABELS.get(topic, topic.title())
+
+    seo_ctx = list_page_seo(
+        request,
+        title=f"{label} posts",
+        description=f"Browse {label.lower()} posts from developers on DevPlace.",
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {"name": "Topics", "url": "/topics"},
+            {"name": label, "url": f"/topics/{topic}"},
+        ],
+        next_url=next_page_url(request, next_cursor),
+    )
+    return respond(
+        request,
+        "topic.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "user": user,
+            "posts": posts,
+            "topic": topic,
+            "topic_label": label,
+            "next_cursor": next_cursor,
+        },
+        model=TopicOut,
+    )

devplacepy/schemas/__init__.py

@@ -43,6 +43,8 @@ from devplacepy.schemas.listings import (
     NewsDetailOut,
     NewsListItemOut,
     NewsListOut,
+    NoteItemOut,
+    NotesOut,
     NotificationGroupOut,
     NotificationItemOut,
     NotificationsOut,
@@ -52,6 +54,9 @@ from devplacepy.schemas.listings import (
     ProjectsOut,
     SavedItemOut,
     SavedOut,
+    TopicOut,
+    TopicSummaryOut,
+    TopicsHubOut,
 )
 from devplacepy.schemas.profile import (
     MediaItemOut,
@@ -59,6 +64,7 @@ from devplacepy.schemas.profile import (
     TelegramPairOut,
 )
 from devplacepy.schemas.issues import (
+    AdminIssuesPlanningOut,
     IssueAttachmentsOut,
     IssueCommentOut,
     IssueDetailOut,
@@ -84,6 +90,8 @@ from devplacepy.schemas.containers import (
 )
 from devplacepy.schemas.jobs import (
     DbQueryJobOut,
+    DeepsearchHistoryItemOut,
+    DeepsearchHistoryOut,
     DeepsearchJobOut,
     DeepsearchSessionOut,
     ForkJobOut,
@@ -120,6 +128,10 @@ from devplacepy.schemas.admin import (
     TrashItemOut,
 )
 from devplacepy.schemas.gateway import (
+    AdminGatewayModelFormOut,
+    AdminGatewayOut,
+    AdminGatewayProviderFormOut,
+    AdminGatewayQuotaFormOut,
     GatewayUsageOut,
     UserAiUsageOut,
 )

devplacepy/schemas/containers.py

@@ -155,8 +155,12 @@ class EditorProfileOut(_Out):
 class WorkspaceViewOut(_Out):
     uid: str = ""
     name: str = ""
+    owner_uid: str = ""
     status: str = ""
     desired_state: str = ""
+    phase: str = ""
+    phase_label: str = ""
+    editor_ready: bool = False
     suspended: bool = False
     flag_reason: Optional[str] = ""
     tunnel_name: Optional[str] = ""
@@ -171,6 +175,7 @@ class WorkspaceViewOut(_Out):
     idle_stop_minutes: int = 0
     retention_days: int = 0
     max_tunnels: int = 0
+    unlimited: bool = False
     tunnels: list[TunnelOut] = []
     flags: list[WorkspaceFlagOut] = []
     editor: Optional[EditorProfileOut] = None
@@ -183,6 +188,7 @@ class WorkspaceOut(_Out):
     viewer_can_workspace: bool = False
     workspace_count: int = 0
     max_workspaces: int = 0
+    unlimited_workspaces: bool = False
     editor_url: str = ""
     editor_password: str = ""
     editor: Optional[EditorProfileOut] = None

devplacepy/schemas/game.py

@@ -2,6 +2,8 @@
 
 from __future__ import annotations
 
+from typing import Optional
+
 from devplacepy.schemas.base import _Out
 
 
@@ -13,8 +15,11 @@ class GameCropOut(_Out):
     reward_coins: int = 0
     reward_xp: int = 0
     min_level: int = 1
+    min_mastery: int = 0
     grow_seconds: int = 0
     locked: bool = False
+    locked_reason: str = ""
+    locked_text: str = ""
     market_state: str = "normal"
 
 
@@ -183,6 +188,7 @@ class GameFarmOut(_Out):
 class GameStateOut(_Out):
     ok: bool = True
     farm: GameFarmOut
+    game_error: Optional[str] = None
 
 
 class GameFarmViewOut(_Out):
@@ -190,6 +196,7 @@ class GameFarmViewOut(_Out):
     page_title: str = ""
     meta_description: str = ""
     stole_coins: int = 0
+    game_error: Optional[str] = None
 
 
 class GameLeaderboardEntryOut(_Out):

devplacepy/schemas/gateway.py

@@ -40,3 +40,37 @@ class UserAiUsageOut(_Out):
     by_backend: list = []
     hourly: list = []
     notes: dict = {}
+
+
+class AdminGatewayOut(_Out):
+    tab: str = "models"
+    providers: list = []
+    models: list = []
+    quota_rules: list = []
+    default_provider: dict = {}
+    quota_defaults: dict = {}
+    stats_ranges: list = []
+    admin_section: Optional[str] = None
+
+
+class AdminGatewayProviderFormOut(_Out):
+    is_edit: bool = False
+    form: dict = {}
+    error: Optional[str] = None
+    admin_section: Optional[str] = None
+
+
+class AdminGatewayModelFormOut(_Out):
+    is_edit: bool = False
+    form: dict = {}
+    providers: list = []
+    fallback_groups: list = []
+    error: Optional[str] = None
+    admin_section: Optional[str] = None
+
+
+class AdminGatewayQuotaFormOut(_Out):
+    is_edit: bool = False
+    form: dict = {}
+    error: Optional[str] = None
+    admin_section: Optional[str] = None

devplacepy/schemas/issues.py

@@ -67,3 +67,16 @@ class IssuesOut(_Out):
     state: str = "open"
     configured: bool = True
     error_message: Optional[str] = None
+    viewer_is_admin: bool = False
+
+
+class AdminPlanningTicketOut(_Out):
+    number: int = 0
+    title: str = ""
+    labels: list[str] = []
+
+
+class AdminIssuesPlanningOut(_Out):
+    configured: bool = True
+    tickets: list[AdminPlanningTicketOut] = []
+    tickets_error: bool = False

devplacepy/schemas/jobs.py

@@ -132,6 +132,7 @@ class DeepsearchSessionOut(_Out):
     sources: list = []
     findings: list = []
     timeline: list = []
+    follow_up_questions: list = []
     chat_ws_url: Optional[str] = None
     export_md_url: Optional[str] = None
     export_json_url: Optional[str] = None
@@ -142,6 +143,27 @@ class DeepsearchSessionOut(_Out):
     completed_at: Optional[str] = None
 
 
+class DeepsearchHistoryItemOut(_Out):
+    uid: str = ""
+    query: Optional[str] = None
+    status: str = ""
+    score: Optional[int] = None
+    confidence: Optional[float] = None
+    source_diversity: Optional[float] = None
+    page_count: int = 0
+    chunk_count: int = 0
+    summary: Optional[str] = None
+    reopen_url: Optional[str] = None
+    chat_available: bool = False
+    available: bool = False
+    created_at: Optional[str] = None
+    completed_at: Optional[str] = None
+
+
+class DeepsearchHistoryOut(_Out):
+    sessions: list = []
+
+
 class DbQueryJobOut(_Out):
     uid: str = ""
     kind: str = ""
@@ -203,6 +225,10 @@ class IsslopReportOut(_Out):
     detected_builder: Optional[str] = None
     dom_slop_score: Optional[float] = None
     error: Optional[str] = None
+    report_url: Optional[str] = None
+    badge_url: Optional[str] = None
+    events_url: Optional[str] = None
+    topic: Optional[str] = None
     content_hash: Optional[str] = None
     markdown: str = ""
     generator_model: str = ""
@@ -225,3 +251,7 @@ class IsslopSourceOut(_Out):
     source: str = ""
     truncated: bool = False
     signals: list = []
+    source_lines: list = []
+    marked_lines: dict = {}
+    focus_line: int = 0
+    report_url: Optional[str] = None

devplacepy/schemas/listings.py

@@ -108,6 +108,23 @@ class SavedItemOut(_Out):
     time_ago: Optional[str] = None
 
 
+class TopicOut(_Out):
+    posts: list[FeedItemOut] = []
+    topic: str = ""
+    topic_label: str = ""
+    next_cursor: Optional[str] = None
+
+
+class TopicSummaryOut(_Out):
+    key: str = ""
+    label: str = ""
+    post_count: int = 0
+
+
+class TopicsHubOut(_Out):
+    topics: list[TopicSummaryOut] = []
+
+
 class FeedOut(_Out):
     posts: list[FeedItemOut] = []
     current_tab: Optional[str] = None
@@ -135,12 +152,17 @@ class PostDetailOut(_Out):
     attachments: list[AttachmentOut] = []
     reactions: ReactionsOut = ReactionsOut()
     bookmarked: bool = False
+    note_content: Optional[str] = None
     poll: Optional[PollOut] = None
     war: Optional[WarOut] = None
     comment_count: Optional[int] = None
     related_posts: list[FeedItemOut] = []
     topics: list[str] = []
     project_link: Optional[ProjectLinkOut] = None
+    featured_topics: list[Any] = []
+    author_gists: list[Any] = []
+    author_projects: list[Any] = []
+    next_post_url: Optional[str] = None
 
 
 class ProjectsOut(_Out):
@@ -166,6 +188,7 @@ class ProjectDetailOut(_Out):
     attachments: list[AttachmentOut] = []
     reactions: ReactionsOut = ReactionsOut()
     bookmarked: bool = False
+    note_content: Optional[str] = None
     platforms: Optional[Any] = None
     is_private: bool = False
     read_only: bool = False
@@ -206,6 +229,7 @@ class GistDetailOut(_Out):
     attachments: list[AttachmentOut] = []
     reactions: ReactionsOut = ReactionsOut()
     bookmarked: bool = False
+    note_content: Optional[str] = None
 
 
 class NewsListOut(_Out):
@@ -222,6 +246,7 @@ class NewsDetailOut(_Out):
     time_ago: Optional[str] = None
     comments: list[CommentItemOut] = []
     bookmarked: bool = False
+    note_content: Optional[str] = None
 
 
 class MessagesOut(_Out):
@@ -254,3 +279,19 @@ class SavedOut(_Out):
     items: list[SavedItemOut] = []
     next_cursor: Optional[str] = None
 
+
+class NoteItemOut(_Out):
+    target_type: Optional[str] = None
+    target_uid: Optional[str] = None
+    type_label: Optional[str] = None
+    title: Optional[str] = None
+    url: Optional[str] = None
+    content: Optional[str] = None
+    time_ago: Optional[str] = None
+    updated_at: Optional[str] = None
+
+
+class NotesOut(_Out):
+    items: list[NoteItemOut] = []
+    next_cursor: Optional[str] = None
+

devplacepy/schemas/quiz.py

@@ -184,6 +184,8 @@ class QuizAttemptOut(_Out):
 class QuizAttemptPageOut(_Out):
     quiz: QuizOut = QuizOut()
     attempt: QuizAttemptOut = QuizAttemptOut()
+    answer_max_chars: int = 0
+    quiz_error: Optional[str] = None
 
 
 class QuizResultOut(_Out):
@@ -225,6 +227,7 @@ class QuizBuilderOut(_Out):
     questions: list[QuizQuestionOut] = []
     kinds: list[Any] = []
     validation_errors: list[str] = []
+    quiz_error: Optional[str] = None
 
 
 class QuizFormPageOut(_Out):

devplacepy/schemas/statistics.py

@@ -39,7 +39,7 @@ class StatisticsHighlightOut(BaseModel):
     value: Any
 
 
-class StatisticsOut(BaseModel):
+class StatisticsPayloadOut(BaseModel):
     tab: str
     window_hours: int
     granularity: str
@@ -49,4 +49,18 @@ class StatisticsOut(BaseModel):
     series: list[StatisticsSeriesOut] = []
     tables: list[StatisticsTableOut] = []
     highlights: list[StatisticsHighlightOut] = []
-    notes: dict[str, Any] = {}
+    notes: dict[str, Any] = {}
+
+
+class StatisticsTabOut(BaseModel):
+    key: str
+    label: str
+    icon: str
+    active: bool
+
+
+class StatisticsOut(BaseModel):
+    active_tab: str
+    window_hours: int
+    tabs: list[StatisticsTabOut] = []
+    initial: StatisticsPayloadOut

devplacepy/seo.py

@@ -91,7 +91,41 @@ def breadcrumb_schema(items, base_url):
     }
 
 
-def discussion_forum_posting(post, author, comment_count, star_count, base_url):
+MAX_SCHEMA_COMMENTS = 20
+
+
+def comment_schema(comment_item, base_url):
+    comment = comment_item["comment"]
+    author = comment_item.get("author")
+    return {
+        "@type": "Comment",
+        "text": truncate(plain_markdown(comment.get("content", "")), 300),
+        "author": {
+            "@type": "Person",
+            "name": author["username"] if author else "Unknown",
+            "url": f"{base_url}/profile/{author['username']}" if author else "",
+        },
+        "datePublished": comment.get("created_at", ""),
+    }
+
+
+def comment_schema_list(comment_tree, base_url, limit=MAX_SCHEMA_COMMENTS):
+    flat = []
+
+    def walk(items):
+        for item in items:
+            if len(flat) >= limit:
+                return
+            flat.append(comment_schema(item, base_url))
+            walk(item.get("children", []))
+
+    walk(comment_tree)
+    return flat
+
+
+def discussion_forum_posting(
+    post, author, comment_count, star_count, base_url, comments=None
+):
     schema = {
         "@type": "DiscussionForumPosting",
         "headline": post.get("title") or "Untitled",
@@ -117,6 +151,8 @@ def discussion_forum_posting(post, author, comment_count, star_count, base_url):
             },
         ],
     }
+    if comments:
+        schema["comment"] = comments
     return schema
 
 
@@ -421,6 +457,13 @@ def _build_sitemap(base_url):
     urlset.append(url_element(f"{base_url}/", changefreq="daily", priority="1.0"))
     urlset.append(url_element(f"{base_url}/feed", changefreq="hourly", priority="0.9"))
     urlset.append(url_element(f"{base_url}/news", changefreq="hourly", priority="0.9"))
+    urlset.append(url_element(f"{base_url}/topics", changefreq="daily", priority="0.7"))
+    from devplacepy.constants import TOPICS
+
+    for topic in TOPICS:
+        urlset.append(
+            url_element(f"{base_url}/topics/{topic}", changefreq="daily", priority="0.7")
+        )
     urlset.append(
         url_element(f"{base_url}/projects", changefreq="daily", priority="0.8")
     )
@@ -434,6 +477,7 @@ def _build_sitemap(base_url):
     urlset.append(url_element(f"{base_url}/tools", changefreq="monthly", priority="0.5"))
     urlset.append(url_element(f"{base_url}/tools/seo", changefreq="monthly", priority="0.5"))
     urlset.append(url_element(f"{base_url}/tools/deepsearch", changefreq="monthly", priority="0.5"))
+    urlset.append(url_element(f"{base_url}/tools/isslop", changefreq="monthly", priority="0.5"))
     urlset.append(
         url_element(f"{base_url}/workspaces/index", changefreq="daily", priority="0.6")
     )