Zip filex

.coveragerc

@@ -0,0 +1,12 @@
+# retoor <retoor@molodetz.nl>
+[run]
+source = devplacepy
+parallel = true
+sigterm = true
+omit =
+    tests/*
+    sitecustomize.py
+
+[report]
+show_missing = true
+skip_covered = false

.env.example

@@ -0,0 +1,28 @@
+# Copy to .env and adjust. Loaded by docker-compose (env_file) and by the app
+# at startup (python-dotenv). .env is git-ignored; this example is committed.
+
+# Session signing key. CHANGE THIS for any real deployment.
+SECRET_KEY=change-me
+
+# Database. Leave unset to use the shared project-root devplace.db (the Docker
+# app container bind-mounts ./ to /app, so it reads and writes the same file as
+# `make dev`). Set only to point at a different SQLite file.
+# DEVPLACE_DATABASE_URL=sqlite:////app/devplace.db
+
+# Public origin for absolute URLs (SEO, canonical links, push). Empty = derive
+# from the request.
+DEVPLACE_SITE_URL=
+
+# Host port the nginx front door binds.
+PORT=10500
+
+# nginx upload ceiling. Must be >= the admin-configurable max_upload_size_mb.
+NGINX_MAX_BODY_SIZE=50m
+
+# Optional nginx micro-cache for proxied GETs.
+NGINX_CACHE_ENABLED=false
+NGINX_CACHE_MAX_SIZE=1g
+
+# Run the app container as this host user so shared files keep dev ownership.
+DEVPLACE_UID=1000
+DEVPLACE_GID=1000

.gitea/workflows/test.yaml

@@ -18,17 +18,34 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pip install -e .
-          pip install playwright
+          pip install -e ".[dev]"
           python -m playwright install chromium --with-deps
 
-      - name: Run integration tests
+      - name: Run integration tests with coverage
+        env:
+          COVERAGE_PROCESS_START: ${{ github.workspace }}/.coveragerc
+          PLAYWRIGHT_HEADLESS: "1"
         run: |
-          python -m pytest tests/ -v --tb=line -x
+          python -m coverage run -m pytest tests/ -p no:xdist --tb=line
+
+      - name: Build coverage report
+        if: always()
+        run: |
+          python -m coverage combine
+          python -m coverage report
+          python -m coverage html
+
+      - name: Publish coverage HTML
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage-html
+          path: htmlcov/
 
       - name: Upload test screenshots
         if: failure()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: failure-screenshots
           path: /tmp/devplace_test_screenshots/
+

.gitignore

@@ -1,7 +1,32 @@
+.cache
+.local
+.devplace_bots/
 __pycache__/
 *.py[cod]
 *.egg-info/
 .env
 devplace.db*
+devplace-services.lock
+devplace-init.lock
+.vapid.lock
+notification-private.pem
+notification-private.pkcs8.pem
+notification-public.pem
 .pytest_cache/
 .opencode
+devii_*.db
+devii_*.db-shm
+devii_*.db-wal
+devii.log
+webdata/
+devplacepy/static/uploads/attachments/
+devplacepy/static/uploads/*.png
+devplacepy/static/uploads/*.jpg
+devplacepy/static/uploads/*.jpeg
+devplacepy/static/uploads/*.gif
+devplacepy/static/uploads/*.webp
+
+# coverage
+.coverage
+.coverage.*
+htmlcov/

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11-slim
+FROM python:3.13-slim
 
 WORKDIR /app
 
@@ -10,13 +10,19 @@ COPY pyproject.toml .
 
 COPY devplacepy/ devplacepy/
 
-RUN pip install --no-cache-dir .
+ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
 
-RUN mkdir -p /app/data /app/devplacepy/static/uploads/attachments
+RUN pip install --no-cache-dir ".[bots]" \
+    && python -m playwright install --with-deps chromium \
+    && chmod -R a+rx /ms-playwright
+
+RUN mkdir -p /app/devplacepy/static/uploads/attachments
 
 EXPOSE 10500
 
+ENV DEVPLACE_WEB_WORKERS=2
+
 HEALTHCHECK --interval=30s --timeout=10s --retries=3 --start-period=10s \
     CMD curl -f http://localhost:10500/ || exit 1
 
-CMD ["uvicorn", "devplacepy.main:app", "--host", "0.0.0.0", "--port", "10500", "--workers", "4", "--backlog", "8192"]
+CMD ["uvicorn", "devplacepy.main:app", "--host", "0.0.0.0", "--port", "10500", "--workers", "2", "--backlog", "8192", "--proxy-headers", "--forwarded-allow-ips", "*"]

Makefile

@@ -5,8 +5,9 @@ LOCUST_DB ?= $(LOCUST_DB_DIR)/datastore.db
 LOCUST_USERS ?= 20
 LOCUST_SPAWN_RATE ?= 5
 LOCUST_RUN_TIME ?= 120s
+DEVPLACE_RATE_LIMIT ?= 1000000
 
-.PHONY: install dev clean test test-headed demo locust locust-headless
+.PHONY: install dev clean test test-headed coverage coverage-headed coverage-html locust locust-headless
 
 install:
 	pip install -e .
@@ -15,35 +16,58 @@ dev:
 	uvicorn devplacepy.main:app --reload --host 0.0.0.0 --port 10500 --backlog 4096
 
 prod:
-	uvicorn devplacepy.main:app --host 0.0.0.0 --port 10500 --workers 2 --backlog 8192
+	DEVPLACE_WEB_WORKERS=2 uvicorn devplacepy.main:app --host 0.0.0.0 --port 10500 --workers 2 --backlog 8192 --proxy-headers --forwarded-allow-ips '*'
 
 test:
-	PLAYWRIGHT_HEADLESS=1 python -m pytest tests/test_landing.py tests/test_auth.py tests/test_feed.py tests/test_post.py tests/test_profile.py tests/test_projects.py tests/test_messages.py tests/test_notifications.py tests/test_services.py tests/test_seo.py -v --tb=line -x
+	PLAYWRIGHT_HEADLESS=1 python -m pytest tests/ -v --tb=line -x
 
 test-headed:
-	PLAYWRIGHT_HEADLESS=0 python -m pytest tests/test_landing.py tests/test_auth.py tests/test_feed.py tests/test_post.py tests/test_profile.py tests/test_projects.py tests/test_messages.py tests/test_notifications.py tests/test_services.py -v --tb=line -x
+	PLAYWRIGHT_HEADLESS=0 python -m pytest tests/ -v --tb=line -x
 
-demo:
-	PLAYWRIGHT_HEADLESS=0 python -m pytest tests/test_demo.py -v -s --tb=line -x
+coverage:
+	rm -f .coverage .coverage.*
+	COVERAGE_PROCESS_START=$(CURDIR)/.coveragerc PLAYWRIGHT_HEADLESS=1 \
+		python -m coverage run -m pytest tests/ -p no:xdist --tb=line
+	python -m coverage combine
+	python -m coverage report
+
+coverage-headed:
+	rm -f .coverage .coverage.*
+	COVERAGE_PROCESS_START=$(CURDIR)/.coveragerc PLAYWRIGHT_HEADLESS=0 \
+		python -m coverage run -m pytest tests/ -p no:xdist --tb=line
+	python -m coverage combine
+	python -m coverage report
+
+coverage-html: coverage
+	python -m coverage html
+	@echo "Report written to htmlcov/index.html"
 
 locust:
 	export DEVPLACE_DATABASE_URL="sqlite:///$(LOCUST_DB)"; \
+	export DEVPLACE_RATE_LIMIT=$(DEVPLACE_RATE_LIMIT); \
+	fuser -k $(LOCUST_PORT)/tcp 2>/dev/null || true; \
+	sleep 1; \
 	mkdir -p $(LOCUST_DB_DIR); \
 	rm -f $(LOCUST_DB); \
 	uvicorn devplacepy.main:app --host 127.0.0.1 --port $(LOCUST_PORT) --backlog 8192 > /tmp/devplace_locust_server.log 2>&1 & \
 	PID=$$!; \
-	while ! curl -s http://127.0.0.1:$(LOCUST_PORT)/ > /dev/null 2>&1; do sleep 0.5; done; \
+	while kill -0 $$PID 2>/dev/null && ! curl -s http://127.0.0.1:$(LOCUST_PORT)/ > /dev/null 2>&1; do sleep 0.5; done; \
+	if ! kill -0 $$PID 2>/dev/null; then echo "Server failed to start (port $(LOCUST_PORT) busy?). See /tmp/devplace_locust_server.log"; exit 1; fi; \
 	locust --host http://127.0.0.1:$(LOCUST_PORT) --web-port $(LOCUST_WEB_PORT); \
 	kill $$PID 2>/dev/null || true; \
 	rm -rf $(LOCUST_DB_DIR)
 
 locust-headless:
 	export DEVPLACE_DATABASE_URL="sqlite:///$(LOCUST_DB)"; \
+	export DEVPLACE_RATE_LIMIT=$(DEVPLACE_RATE_LIMIT); \
+	fuser -k $(LOCUST_PORT)/tcp 2>/dev/null || true; \
+	sleep 1; \
 	mkdir -p $(LOCUST_DB_DIR); \
 	rm -f $(LOCUST_DB); \
 	uvicorn devplacepy.main:app --host 127.0.0.1 --port $(LOCUST_PORT) --backlog 8192 > /tmp/devplace_locust_server.log 2>&1 & \
 	PID=$$!; \
-	while ! curl -s http://127.0.0.1:$(LOCUST_PORT)/ > /dev/null 2>&1; do sleep 0.5; done; \
+	while kill -0 $$PID 2>/dev/null && ! curl -s http://127.0.0.1:$(LOCUST_PORT)/ > /dev/null 2>&1; do sleep 0.5; done; \
+	if ! kill -0 $$PID 2>/dev/null; then echo "Server failed to start (port $(LOCUST_PORT) busy?). See /tmp/devplace_locust_server.log"; exit 1; fi; \
 	locust --host http://127.0.0.1:$(LOCUST_PORT) --headless -u $(LOCUST_USERS) -r $(LOCUST_SPAWN_RATE) --run-time $(LOCUST_RUN_TIME) --html=$(LOCUST_DB_DIR)/report.html; \
 	kill $$PID 2>/dev/null || true; \
 	rm -rf $(LOCUST_DB_DIR)
@@ -70,3 +94,8 @@ docker-logs:
 
 docker-clean:
 	docker compose down -v
+
+deploy:
+	git checkout production
+	git merge master
+	git push origin production

README.md

@@ -1,4 +1,4 @@
-# DevPlace — The Developer Social Network
+# DevPlace - The Developer Social Network
 
 Server-rendered social network for developers. FastAPI backend serving Jinja2 templates with ES6 interactivity. Avatar generation via Multiavatar (local, no network). SQLite via `dataset` with WAL mode and concurrency tuning.
 
@@ -11,7 +11,6 @@ make install        # pip install -e .
 make dev            # uvicorn --reload on port 10500
 make test           # Playwright integration + unit tests, headless, fail-fast
 make test-headed    # same tests in visible browser
-make demo           # full-journey GUI demo (headed)
 ```
 
 Open `http://localhost:10500`.
@@ -20,13 +19,13 @@ Open `http://localhost:10500`.
 
 | Layer | Technology |
 |-------|-----------|
-| Backend | Python 3.13+, FastAPI, Uvicorn (single worker) |
+| Backend | Python 3.13+, FastAPI, Uvicorn (multi-worker in production) |
 | Templates | Jinja2 (server-side rendered) |
 | Frontend | Pure ES6 JavaScript, one class per file |
 | Database | SQLite via `dataset` (auto-sync schema, `uid` PKs, WAL mode, 30s busy timeout) |
-| Auth | Session cookies, SHA256+SALT via passlib |
+| Auth | Session cookie, API key (`X-API-KEY`/Bearer), or HTTP Basic; PBKDF2-SHA256 via passlib |
 | Avatars | Multiavatar (local SVG generation, no external API, <5ms) |
-| Validation | `hawk` (Python/JS/CSS/HTML) |
+| Coverage | `coverage.py` (`.coveragerc`, subprocess-aware) |
 | Load testing | Locust (locustfile.py) |
 
 ## Project structure
@@ -38,9 +37,10 @@ devplacepy/
   database.py         # dataset connection, index creation
   templating.py       # Shared Jinja2 environment + globals
   avatar.py           # Multiavatar generation, URL builder
-  utils.py            # Password hashing, session mgmt, time_ago
+  utils.py            # Password hashing, session mgmt, time_ago, notification hook
   models.py           # Pydantic schemas
-  routers/            # One file per domain (auth, feed, posts, ...)
+  push.py             # Web push crypto, VAPID keys, encrypt/send/register
+  routers/            # One file per domain (auth, feed, posts, push, ...)
   templates/          # Jinja2 HTML templates
   static/css/         # Page-specific CSS files
   static/js/          # Application.js (ES6 module)
@@ -57,16 +57,46 @@ devplacepy/
 | `/posts` | Post detail, creation |
 | `/comments` | Comment creation, deletion |
 | `/projects` | Project listing, creation |
+| `/projects/{slug}/files` | Per-project filesystem: directory and file CRUD, upload, inline editing (public read, owner write) |
+| `/zips` | Zip job status (`/zips/{uid}`) and archive download (`/zips/{uid}/download`); archives are queued via `/projects/{slug}/zip` and `/projects/{slug}/files/zip` |
 | `/profile` | Profile view, editing |
 | `/messages` | Direct messaging |
-| `/notifications` | Notification list, mark read |
+| `/notifications` | Notification list, mark read, live unread counts (`/notifications/counts`) |
 | `/votes` | Upvote/downvote on posts, comments, projects |
+| `/reactions` | Emoji reactions on posts, comments, gists, projects |
+| `/bookmarks` | Save/unsave content; `/bookmarks/saved` personal list |
+| `/polls` | Vote on post-attached polls |
 | `/follow` | Follow/unfollow users |
+| `/leaderboard` | Contributor ranking by total stars earned |
 | `/avatar` | Multiavatar proxy with in-memory cache |
 | `/bugs` | Bug reports listing, creation |
-| `/services` | Background service monitoring (status, logs) |
+| `/admin/services` | Background service management (start/stop, config, status, logs) |
 | `/admin` | Admin panel (user management, news curation, settings) |
+| `/docs` | Developer documentation site with a complete, interactive HTTP API reference |
+| `/openai` | OpenAI-compatible LLM gateway service (`/openai/v1/chat/completions`, `/openai/v1/*`) |
+| `/devii` | Devii agentic assistant: WebSocket terminal (`/devii/ws`), standalone page, usage (`/devii/usage`), session bootstrap |
 | `(none)` | `/robots.txt`, `/sitemap.xml` (SEO) |
+| `(none)` | `/push.json` (VAPID key + subscribe), `/service-worker.js`, `/manifest.json` (push + PWA) |
+
+## Gamification
+
+Member progression is driven by activity and peer recognition.
+
+- **Stars** are the net vote score (`upvotes - downvotes`) on a post, project, or gist. A member's total stars is the sum across all their content and is the basis for ranking.
+- **XP and levels.** Members earn XP for contributing: posting (10), commenting (2), publishing a project (15) or gist (5), receiving an upvote (5), and gaining a follower (5). Each level requires 100 XP (`level = 1 + xp // 100`). The profile shows the current level and progress to the next.
+- **Badges** are awarded once per milestone: first post/comment/project/gist, 10 posts (Prolific), 25 stars (Rising Star), 100 stars (Star Author), 10 followers (Popular), a 7-day activity streak (On Fire), and reaching levels 5 and 10. Badges render with an icon and description on the profile.
+- **Leaderboard** (`/leaderboard`) ranks the top 50 members by total stars (single page, no pagination); a member's own rank is shown on their profile.
+- **Contribution heatmap and streaks.** Each profile shows a 12-month activity heatmap and the current/longest daily streak, derived from post/comment/gist/project timestamps (no extra storage).
+- **Social graph listings.** Each profile has Followers and Following tabs that paginate the follow graph (25 per page) and show a follow/unfollow control for each person. The same data is available as JSON at `GET /profile/{username}/followers` and `GET /profile/{username}/following`.
+- **Reward notifications** fire when a member levels up or earns a badge.
+
+## Engagement
+
+- **Emoji reactions** - a fixed palette of reactions on posts, comments, gists, and projects, separate from voting and carrying no ranking weight.
+- **Polls** - a post can carry a poll (question plus up to six options); results appear as live bars once the viewer votes, one vote per member. A poll can be attached when the post is created or added later by editing a post that has none.
+- **Bookmarks** - save posts, gists, projects, and news to a personal list at `/bookmarks/saved`.
+
+XP awards are wired at the existing content-creation, vote, and follow hook points in the routers and centralized in `award_xp()` / `check_milestone_badges()` (`devplacepy/utils.py`). Existing accounts have their XP and levels backfilled once from prior activity at startup (`init_db()`).
 
 ## Configuration
 
@@ -74,41 +104,365 @@ devplacepy/
 |---------|---------|---------|
 | `DEVPLACE_DATABASE_URL` | `sqlite:///devplace.db` | Database connection string |
 | `SECRET_KEY` | hardcoded fallback | Session signing key |
+| `DEVPLACE_VAPID_SUB` | `mailto:retoor@molodetz.nl` | Contact address in the VAPID JWT `sub` claim |
+| `DEVPLACE_INTERNAL_BASE_URL` | `http://localhost:10500` | Base URL the platform's own services dial for the AI gateway |
+| `DEEPSEEK_API_KEY` / `OPENROUTER_API_KEY` | unset | Upstream provider keys; migrated into the gateway settings on first boot |
+
+### Runtime settings
+
+Operational behavior is tunable live from `/admin/settings` (stored in `site_settings`, no redeploy). The Operational group covers:
+
+| Setting | Default | Effect |
+|---------|---------|--------|
+| `rate_limit_per_minute` | `60` | Mutating requests allowed per IP per window |
+| `rate_limit_window_seconds` | `60` | Rolling window for the request limit |
+| `session_max_age_days` | `7` | Standard session cookie lifetime |
+| `session_remember_days` | `30` | Remember-me session cookie lifetime |
+| `registration_open` | `1` | When `0`, new sign-ups are rejected |
+| `maintenance_mode` | `0` | When `1`, non-admins see the maintenance page; admins retain access |
+| `maintenance_message` | scheduled-maintenance text | Message shown during maintenance |
+
+Numeric values are floored to safe minimums so an invalid entry cannot lock out writes or stall services. Consumers read via `get_setting`/`get_int_setting`, which fall back to these defaults when a row is absent.
+
+## Authentication
+
+The website uses a `session` cookie. For automation, every page and action also
+accepts three header-based methods, resolved centrally in `get_current_user`
+(`utils.py`) so they work everywhere with no per-route changes:
+
+- **API key** - `X-API-KEY: <key>`
+- **Bearer** - `Authorization: Bearer <key>`
+- **HTTP Basic** - `Authorization: Basic base64(username-or-email:password)`
+
+Every user has an `api_key` (a uuid7), set at signup and backfilled for existing
+users on startup (and via `devplace apikey backfill`). The key is shown on the
+profile page to its owner (and to admins for any user) with a live **Regenerate**
+button; regenerating invalidates the old key immediately. Invalid credentials
+return `401`; requests with no credentials behave as anonymous (browser redirect to
+login). Full details and copy-paste `curl` examples live at `/docs/authentication.html`.
+
+```bash
+curl -H "X-API-KEY: <your-key>" https://your-host/notifications
+curl -u <username>:<password> -X POST https://your-host/follow/<username>
+```
+
+CLI: `devplace apikey get <username>` / `reset <username>` / `backfill`.
+
+## Content negotiation (HTML or JSON)
+
+Every endpoint that renders a page or returns a redirect also speaks JSON, so anything the
+website does is automatable from the same URLs. A request gets JSON when it sends
+`Accept: application/json` or `Content-Type: application/json`; a normal browser navigation
+(`Accept: text/html`) always gets HTML, so existing behaviour is unchanged (the legacy
+`X-Requested-With: fetch` AJAX header still drives the four engagement endpoints only). JSON responses are defined by Pydantic models in `devplacepy/schemas.py` and built
+from the same context the templates use (sensitive user fields like `email`/`api_key`/
+`password_hash` are never exposed). Page GETs return the page payload; form actions return a
+uniform envelope `{ "ok": true, "redirect": "…", "data": {…} }`; errors return
+`{ "error": { "status", "message" } }` (validation → `422` with a `fields` map, unauthenticated
+JSON → `401`, non-admin → `403`). The core lives in `devplacepy/responses.py`
+(`wants_json`, `respond`, `action_result`). Full details: `/docs/conventions.html`.
+
+```bash
+curl -H "Accept: application/json" https://your-host/feed
+curl -H "Accept: application/json" -X POST -d "content=hi&title=T&topic=devlog" https://your-host/posts/create
+```
+
+## Documentation site
+
+`/docs` serves a server-rendered docs site with a feed-style left sidebar
+(`routers/docs.py`). `/docs/index.html` is the start page and `/docs/authentication.html`
+documents the auth methods; both are curated markdown pages rendered through the shared
+content renderer (markdown + highlight.js) with the current host and, when logged in, your
+own API key and username filled in.
+
+Every other endpoint is documented from a single source of truth, the `API_GROUPS` registry
+in `devplacepy/docs_api.py`. Each group becomes a reference page listing its endpoints with
+copy-paste cURL, JavaScript, and Python examples and an interactive "try it" panel
+(`static/js/ApiTester.js`) that executes the real call from the browser with your API key
+pre-filled. Each panel has a response-format selector (defaulting to JSON) that drives the
+`Accept` header for both the live request and the generated snippets, an always-visible
+**Expected** tab showing the modeled response, and a **Live response** tab for the real
+result. To document a new endpoint, add an entry to `docs_api.py`; the page, sidebar link,
+examples, runner, and expected-response sample are generated automatically. FastAPI's
+built-in Swagger is disabled so `/docs` belongs to this site.
+
+Operator pages are admin-only: `/docs/admin.html` (user/news/settings administration) and
+`/docs/services.html` (Background Services) are hidden from the sidebar and return 404 for
+non-admins. The Background Services page is generated live from the service registry
+(`build_services_group()` over `service_manager.describe_all()`), so every registered service
+and its full configuration are documented automatically - including future services.
 
 ## Background Services
 
-`devplacepy/services/` provides a generic async service framework. Services start on server boot, run at configurable intervals, and are gracefully cancelled on shutdown.
+`devplacepy/services/` provides a generic async service framework. `/admin/services` is a slim index listing each service with its description and live status; clicking a service opens its detail page where everything for that service lives - start/stop, enable-on-boot, run-now, clear logs, adjustable log size, and live status/metrics - organized into **Overview / Configuration / Logs** tabs (configuration grouped into sections). State is stored in the database (desired state in `site_settings`, observed state in the `service_state` table), so control and status are correct even with multiple workers where only one runs the services.
 
 ### Service framework
 
-- **`BaseService`** — abstract class with async run loop, `deque(maxlen=20)` log buffer, `name`, `interval_seconds`
-- **`ServiceManager`** — singleton that registers, starts, and stops all services
-- **`NewsService`** — fetches news from `news.app.molodetz.nl/api`, grades with AI, stores articles >= threshold
+- **`ConfigField`** - declarative parameter spec (type, default, validation, secret) a service uses to declare its editable settings
+- **`BaseService`** - abstract class with a reconciling run loop that honors the persisted `enabled`/command/interval state, plus `config_fields`, `get_config()`, `describe()`, and a log buffer
+- **`ServiceManager`** - singleton: `register`, `describe_all`, `set_enabled`, `send_command`, `save_config`, `supervise`, `shutdown_all`
+- **`NewsService`** - fetches news from `news.app.molodetz.nl/api`, grades with AI, stores articles >= threshold
+- **`BotsService`** - Playwright fleet of AI personas that browse and interact with a DevPlace instance, with live cost/usage metrics (opt-in; install the `bots` extra)
+- **`GatewayService`** - OpenAI-compatible LLM gateway at `/openai/v1/*`, forwarding to DeepSeek (default) with optional vision augmentation; the single point of truth for AI that every other service routes through (enabled by default)
+- **`JobService` / `ZipService`** - generic async job framework (`services/jobs/`) for heavy, blocking work run off the request path; `ZipService` is the first consumer, building project zip archives in a subprocess
+
+### Async job framework and zip downloads
+
+`services/jobs/` is the standard way to run blocking work asynchronously and hand the caller a result URL. A shared `jobs` table is the queue (discriminated by `kind`); `queue.enqueue()` inserts a `pending` row from any worker, the lock-owning worker processes jobs in `JobService.run_once` (reap, recover orphans, refill up to a concurrency limit, prune expired), and status is polled from the database. Retention is built in: each job service deletes its own expired artifacts via a `cleanup` hook (default 7 days, admin-configurable). To add a kind, subclass `JobService`, set `kind`, and implement `process()` and `cleanup()`.
+
+`ZipService` archives a project (or a file/folder subtree) into `{crc32}.{slug}.zip` using the standalone `zip_worker` subprocess, with a CRC32 over the output. The frontend `app.zipDownloader` (any `data-zip-download` element, plus the files context menu) enqueues, polls `/zips/{uid}`, and downloads from `/zips/{uid}/download`. The job uid is the capability token, so anyone with the link can download. CLI: `devplace zips prune` (delete expired) / `devplace zips clear` (delete all).
 
 ### Adding a service
 
-Create a class extending `BaseService`, override `run_once()`, register in `main.py`:
+Create a class extending `BaseService`, declare its `config_fields`, override `run_once()` (read parameters via `self.get_config()`), and register in `main.py`:
 ```python
 service_manager.register(YourService())
 ```
-The service appears at `/services` with live status and log tail.
+The service appears at `/admin/services` with controls, a generated config form, live status, and the log tail - no template, JS, or router changes.
 
 ### News service
 
 Fetches news from a configurable API, grades each article via AI, stores ALL articles in the `news` table (nothing is silently skipped). Articles with grade >= the configurable threshold are auto-published; the rest go to draft. Admin can manually publish/draft, toggle for landing page appearance, and delete. Images are extracted from article URLs.
 
-Configuration via admin site settings:
+Configuration on the Services tab (`/admin/services`):
 
-| Setting key | Default | Purpose |
-|-------------|---------|---------|
+| Parameter | Default | Purpose |
+|-----------|---------|---------|
 | `news_grade_threshold` | `7` | Minimum AI grade for auto-publish |
 | `news_api_url` | `https://news.app.molodetz.nl/api` | News source |
-| `news_ai_url` | `https://openai.app.molodetz.nl/v1/chat/completions` | AI grading endpoint |
-| `news_ai_model` | `molodetz` | AI model |
+| `news_ai_url` | `http://localhost:10500/openai/v1/chat/completions` | AI grading endpoint (the internal gateway) |
+| `news_ai_model` | `molodetz` | Generic model name; the gateway maps it to the real model |
+| `news_ai_key` | internal key | AI API key (`NEWS_AI_KEY` env, then the auto-generated gateway internal key) |
+| `news_service_interval` | `3600` | Seconds between fetch cycles (min 60) |
 
 News articles have detail pages at `/news/{slug}` with full comment support (same component as posts/projects). The landing page can display curated articles toggled from admin.
 
-CLI: `devplace news clear` — delete all news from local database.
+CLI: `devplace news clear` - delete all news from local database.
+
+### Bots service
+
+A Playwright-driven fleet of AI personas (`devplacepy/services/bot/`) that browse and interact with a DevPlace instance: posting, commenting, voting, reacting, creating gists/projects, filing bugs, following, and messaging. It is the former standalone `dpbot.py`, refactored into a package and managed entirely from the Services tab. Disabled by default.
+
+Install the extra and the browser binary, then enable it on `/admin/services`:
+
+```bash
+pip install -e ".[bots]"
+playwright install chromium
+```
+
+Configuration on the Services tab:
+
+| Parameter | Default | Purpose |
+|-----------|---------|---------|
+| `bot_fleet_size` | `1` | Number of concurrent bot accounts |
+| `bot_headless` | enabled | Run Chromium headless |
+| `bot_max_actions` | `0` | Stop a bot after N actions (0 = forever) |
+| `bot_base_url` | `https://pravda.education` | Target DevPlace instance |
+| `bot_api_url` | `http://localhost:10500/openai/v1/chat/completions` | LLM endpoint (the internal gateway) |
+| `bot_news_api` | `https://news.app.molodetz.nl/api` | Article source |
+| `bot_model` | `molodetz` | Generic model name; the gateway maps it to the real model |
+| `bot_api_key` | internal key | LLM key (defaults to the auto-generated gateway internal key) |
+| `bot_input_cost_per_1m` / `bot_output_cost_per_1m` | `0.27` / `1.10` | Token pricing for live cost tracking |
+
+The Services tab shows live fleet metrics (bots running, total cost, LLM calls, tokens, posts/comments/votes) and a per-bot breakdown that update on the page poll. Each bot keeps a stable identity and cumulative cost in `~/.devplace_bots/state_slot{N}.json`.
+
+### LLM gateway service
+
+`GatewayService` (`devplacepy/services/openai_gateway/`) exposes an OpenAI-compatible
+endpoint at `/openai/v1/chat/completions` (plus a transparent `/openai/v1/*`
+passthrough). It forwards to the configured upstream (DeepSeek by default) and, when a
+request carries image content, first describes the image(s) via a vision model
+(OpenRouter/Gemma by default) and inlines the description as text - so a vision-less
+upstream can still answer. Enabled by default and configurable on `/admin/services`.
+
+It is the **single point of truth for AI**: the news, bots, and guest Devii sessions call this
+gateway by default instead of an external provider, send the generic model name `molodetz`,
+and authenticate with an auto-generated internal key. The real provider URLs, models, and keys
+live only here, so providers and backends are switched in one place. `DEEPSEEK_API_KEY` and
+`OPENROUTER_API_KEY` are migrated into the editable settings on first boot, and the value in use
+is shown. The internal base URL the other services dial is `DEVPLACE_INTERNAL_BASE_URL`
+(default `http://localhost:10500`).
+
+Authenticated users may call the gateway with their own API key (the `Allow users` setting,
+on by default), and Devii operating a signed-in user authenticates its LLM calls with that
+user's key. Every call is therefore attributed to the user, so their full AI spend (through
+Devii or direct API calls) is tracked and limitable per user. Administrators see a per-user
+"AI usage (last 24h)" panel on each profile page - request volume, success and error rates,
+token totals, cost, average latency and throughput, a per-model breakdown, a 30-day cost
+projection, and a quota bar - served from `GET /admin/users/{uid}/ai-usage`. A non-admin user
+sees only one figure on their own profile: the percentage of their daily AI quota used so far.
+The same rule governs the Devii assistant: monetary figures (cost, pricing, spend, limit) are
+disclosed only to administrators, while members and guests can see only the percentage of their
+24-hour quota used. The `/devii/usage` endpoint returns that percentage and the day's turn count to
+everyone, and includes dollar figures only for administrators.
+
+Configuration on the Services tab:
+
+| Parameter | Default | Purpose |
+|-----------|---------|---------|
+| `gateway_upstream_url` | `https://api.deepseek.com/chat/completions` | Where requests are forwarded |
+| `gateway_model` | `deepseek-v4-flash` | Real model sent upstream (what `molodetz` maps to); 1M-token context, 384K max output |
+| `gateway_force_model` | on | Override the client-requested model (and the `molodetz` alias) |
+| `gateway_api_key` | migrated from env | Upstream key, shown and editable (auto-migrated from `DEEPSEEK_API_KEY`/`OPENROUTER_API_KEY`) |
+| `gateway_instances` | `4` | Max concurrent upstream forwards per worker (pool + semaphore) |
+| `gateway_timeout` | `300` | Upstream timeout (seconds); minimum five minutes; also bounds the vision describe-image call |
+| `gateway_vision_enabled` / `_url` / `_model` / `_key` / `_cache_size` | on / OpenRouter / Gemma / env / 256 | Image-description augmentation |
+| `gateway_require_auth` | on | When off, the gateway is open |
+| `gateway_allow_admins` / `gateway_allow_users` | on / off | Which DevPlace users may call it (any auth scheme) |
+| `gateway_access_key` | empty | A standalone key (sent as `X-API-KEY`/Bearer) that always grants access |
+| `gateway_internal_key` | auto (uuid4) | Auto-generated on boot; DevPlace's own services authenticate with this. Clear and restart to rotate |
+| `gateway_price_cache_hit_per_m` / `_cache_miss_per_m` / `_output_per_m` | 0.0028 / 0.14 / 0.28 | Chat cost per 1M tokens, used when the upstream returns no native cost (DeepSeek) |
+| `gateway_vision_price_input_per_m` / `_output_per_m` | 0 / 0 | Vision cost per 1M tokens, used only when the vision upstream returns no native cost |
+| `gateway_max_retries` / `gateway_retry_backoff_ms` | 2 / 250 | Retry attempts and linear backoff on timeout, connection error, or upstream 5xx |
+| `gateway_circuit_threshold` / `gateway_circuit_cooldown_seconds` | 5 / 30 | Consecutive failures before the circuit breaker opens, and its cooldown |
+| `gateway_usage_retention_hours` | 720 | How long per-call usage rows are kept before pruning (30 days) |
+| `gateway_model_context_map` | JSON map | Model to max-context-tokens map for context-window utilization tracking |
+
+Callers authenticate with the static `gateway_access_key`, the auto-generated
+`gateway_internal_key` (used by the platform's own services), or - when allowed - their
+DevPlace credentials via any scheme (`X-API-KEY`, Bearer, Basic). The endpoint is
+exempt from the per-IP rate limit and the maintenance gate, and its concurrency is
+bounded by `gateway_instances`; scale further with uvicorn workers. The Services tab
+shows live request/error/latency/vision-call counters.
+
+#### Usage and cost metrics
+
+Every upstream call (chat, vision, and passthrough) is recorded to `gateway_usage_ledger`
+with its tokens, cost, latency breakdown, status, and caller. Cost is taken from the
+upstream native `cost` field when present (OpenRouter) and computed from the configured
+per-million pricing otherwise (DeepSeek, which reports no cost). The admin **AI usage**
+page (`/admin/ai-usage`) reports per-hour and 24h metrics - request volume and throughput,
+token usage with averages and percentiles (p50/p90/p95/p99), latency (upstream round-trip,
+gateway overhead, semaphore queue wait, connection establishment), error rates by category,
+cost (per model, per caller, input vs output, projected monthly burn, caching savings),
+caller behavior, and an hourly breakdown - divided and combined. The gateway also retries
+transient upstream failures and opens a circuit breaker after repeated failures. Time to
+first token and inter-token latency are not reported because the gateway forwards
+non-streaming to the upstream.
+
+```bash
+curl -H "X-API-KEY: <gateway_access_key>" \
+  -d '{"messages":[{"role":"user","content":"hi"}]}' \
+  https://your-host/openai/v1/chat/completions
+```
+
+### Devii assistant service
+
+`DeviiService` (`devplacepy/services/devii/`) is an in-platform agentic assistant exposed
+as a WebSocket terminal at `/devii/ws`, reachable from the **Devii** item in the user
+dropdown menu and embedded on the documentation page for guests. It runs a ReAct loop
+(planning, reflection, a verification gate, self-learning memory, autonomous tasks) over a
+declarative tool catalog. A signed-in user's Devii operates their **own** DevPlace account -
+the agent authenticates to this instance with the user's `api_key` - while guests get a
+sandbox session. Disabled by default; enable and configure it on `/admin/services`.
+
+Sessions are durable. A logged-in user's conversation is keyed by user uid, shared live
+across every tab, window, and device (each turn is broadcast to all connected sockets), and
+persisted to `devii_conversations` so it survives a server restart - on reconnect the prior
+conversation is rehydrated. Guests get a non-persistent session keyed by a `devii_guest`
+cookie. To keep one logical session and one cost/broadcast source per user, the `/devii/ws`
+endpoint is served only by the worker that holds the background-service lock; other workers
+close the socket so the client reconnects to the owning worker.
+
+Spend is capped per owner over a rolling 24 hours. Every turn appends a row to
+`devii_usage_ledger` (the authoritative source for the cap - the in-memory cost tracker is
+display-only) and an audit row to `devii_turns`. The cap is checked before each turn.
+
+Configuration on the Services tab:
+
+| Parameter | Default | Purpose |
+|-----------|---------|---------|
+| `devii_ai_url` | `https://openai.app.molodetz.nl/v1/chat/completions` | OpenAI-compatible reasoning endpoint |
+| `devii_ai_model` | `molodetz` | Model name |
+| `devii_ai_key` | env fallback (`DEVII_AI_KEY`) | AI API key |
+| `devii_base_url` | this instance's origin | Platform Devii drives via each user's API key |
+| `devii_plan_required` / `devii_verify_required` | on / on | Enforce plan-first and verify-after-mutation |
+| `devii_max_iterations` | `40` | Tool-loop iterations per turn |
+| `devii_timeout` | `300` | Read timeout (seconds) for each AI chat-completions call; minimum five minutes |
+| `devii_fetch_timeout` | `300` | Read timeout (seconds) for the fetch tool; minimum five minutes |
+| `devii_user_daily_usd` | `1.0` | Rolling 24h spend cap per signed-in user |
+| `devii_guest_daily_usd` | `0.05` | Rolling 24h spend cap per guest |
+| `devii_admin_daily_usd` | `0.0` | Rolling 24h spend cap per administrator (`0` = unlimited; admins are exempt by default) |
+| `devii_guests_enabled` | on | Allow anonymous guests |
+| `devii_price_cache_hit` / `_cache_miss` / `_output` | `0.0028` / `0.14` / `0.28` | USD per 1M tokens (drives the cap) |
+| `devii_rsearch_enabled` | on | Enable the external web search tools (`rsearch_*`) |
+| `devii_rsearch_url` | `https://rsearch.app.molodetz.nl` | Base URL of the web search service those tools call |
+| `devii_rsearch_timeout` | `300` | Read timeout (seconds) for `rsearch_*` calls; web-grounded answers can take minutes; minimum five minutes |
+
+Beyond the platform tools, Devii has external **web search** tools - `rsearch` (web/image
+search), `rsearch_answer` (a web-grounded AI answer with sources), `rsearch_chat` (direct AI
+chat), and `rsearch_describe_image` (vision). These reach an external public service rather
+than this platform, so platform tools are always preferred; Devii uses them only when the user
+explicitly asks to search the web or an outside source. They are gated by `devii_rsearch_enabled`
+and the service URL is configurable via `devii_rsearch_url`.
+
+A `devii` console script ships the same agent as an interactive terminal:
+
+```bash
+pip install -e .
+devii --api-key <your DevPlace api_key> --base-url https://your-host
+devii -p "List my unread notifications as a bullet list."   # one-shot
+```
+
+## Push notifications & PWA
+
+Authenticated users can receive native web push notifications, and the site is an
+installable Progressive Web App. Push uses only standard libraries (`cryptography`,
+`PyJWT`, `httpx`) against the Web Push Protocol - no third-party push wrapper.
+
+### Events
+
+Every event that already produces an in-app notification also sends a web push,
+because both share a single funnel - `create_notification()` in `utils.py`:
+
+| Event | Recipient |
+|-------|-----------|
+| Direct message received | receiver |
+| Comment on your post | post author |
+| Reply to your comment | comment author |
+| `@mention` in any content | mentioned user |
+| Upvote on your content | content owner |
+| New follower | followed user |
+
+`create_notification` schedules delivery as a fire-and-forget async task, so a dead
+subscription or push-service error never blocks the triggering request. Delivery
+(`push.notify_user`) iterates a user's subscriptions, encrypts the payload
+(legacy `aesgcm` content encoding), and POSTs to each endpoint; subscriptions that
+return `404`/`410` are soft-deleted.
+
+### VAPID keys
+
+The server identity is three PEM files generated once at startup in the repository
+root: `notification-private.pem`, `notification-private.pkcs8.pem`,
+`notification-public.pem`. They are git-ignored.
+
+**These keys are the application's identity to the push services. If they are lost or
+regenerated, every existing subscription becomes permanently undeliverable.** Persist
+them across deployments and back them up; do not regenerate them.
+
+### Opt-in
+
+The browser requires the first permission prompt to originate from a user gesture, so
+opt-in is exposed as a button in both the top navigation (bell-with-slash icon) and on
+the `/notifications` page. After opt-in, the subscription is refreshed silently on
+every page load. `PushManager.js` owns registration, subscription, and the opt-in UI.
+
+### PWA
+
+`manifest.json` (192/512 and maskable icons), `service-worker.js`, and an install
+button (`PwaInstaller.js`) make the app installable. The service worker uses a
+network-first strategy for navigations and falls back to `static/offline.html` when
+offline. Installation requires a secure origin (HTTPS, or `localhost` for development).
+
+| File | Role |
+|------|------|
+| `devplacepy/push.py` | VAPID keys, payload encryption, send, register |
+| `devplacepy/routers/push.py` | `/push.json`, `/service-worker.js`, `/manifest.json` |
+| `static/js/PushManager.js` | Service-worker registration + subscribe + opt-in UI |
+| `static/js/PwaInstaller.js` | `beforeinstallprompt` capture + install button |
+| `static/service-worker.js` | Receives push, shows notification, offline fallback |
+| `static/manifest.json` | PWA manifest (icons, display, theme) |
+| `static/offline.html` | Offline fallback page |
 
 ## Database
 
@@ -123,12 +477,14 @@ PRAGMA temp_store=MEMORY;      -- temp tables in memory
 PRAGMA mmap_size=268435456;    -- 256MB memory map for reads
 ```
 
-All indexes are created via `CREATE INDEX IF NOT EXISTS` wrapped in try/except — safe to run on every startup regardless of table state.
+All indexes are created via `CREATE INDEX IF NOT EXISTS` wrapped in try/except - safe to run on every startup regardless of table state.
+
+SQLite is synchronous by design and will never be made async. It is more than fast enough for this platform: the database is a local file with WAL, a 30s busy timeout, and a 256MB memory map, so queries are sub-millisecond. The synchronous database layer is a deliberate architectural decision, not a limitation, and is not subject to change.
 
 ## Testing
 
-- **148 tests** across 14 files: Playwright integration + unit tests
-- Playwright (NOT pytest-playwright plugin — conflicts, uninstall it)
+- **419 tests** across 34 files: Playwright integration + in-process unit tests
+- Playwright (NOT pytest-playwright plugin - conflicts, uninstall it)
 - Server starts as subprocess on port 10501 with isolated temp database
 - Test users `alice_test` / `bob_test` seeded via HTTP at session start
 - Tests stop at first failure (`-x` flag)
@@ -137,34 +493,81 @@ All indexes are created via `CREATE INDEX IF NOT EXISTS` wrapped in try/except
 
 ### Key test patterns
 
-- Every `page.goto()` and `page.wait_for_url()` uses `wait_until="domcontentloaded"` — avatar images don't block test execution
+- Every `page.goto()` and `page.wait_for_url()` uses `wait_until="domcontentloaded"` - avatar images don't block test execution
 - Session-scoped browser context with per-test cookie clearing
 - `page.locator(...).wait_for(state="visible")` preferred over bare selectors
 
 ## Avatars
 
-Uses [Multiavatar](https://github.com/multiavatar/multiavatar-python) — generates deterministic SVG avatars locally from a seed string (the username). No external API calls, no network dependency. Generation takes <5ms. Results are cached in-memory (cleared on server restart).
+Uses [Multiavatar](https://github.com/multiavatar/multiavatar-python) - generates deterministic SVG avatars locally from a seed string (the username). No external API calls, no network dependency. Generation takes <5ms. Results are cached in-memory (cleared on server restart).
 
 Avatar URL format: `/avatar/multiavatar/{username}?size={size}`
 
+## Deployment
+
+Production runs on a single host via Docker Compose: an **app** container (Uvicorn, 2 workers) behind an **nginx** container that terminates HTTP, serves static assets, and proxies the rest.
+
+### Shared database and uploads
+
+The app container bind-mounts the host project directory (`./` to `/app`) and runs as the host user (`DEVPLACE_UID`/`DEVPLACE_GID`, default `1000`). Because `config.py` resolves the database to an absolute path under the project root, the container reads and writes the **same `devplace.db`** as `make dev`, plus the same `static/uploads/`, `devii_lessons.db`, `devii_tasks.db`, VAPID keys, and `devplace-services.lock`. SQLite WAL mode allows the concurrent access, and the file lock (`fcntl.flock` on `devplace-services.lock`) guarantees only one process - dev or prod - runs the background services (news, bots, Devii hub), so they never double-fire.
+
+This requires prod and dev to be on the **same host/filesystem**; SQLite is a local-file database and cannot be shared across machines. No `DEVPLACE_DATABASE_URL` override is set, so nothing diverges.
+
+### First deploy
+
+```bash
+cp .env.example .env        # set SECRET_KEY; adjust PORT, DEVPLACE_SITE_URL
+make docker-up              # docker compose up -d (builds on first run)
+```
+
+Open `http://<host>:${PORT}` (default 10500). `make docker-logs` tails output; `make docker-down` stops.
+
+### Updating
+
+```bash
+git pull
+docker compose up -d            # restart with new code (bind-mounted, no rebuild)
+docker compose build && \
+  docker compose up -d          # only when dependencies in pyproject.toml change
+```
+
+The `make deploy` target fast-forwards the `production` branch (`git checkout production && git merge master && git push origin production`); pull that branch on the server.
+
+### nginx specifics
+
+- **WebSockets:** `/devii/ws` (the Devii terminal) is proxied with `Upgrade`/`Connection` headers and a 1h read timeout.
+- **Uploads:** `/static/uploads/` is served with `X-Content-Type-Options: nosniff` and a `Content-Disposition` that mirrors the app's `UploadStaticFiles` control: known-safe media (images, video, audio) is served `inline` so it embeds and plays in the page, while every other type is forced to `attachment` so it downloads instead of rendering (stored-XSS defense). SVG is deliberately excluded from the inline set.
+- **Upload size:** `NGINX_MAX_BODY_SIZE` (default `50m`) must be **>= the admin-configurable `max_upload_size_mb`** (Admin -> Settings), or large uploads are rejected with HTTP 413 before reaching the app.
+- **Micro-cache:** off by default; enable with `NGINX_CACHE_ENABLED=true`.
+
+### Bare-metal alternative
+
+`make prod` runs the same app without containers (`uvicorn ... --workers 2 --proxy-headers`) from the project root, sharing the identical database and files. Note it binds port 10500, so it conflicts with the Docker front door on the same port - run one, or set a different `PORT`.
+
+### Multi-worker safety
+
+The app runs correctly under multiple Uvicorn workers (independent processes sharing only the DB and lock files). Cross-worker invalidation of the auth and settings caches is coordinated through a `cache_state` version table (propagation bounded to ~2s); the rate limiter enforces `ceil(limit / DEVPLACE_WEB_WORKERS)` per process so the aggregate honours the configured value - **set `DEVPLACE_WEB_WORKERS` to match `--workers`** (done in `make prod` and the Dockerfile). First-boot DB init and VAPID key generation are `flock`-serialized. Full detail and the design rules are in the admin docs at `/docs/production-concurrency.html`.
+
 ## CI/CD
 
 Gitea Actions workflow at `.gitea/workflows/test.yaml`:
-- Runs on push/PR to main
-- Sets up Python 3.13, installs dependencies + Playwright
-- Validates all source files with `hawk`
-- Runs all tests with fail-fast
-- Uploads failure screenshots as artifacts
+- Runs on push/PR to `master`
+- Sets up Python 3.13, installs dependencies + Playwright Chromium
+- Runs the full test suite under coverage (`coverage run -m pytest -p no:xdist`)
+- Builds and publishes the coverage HTML as an artifact on every run
+- Uploads failure screenshots as artifacts when a test fails
+
+Changes are promoted through automated DTAP streets: Development (`make dev`), Test (the CI suite + coverage on `master`), Acceptance (the `master` to `production` promotion via `make deploy`), and Production (the Docker Compose stack on the host). Only CI-green `master` commits reach `production`.
 
 ## Feature workflow
 
 1. Implement the feature (router + template + CSS + JS)
-2. `hawk .` — validate all source files
-3. `make test` — run all tests (fail-fast)
+2. Validate each touched language (Python compiles/imports, JS parses, CSS and HTML balance)
+3. `make test` - run all tests (fail-fast)
 4. Add Playwright tests in `tests/test_*.py` for new functionality
 5. For visual features: `falcon take --output /tmp/verify.png && falcon describe /tmp/verify.png`
 6. Update `AGENTS.md` and `README.md` if new conventions were introduced
 
 ## License
 
-MIT — DevPlace
+MIT - DevPlace

devplacepy/attachments.py

@@ -1,33 +1,113 @@
 import logging
-from datetime import datetime
+from datetime import datetime, timezone
 from pathlib import Path
 from PIL import Image
 from io import BytesIO
-from devplacepy.database import get_table
+from devplacepy.database import get_table, db, get_setting
 from devplacepy.config import STATIC_DIR
 from devplacepy.utils import generate_uid
 
 logger = logging.getLogger(__name__)
-ATTACHMENTS_DIR = STATIC_DIR / "uploads" / "attachments"
+
+UPLOADS_DIR = STATIC_DIR / "uploads"
+ATTACHMENTS_DIR = UPLOADS_DIR / "attachments"
 THUMBNAIL_SIZE = (200, 200)
 THUMBNAIL_QUALITY = 80
 IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".tiff"}
 THUMBNAIL_EXTENSIONS = IMAGE_EXTENSIONS - {".gif"}
+POST_IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".webp"}
+
+ALLOWED_UPLOAD_TYPES = {
+    ".jpg": "image/jpeg",
+    ".jpeg": "image/jpeg",
+    ".png": "image/png",
+    ".gif": "image/gif",
+    ".webp": "image/webp",
+    ".bmp": "image/bmp",
+    ".tiff": "image/tiff",
+    ".pdf": "application/pdf",
+    ".zip": "application/zip",
+    ".mp4": "video/mp4",
+    ".webm": "video/webm",
+    ".ogv": "video/ogg",
+    ".mov": "video/quicktime",
+    ".m4v": "video/x-m4v",
+    ".mp3": "audio/mpeg",
+    ".txt": "text/plain",
+    ".py": "text/x-python",
+    ".js": "text/javascript",
+    ".css": "text/css",
+    ".md": "text/markdown",
+}
+
+FILE_ICONS = {
+    ".pdf": "\U0001F4C4",
+    ".zip": "\U0001F4E6",
+    ".gz": "\U0001F4E6",
+    ".tar": "\U0001F4E6",
+    ".rar": "\U0001F4E6",
+    ".7z": "\U0001F4E6",
+    ".mp4": "\U0001F3AC",
+    ".webm": "\U0001F3AC",
+    ".ogv": "\U0001F3AC",
+    ".mov": "\U0001F3AC",
+    ".m4v": "\U0001F3AC",
+    ".mp3": "\U0001F3B5",
+    ".py": "\U0001F4BB",
+    ".js": "\U0001F4BB",
+    ".ts": "\U0001F4BB",
+    ".html": "\U0001F4BB",
+    ".css": "\U0001F4BB",
+    ".json": "\U0001F4BB",
+    ".md": "\U0001F4BB",
+    ".csv": "\U0001F4CA",
+    ".xls": "\U0001F4CA",
+    ".xlsx": "\U0001F4CA",
+    ".doc": "\U0001F4DD",
+    ".docx": "\U0001F4DD",
+    ".txt": "\U0001F4C4",
+    ".exe": "\u2699",
+    ".bin": "\u2699",
+}
+DEFAULT_FILE_ICON = "\U0001F4CE"
 
-def _get_setting(key, default):
-    row = get_table("site_settings").find_one(key=key)
-    return row["value"] if row else default
 
 def _get_max_upload_bytes():
-    return int(_get_setting("max_upload_size_mb", "10")) * 1024 * 1024
+    return int(get_setting("max_upload_size_mb", "10")) * 1024 * 1024
+
+
+def allowed_extensions():
+    raw = get_setting("allowed_file_types", "").strip()
+    if raw:
+        return {
+            ext if ext.startswith(".") else f".{ext}"
+            for ext in (part.strip().lower() for part in raw.split(","))
+            if ext
+        }
+    return set(ALLOWED_UPLOAD_TYPES)
+
+
+def is_extension_allowed(ext):
+    return ext in allowed_extensions()
+
 
 def _directory_for(uid):
     return f"{uid[:2]}/{uid[2:4]}"
 
+
 def _detect_mime(file_bytes, original_filename):
     ext = Path(original_filename).suffix.lower()
-    m = {".jpg":"image/jpeg",".jpeg":"image/jpeg",".png":"image/png",".gif":"image/gif",".webp":"image/webp",".svg":"image/svg+xml",".bmp":"image/bmp",".tiff":"image/tiff",".pdf":"application/pdf",".zip":"application/zip",".mp4":"video/mp4",".mp3":"audio/mpeg",".txt":"text/plain",".py":"text/x-python",".js":"text/javascript",".html":"text/html",".css":"text/css",".md":"text/markdown"}
-    return m.get(ext, "application/octet-stream")
+    return ALLOWED_UPLOAD_TYPES.get(ext, "application/octet-stream")
+
+
+def _image_dimensions(file_bytes):
+    try:
+        img = Image.open(BytesIO(file_bytes))
+        return img.width, img.height
+    except Exception as e:
+        logger.warning(f"Could not read image dimensions: {e}")
+        return None, None
+
 
 def _generate_thumbnail(file_bytes, thumb_path):
     try:
@@ -45,95 +125,202 @@ def _generate_thumbnail(file_bytes, thumb_path):
         logger.warning(f"Thumbnail generation failed: {e}")
         return None
 
+
+def save_inline_image(file_bytes, original_filename):
+    if len(file_bytes) > _get_max_upload_bytes():
+        logger.warning(f"Inline image too large: {original_filename}")
+        return None
+    ext = Path(original_filename).suffix.lower()
+    if ext not in POST_IMAGE_EXTENSIONS:
+        logger.warning(f"Unsupported inline image type: {ext}")
+        return None
+    UPLOADS_DIR.mkdir(parents=True, exist_ok=True)
+    filename = f"{generate_uid()}{ext}"
+    (UPLOADS_DIR / filename).write_bytes(file_bytes)
+    logger.info(f"Inline image saved: {filename}")
+    return filename
+
+
+def delete_inline_image(filename):
+    if not filename:
+        return
+    try:
+        (UPLOADS_DIR / filename).unlink(missing_ok=True)
+    except Exception as e:
+        logger.warning(f"Failed to delete inline image {filename}: {e}")
+
+
 def store_attachment(file_bytes, original_filename, user_uid):
     if len(file_bytes) > _get_max_upload_bytes():
         return None
+    ext = Path(original_filename).suffix.lower()
+    if not is_extension_allowed(ext):
+        return None
+
     uid = generate_uid()
-    ext = Path(original_filename).suffix.lower() or ".bin"
     stored_name = f"{uid}{ext}"
     directory = _directory_for(uid)
     file_dir = ATTACHMENTS_DIR / directory
     file_dir.mkdir(parents=True, exist_ok=True)
     (file_dir / stored_name).write_bytes(file_bytes)
+
     mime = _detect_mime(file_bytes, original_filename)
-    is_img = mime.startswith("image/")
-    img_w, img_h = None, None
-    thumb = None
-    if is_img:
-        try:
-            img = Image.open(BytesIO(file_bytes))
-            img_w, img_h = img.width, img.height
-            if ext not in (".gif",):
-                thumb_name = f"{uid}_thumb.jpg"
-                r = _generate_thumbnail(file_bytes, file_dir / thumb_name)
-                if r: thumb = r
-        except: pass
+    is_image = mime.startswith("image/")
+    image_width, image_height = None, None
+    thumbnail = None
+    if is_image:
+        image_width, image_height = _image_dimensions(file_bytes)
+        if ext not in (".gif",):
+            thumbnail = _generate_thumbnail(file_bytes, file_dir / f"{uid}_thumb.jpg")
+
     get_table("attachments").insert({
-        "uid": uid, "target_type": "", "target_uid": "", "user_uid": user_uid,
-        "original_filename": original_filename, "stored_name": stored_name,
-        "directory": directory, "file_size": len(file_bytes), "mime_type": mime,
-        "image_width": img_w, "image_height": img_h,
-        "has_thumbnail": 1 if thumb else 0,
-        "created_at": datetime.utcnow().isoformat(),
+        "uid": uid,
+        "target_type": "",
+        "target_uid": "",
+        "user_uid": user_uid,
+        "original_filename": original_filename,
+        "stored_name": stored_name,
+        "directory": directory,
+        "file_size": len(file_bytes),
+        "mime_type": mime,
+        "image_width": image_width,
+        "image_height": image_height,
+        "has_thumbnail": 1 if thumbnail else 0,
+        "thumbnail_name": thumbnail,
+        "created_at": datetime.now(timezone.utc).isoformat(),
     })
-    return {"uid": uid, "original_filename": original_filename, "file_size": len(file_bytes), "mime_type": mime, "url": f"/static/uploads/attachments/{directory}/{stored_name}", "thumbnail_url": f"/static/uploads/attachments/{directory}/{thumb}" if thumb else None, "has_thumbnail": thumb is not None, "is_image": is_img}
+    return {
+        "uid": uid,
+        "original_filename": original_filename,
+        "file_size": len(file_bytes),
+        "mime_type": mime,
+        "url": f"/static/uploads/attachments/{directory}/{stored_name}",
+        "thumbnail_url": f"/static/uploads/attachments/{directory}/{thumbnail}" if thumbnail else None,
+        "has_thumbnail": thumbnail is not None,
+        "is_image": is_image,
+        "is_video": mime.startswith("video/"),
+    }
+
 
 def link_attachments(uids, target_type, target_uid):
-    if not uids: return
-    atts = get_table("attachments")
-    for auid in uids:
-        auid = auid.strip()
-        if not auid: continue
-        e = atts.find_one(uid=auid)
-        if e: atts.update({"id": e["id"], "uid": auid, "target_type": target_type, "target_uid": target_uid}, ["id"])
+    flat = [uid.strip() for raw in uids or [] for uid in str(raw).split(",") if uid.strip()]
+    if not flat:
+        return
+    placeholders = ",".join(f":p{i}" for i in range(len(flat)))
+    params = {f"p{i}": uid for i, uid in enumerate(flat)}
+    db.query(
+        f"UPDATE attachments SET target_type=:tt, target_uid=:tu WHERE uid IN ({placeholders})",
+        tt=target_type, tu=target_uid, **params,
+    )
+
+
+def _unlink_attachment_files(row):
+    stored_name = row.get("stored_name", "")
+    directory = row.get("directory", "")
+    if not (stored_name and directory):
+        return
+    file_path = ATTACHMENTS_DIR / directory / stored_name
+    try:
+        file_path.unlink(missing_ok=True)
+    except Exception as e:
+        logger.warning(f"Failed to delete attachment file {file_path}: {e}")
+    for thumb_path in (ATTACHMENTS_DIR / directory).glob(f"{Path(stored_name).stem}_thumb.*"):
+        try:
+            thumb_path.unlink(missing_ok=True)
+        except Exception as e:
+            logger.warning(f"Failed to delete thumbnail {thumb_path}: {e}")
+
+
+def _delete_attachment_row(row):
+    _unlink_attachment_files(row)
+    get_table("attachments").delete(id=row["id"])
+
 
 def delete_attachment(uid):
-    atts = get_table("attachments")
-    att = atts.find_one(uid=uid)
-    if not att: return
-    sn, d = att.get("stored_name",""), att.get("directory","")
-    if sn and d:
-        fp = ATTACHMENTS_DIR / d / sn
-        try: fp.unlink(missing_ok=True)
-        except: pass
-        for tp in (ATTACHMENTS_DIR / d).glob(f"{Path(sn).stem}_thumb.*"):
-            try: tp.unlink(missing_ok=True)
-            except: pass
-    atts.delete(id=att["id"])
+    row = get_table("attachments").find_one(uid=uid)
+    if row:
+        _delete_attachment_row(row)
 
-def delete_target_attachments(tt, tu):
-    for a in get_table("attachments").find(target_type=tt, target_uid=tu):
-        delete_attachment(a["uid"])
 
-def get_attachments(tt, tu):
-    try: rows = list(get_table("attachments").find(target_type=tt, target_uid=tu, order_by=["created_at"]))
-    except: return []
-    return [_r2a(r) for r in rows]
+def delete_target_attachments(target_type, target_uid):
+    for row in get_table("attachments").find(target_type=target_type, target_uid=target_uid):
+        _delete_attachment_row(row)
 
-def get_attachments_batch(tt, uids):
-    if not uids: return {}
-    from devplacepy.database import db
-    if "attachments" not in db.tables: return {u:[] for u in uids}
-    ph = ",".join(f":p{i}" for i in range(len(uids)))
-    try:
-        rows = db.query(f"SELECT * FROM attachments WHERE target_type=:tt AND target_uid IN ({ph}) ORDER BY created_at", tt=tt, **{f"p{i}":u for i,u in enumerate(uids)})
-    except: return {u:[] for u in uids}
-    r = {u:[] for u in uids}
+
+def delete_attachments_for(target_type, target_uids):
+    uids = [uid for uid in target_uids if uid]
+    if not uids or "attachments" not in db.tables:
+        return
+    placeholders = ",".join(f":p{i}" for i in range(len(uids)))
+    params = {f"p{i}": uid for i, uid in enumerate(uids)}
+    rows = list(db.query(
+        f"SELECT * FROM attachments WHERE target_type=:tt AND target_uid IN ({placeholders})",
+        tt=target_type, **params,
+    ))
+    if not rows:
+        return
     for row in rows:
-        if row["target_uid"] in r: r[row["target_uid"]].append(_r2a(row))
-    return r
+        _unlink_attachment_files(row)
+    ids = ",".join(str(row["id"]) for row in rows)
+    db.query(f"DELETE FROM attachments WHERE id IN ({ids})")
 
-def _r2a(r):
-    sn, d = r.get("stored_name",""), r.get("directory","")
-    ts = f"{Path(sn).stem}_thumb.jpg" if r.get("has_thumbnail") else None
-    return {"uid":r["uid"],"original_filename":r.get("original_filename",""),"file_size":r.get("file_size",0),"mime_type":r.get("mime_type",""),"url":f"/static/uploads/attachments/{d}/{sn}","thumbnail_url":f"/static/uploads/attachments/{d}/{ts}" if ts else None,"has_thumbnail":bool(r.get("has_thumbnail")),"is_image":r.get("mime_type","").startswith("image/")}
 
-def format_file_size(b):
-    if b < 1024: return f"{b} B"
-    if b < 1048576: return f"{b/1024:.1f} KB"
-    return f"{b/1048576:.1f} MB"
+def get_attachments(target_type, target_uid):
+    if "attachments" not in db.tables:
+        return []
+    rows = list(get_table("attachments").find(target_type=target_type, target_uid=target_uid, order_by=["created_at"]))
+    return [_row_to_attachment(r) for r in rows]
 
-def file_icon_emoji(fn):
-    ext = Path(fn).suffix.lower()
-    m = {".pdf":"\U0001F4C4",".zip":"\U0001F4E6",".gz":"\U0001F4E6",".tar":"\U0001F4E6",".rar":"\U0001F4E6",".7z":"\U0001F4E6",".mp4":"\U0001F3AC",".mp3":"\U0001F3B5",".py":"\U0001F4BB",".js":"\U0001F4BB",".ts":"\U0001F4BB",".html":"\U0001F4BB",".css":"\U0001F4BB",".json":"\U0001F4BB",".md":"\U0001F4BB",".csv":"\U0001F4CA",".xls":"\U0001F4CA",".xlsx":"\U0001F4CA",".doc":"\U0001F4DD",".docx":"\U0001F4DD",".txt":"\U0001F4C4",".exe":"\u2699",".bin":"\u2699"}
-    return m.get(ext, "\U0001F4CE")
+
+def get_attachments_batch(target_type, uids):
+    if not uids:
+        return {}
+    if "attachments" not in db.tables:
+        return {uid: [] for uid in uids}
+    placeholders = ",".join(f":p{i}" for i in range(len(uids)))
+    params = {f"p{i}": uid for i, uid in enumerate(uids)}
+    rows = db.query(
+        f"SELECT * FROM attachments WHERE target_type=:tt AND target_uid IN ({placeholders}) ORDER BY created_at",
+        tt=target_type, **params,
+    )
+    result = {uid: [] for uid in uids}
+    for row in rows:
+        if row["target_uid"] in result:
+            result[row["target_uid"]].append(_row_to_attachment(row))
+    return result
+
+
+def _row_to_attachment(row):
+    stored_name = row.get("stored_name", "")
+    directory = row.get("directory", "")
+    thumb_name = None
+    if row.get("has_thumbnail"):
+        thumb_name = row.get("thumbnail_name")
+        if not thumb_name:
+            stem = Path(stored_name).stem
+            png = f"{stem}_thumb.png"
+            thumb_name = png if (ATTACHMENTS_DIR / directory / png).exists() else f"{stem}_thumb.jpg"
+    return {
+        "uid": row["uid"],
+        "original_filename": row.get("original_filename", ""),
+        "file_size": row.get("file_size", 0),
+        "mime_type": row.get("mime_type", ""),
+        "url": f"/static/uploads/attachments/{directory}/{stored_name}",
+        "thumbnail_url": f"/static/uploads/attachments/{directory}/{thumb_name}" if thumb_name else None,
+        "has_thumbnail": bool(row.get("has_thumbnail")),
+        "is_image": row.get("mime_type", "").startswith("image/"),
+        "is_video": row.get("mime_type", "").startswith("video/"),
+    }
+
+
+def format_file_size(size):
+    if size < 1024:
+        return f"{size} B"
+    if size < 1048576:
+        return f"{size / 1024:.1f} KB"
+    return f"{size / 1048576:.1f} MB"
+
+
+def file_icon_emoji(filename):
+    ext = Path(filename).suffix.lower()
+    return FILE_ICONS.get(ext, DEFAULT_FILE_ICON)

devplacepy/avatar.py

@@ -9,11 +9,12 @@ def avatar_url(style: str, seed: str, size: int = 128) -> str:
 
 def generate_avatar_svg(seed: str) -> str:
     try:
-        from multiavatar import multiavatar
-        svg = multiavatar(seed)
+        from multiavatar.multiavatar import multiavatar
+        svg = multiavatar(seed, None, None)
         if svg and svg.strip().startswith("<svg"):
             return svg
     except Exception as e:
+        logger.exception(e)
         logger.warning(f"Avatar generation failed for {seed}: {e}")
     initial = seed[:1].upper() if seed else "?"
     return (

devplacepy/cache.py

@@ -0,0 +1,36 @@
+import time
+from collections import OrderedDict
+
+
+class TTLCache:
+    def __init__(self, ttl: int, max_size: int = 0):
+        self.ttl = ttl
+        self.max_size = max_size
+        self._store = OrderedDict()
+
+    def get(self, key):
+        entry = self._store.get(key)
+        if entry is None:
+            return None
+        value, expiry = entry
+        if time.time() >= expiry:
+            self._store.pop(key, None)
+            return None
+        self._store.move_to_end(key)
+        return value
+
+    def set(self, key, value):
+        self._store[key] = (value, time.time() + self.ttl)
+        self._store.move_to_end(key)
+        if self.max_size and len(self._store) > self.max_size:
+            self._store.popitem(last=False)
+
+    def pop(self, key):
+        self._store.pop(key, None)
+
+    def clear(self):
+        self._store.clear()
+
+    def items(self):
+        now = time.time()
+        return [(key, value) for key, (value, expiry) in self._store.items() if now < expiry]

devplacepy/cli.py

@@ -1,6 +1,7 @@
 import argparse
 import sys
 from devplacepy.database import get_table
+from devplacepy.utils import strip_html
 
 
 def cmd_role_get(args):
@@ -28,11 +29,68 @@ def cmd_role_set(args):
     print(f"User '{args.username}' role set to '{role}'")
 
 
+def cmd_apikey_get(args):
+    users = get_table("users")
+    user = users.find_one(username=args.username)
+    if not user:
+        print(f"User '{args.username}' not found")
+        sys.exit(1)
+    print(user.get("api_key", "") or "")
+
+
+def cmd_apikey_reset(args):
+    from devplacepy.utils import generate_uid, clear_user_cache
+    users = get_table("users")
+    user = users.find_one(username=args.username)
+    if not user:
+        print(f"User '{args.username}' not found")
+        sys.exit(1)
+    new_key = generate_uid()
+    users.update({"uid": user["uid"], "api_key": new_key}, ["uid"])
+    clear_user_cache(user["uid"])
+    print(new_key)
+
+
+def cmd_apikey_backfill(args):
+    from devplacepy.database import backfill_api_keys
+    updated = backfill_api_keys()
+    print(f"Assigned API keys to {updated} user(s) without one")
+
+
+def cmd_devii_reset_quota(args):
+    from devplacepy.database import db
+    table_name = "devii_usage_ledger"
+    if table_name not in db.tables:
+        print(f"Table '{table_name}' does not exist, nothing to reset")
+        return
+    table = db[table_name]
+    if args.all:
+        count = table.count()
+        table.delete()
+        print(f"Reset all AI quotas ({count} ledger rows deleted)")
+        return
+    if args.guests:
+        count = table.count(owner_kind="guest")
+        table.delete(owner_kind="guest")
+        print(f"Reset all guest AI quotas ({count} ledger rows deleted)")
+        return
+    if not args.username:
+        print("Provide a username, or --guests, or --all")
+        sys.exit(1)
+    user = get_table("users").find_one(username=args.username)
+    if not user:
+        print(f"User '{args.username}' not found")
+        sys.exit(1)
+    count = table.count(owner_kind="user", owner_id=user["uid"])
+    table.delete(owner_kind="user", owner_id=user["uid"])
+    print(f"Reset AI quota for '{args.username}' ({count} ledger rows deleted)")
+
+
 def cmd_news_clear(args):
     from devplacepy.database import db
     for table in ("news", "news_images", "news_sync"):
         if table in db.tables:
-            count = len(list(db[table].all()))
+            count = db[table].count()
             db[table].delete()
             print(f"Deleted {count} rows from '{table}'")
         else:
@@ -40,45 +98,78 @@ def cmd_news_clear(args):
     print("News data cleared")
 
 
-def cmd_attachments_prune(args):
+def cmd_news_sanitize(args):
     from devplacepy.database import db
-    from devplacepy.config import STATIC_DIR
-    import os
-    deleted_records = 0
-    deleted_files = 0
-    freed_bytes = 0
+    if "news" not in db.tables:
+        print("News table does not exist")
+        return
+    news_table = db["news"]
+    updated = 0
+    for row in news_table.all():
+        desc = (strip_html(row.get("description", "") or ""))[:5000]
+        content = (strip_html(row.get("content", "") or ""))[:10000]
+        if desc != row.get("description", "") or content != row.get("content", ""):
+            news_table.update({"id": row["id"], "description": desc, "content": content}, ["id"])
+            updated += 1
+    print(f"Sanitized {updated} news article(s)")
 
-    if "attachments" in db.tables:
-        orphans = list(db["attachments"].find(resource_uid=""))
-        orphans += list(db["attachments"].find(resource_type=""))
-        seen = set()
-        unique_orphans = []
-        for o in orphans:
-            if o["uid"] not in seen:
-                seen.add(o["uid"])
-                unique_orphans.append(o)
 
-        for att in unique_orphans:
-            sp = att.get("storage_path", "")
-            if sp:
-                fp = STATIC_DIR / "uploads" / sp
-                try:
-                    if fp.exists():
-                        freed_bytes += fp.stat().st_size
-                        fp.unlink()
-                        deleted_files += 1
-                    parent = fp.parent
-                    if parent.exists() and not any(parent.iterdir()):
-                        parent.rmdir()
-                    grandparent = parent.parent
-                    if grandparent.exists() and not any(grandparent.iterdir()):
-                        grandparent.rmdir()
-                except Exception as e:
-                    print(f"  Error deleting {sp}: {e}")
-            db["attachments"].delete(id=att["id"])
-            deleted_records += 1
+def cmd_attachments_prune(args):
+    from datetime import datetime, timezone, timedelta
+    from devplacepy.database import db
+    from devplacepy.attachments import delete_attachment
 
-    print(f"Pruned {deleted_records} orphan records, {deleted_files} files, {freed_bytes / 1024:.1f} KB freed")
+    if "attachments" not in db.tables:
+        print("Attachments table does not exist")
+        return
+
+    cutoff = (datetime.now(timezone.utc) - timedelta(hours=args.hours)).isoformat()
+    orphans = [
+        att for att in db["attachments"].find(target_type="", target_uid="")
+        if att.get("created_at", "") < cutoff
+    ]
+    for att in orphans:
+        delete_attachment(att["uid"])
+    print(f"Pruned {len(orphans)} orphan attachment(s) older than {args.hours}h")
+
+
+def _remove_zip_artifacts(job):
+    import shutil
+    from pathlib import Path
+    from devplacepy.services.jobs.zip_service import STAGING_DIR
+    local_path = (job.get("result") or {}).get("local_path")
+    if local_path:
+        Path(local_path).unlink(missing_ok=True)
+    shutil.rmtree(STAGING_DIR / job["uid"], ignore_errors=True)
+
+
+def cmd_zips_prune(args):
+    from datetime import datetime, timezone
+    from devplacepy.services.jobs import queue
+    now = datetime.now(timezone.utc)
+    removed = 0
+    for job in queue.list_jobs(kind="zip", status=queue.DONE):
+        expires_at = job.get("expires_at")
+        if not expires_at:
+            continue
+        try:
+            expiry = datetime.fromisoformat(expires_at)
+        except (ValueError, TypeError):
+            continue
+        if expiry < now:
+            _remove_zip_artifacts(job)
+            get_table("jobs").delete(uid=job["uid"])
+            removed += 1
+    print(f"Pruned {removed} expired zip job(s)")
+
+
+def cmd_zips_clear(args):
+    from devplacepy.services.jobs import queue
+    jobs = queue.list_jobs(kind="zip")
+    for job in jobs:
+        _remove_zip_artifacts(job)
+        get_table("jobs").delete(uid=job["uid"])
+    print(f"Cleared {len(jobs)} zip job(s) and their archives")
 
 
 def main():
@@ -97,16 +188,45 @@ def main():
     role_set.add_argument("role", choices=["member", "admin"])
     role_set.set_defaults(func=cmd_role_set)
 
+    apikey = sub.add_parser("apikey", help="Manage user API keys")
+    apikey_sub = apikey.add_subparsers(title="action", dest="action")
+    apikey_get = apikey_sub.add_parser("get", help="Print a user's API key")
+    apikey_get.add_argument("username")
+    apikey_get.set_defaults(func=cmd_apikey_get)
+    apikey_reset = apikey_sub.add_parser("reset", help="Regenerate a user's API key")
+    apikey_reset.add_argument("username")
+    apikey_reset.set_defaults(func=cmd_apikey_reset)
+    apikey_backfill = apikey_sub.add_parser("backfill", help="Assign API keys to users that lack one")
+    apikey_backfill.set_defaults(func=cmd_apikey_backfill)
+
     news = sub.add_parser("news", help="News management")
     news_sub = news.add_subparsers(title="action", dest="action")
     news_clear = news_sub.add_parser("clear", help="Delete all news from local database")
     news_clear.set_defaults(func=cmd_news_clear)
+    news_sanitize = news_sub.add_parser("sanitize", help="Strip HTML from all existing news descriptions and content")
+    news_sanitize.set_defaults(func=cmd_news_sanitize)
 
     attachments = sub.add_parser("attachments", help="Attachment management")
     att_sub = attachments.add_subparsers(title="action", dest="action")
     att_prune = att_sub.add_parser("prune", help="Remove orphaned attachment records and files")
+    att_prune.add_argument("--hours", type=int, default=24, help="Only prune orphans older than this many hours")
     att_prune.set_defaults(func=cmd_attachments_prune)
 
+    devii = sub.add_parser("devii", help="Devii assistant management")
+    devii_sub = devii.add_subparsers(title="action", dest="action")
+    devii_reset = devii_sub.add_parser("reset-quota", help="Reset the rolling 24h AI spend quota")
+    devii_reset.add_argument("username", nargs="?", help="Reset the quota for a single user")
+    devii_reset.add_argument("--guests", action="store_true", help="Reset every guest quota")
+    devii_reset.add_argument("--all", action="store_true", help="Reset every quota (users and guests)")
+    devii_reset.set_defaults(func=cmd_devii_reset_quota)
+
+    zips = sub.add_parser("zips", help="Zip archive job management")
+    zips_sub = zips.add_subparsers(title="action", dest="action")
+    zips_prune = zips_sub.add_parser("prune", help="Delete expired zip archives and their job rows")
+    zips_prune.set_defaults(func=cmd_zips_prune)
+    zips_clear = zips_sub.add_parser("clear", help="Delete every zip archive and job row")
+    zips_clear.set_defaults(func=cmd_zips_clear)
+
     args = parser.parse_args()
     if hasattr(args, "func"):
         args.func(args)

devplacepy/config.py

@@ -9,5 +9,20 @@ STATIC_DIR = BASE_DIR / "devplacepy" / "static"
 TEMPLATES_DIR = BASE_DIR / "devplacepy" / "templates"
 DATABASE_URL = environ.get("DEVPLACE_DATABASE_URL", f"sqlite:///{BASE_DIR / 'devplace.db'}")
 SECRET_KEY = environ.get("SECRET_KEY", "devplace-secret-key-change-in-production")
-SESSION_MAX_AGE = 86400 * 7
+SECONDS_PER_DAY = 86400
+SESSION_MAX_AGE = SECONDS_PER_DAY * 7
+SESSION_MAX_AGE_REMEMBER = SECONDS_PER_DAY * 30
 PORT = 10500
+SITE_URL = environ.get("DEVPLACE_SITE_URL", "").rstrip("/")
+
+INTERNAL_BASE_URL = environ.get("DEVPLACE_INTERNAL_BASE_URL", f"http://localhost:{PORT}").rstrip("/")
+INTERNAL_GATEWAY_URL = f"{INTERNAL_BASE_URL}/openai/v1/chat/completions"
+INTERNAL_MODEL = "molodetz"
+
+SERVICE_LOCK_FILE = BASE_DIR / "devplace-services.lock"
+INIT_LOCK_FILE = BASE_DIR / "devplace-init.lock"
+
+VAPID_PRIVATE_KEY_FILE = BASE_DIR / "notification-private.pem"
+VAPID_PRIVATE_KEY_PKCS8_FILE = BASE_DIR / "notification-private.pkcs8.pem"
+VAPID_PUBLIC_KEY_FILE = BASE_DIR / "notification-public.pem"
+VAPID_SUB = environ.get("DEVPLACE_VAPID_SUB", "mailto:retoor@molodetz.nl")

devplacepy/constants.py

@@ -1 +1,3 @@
 TOPICS = ["devlog", "showcase", "question", "rant", "fun", "random", "signals"]
+
+REACTION_EMOJI = ["\U0001F44D", "❤️", "\U0001F680", "\U0001F389", "\U0001F602", "\U0001F440", "\U0001F525", "\U0001F92F"]

devplacepy/content.py

@@ -0,0 +1,176 @@
+# retoor <retoor@molodetz.nl>
+import logging
+from typing import Any
+from datetime import datetime, timezone
+from fastapi.responses import RedirectResponse
+from devplacepy.database import (
+    get_table,
+    resolve_by_slug,
+    get_users_by_uids,
+    get_vote_counts,
+    get_user_votes,
+    get_reactions_by_targets,
+    get_user_bookmarks,
+    get_poll_for_post,
+    delete_engagement,
+    load_comments,
+    db,
+)
+
+BOOKMARKABLE_TYPES = {"post", "gist", "project", "news"}
+REACTABLE_TYPES = {"post", "comment", "gist", "project"}
+from devplacepy.attachments import delete_attachments_for, delete_inline_image, get_attachments, link_attachments
+from devplacepy.utils import time_ago, generate_uid, make_combined_slug, award_rewards, create_mention_notifications
+
+logger = logging.getLogger(__name__)
+
+
+def is_owner(item: dict | None, user: dict | None) -> bool:
+    return bool(item and user and item["user_uid"] == user["uid"])
+
+
+def canonical_redirect(area: str, item: dict, requested: str) -> RedirectResponse | None:
+    canonical = item.get("slug") or item["uid"]
+    if requested == canonical:
+        return None
+    return RedirectResponse(url=f"/{area}/{canonical}", status_code=301)
+
+
+def first_image_url(item: dict, attachments: list | None) -> str | None:
+    inline = item.get("image")
+    if inline:
+        return f"/static/uploads/{inline}"
+    for attachment in attachments or []:
+        if attachment.get("is_image"):
+            return attachment["url"]
+    return None
+
+
+def create_content_item(table_name: str, target_type: str, user: dict, fields: dict, slug_source: str, xp: int, badge: str, mention_text: str, attachment_uids: list | None) -> tuple[str, str]:
+    uid = generate_uid()
+    slug = make_combined_slug(slug_source, uid)
+    get_table(table_name).insert({
+        "uid": uid,
+        "user_uid": user["uid"],
+        "slug": slug,
+        "stars": 0,
+        "created_at": datetime.now(timezone.utc).isoformat(),
+        **fields,
+    })
+    award_rewards(user["uid"], xp, badge)
+    if attachment_uids:
+        link_attachments(attachment_uids, target_type, uid)
+    create_mention_notifications(mention_text, user["uid"], f"/{table_name}/{slug}")
+    logger.info(f"{target_type} {uid} created by {user['username']}")
+    return uid, slug
+
+
+def detail_context(request, user: dict | None, detail: dict, key: str, seo_ctx: dict, extra: dict | None = None) -> dict:
+    context = {
+        **seo_ctx,
+        "request": request,
+        "user": user,
+        key: detail["item"],
+        "author": detail["author"],
+        "is_owner": detail["is_owner"],
+        "star_count": detail["star_count"],
+        "my_vote": detail["my_vote"],
+        "time_ago": detail["time_ago"],
+        "comments": detail["comments"],
+        "attachments": detail["attachments"],
+        "reactions": detail.get("reactions", {"counts": {}, "mine": []}),
+        "bookmarked": detail.get("bookmarked", False),
+        "poll": detail.get("poll"),
+    }
+    if extra:
+        context.update(extra)
+    return context
+
+
+def edit_content_item(request, table_name: str, user: dict, slug: str, update_fields: dict, redirect_fail: str):
+    from devplacepy.responses import action_result, wants_json, json_error
+    table = get_table(table_name)
+    item = resolve_by_slug(table, slug)
+    if not is_owner(item, user):
+        if wants_json(request):
+            return json_error(403, "Not allowed")
+        return RedirectResponse(url=redirect_fail, status_code=302)
+    update_fields = {**update_fields, "updated_at": datetime.now(timezone.utc).isoformat()}
+    table.update({"uid": item["uid"], **update_fields}, ["uid"])
+    logger.info(f"{table_name} {item['uid']} edited by {user['username']}")
+    url = f"/{table_name}/{item['slug'] or item['uid']}"
+    return action_result(request, url, data={"uid": item["uid"], "slug": item.get("slug"), "url": url})
+
+
+def delete_content_item(request, table_name: str, target_type: str, user: dict, slug: str, redirect_url: str, inline_image_field: str | None = None):
+    from devplacepy.responses import action_result, wants_json, json_error
+    table = get_table(table_name)
+    item = resolve_by_slug(table, slug)
+    if not is_owner(item, user):
+        if wants_json(request):
+            return json_error(403, "Not allowed")
+        return RedirectResponse(url=redirect_url, status_code=302)
+    if is_owner(item, user):
+        delete_attachments_for(target_type, [item["uid"]])
+        comment_uids = []
+        if "comments" in db.tables:
+            comments = get_table("comments")
+            comment_uids = [comment["uid"] for comment in comments.find(target_uid=item["uid"])]
+            delete_attachments_for("comment", comment_uids)
+            comments.delete(target_uid=item["uid"])
+        if "votes" in db.tables:
+            votes = get_table("votes")
+            votes.delete(target_uid=item["uid"])
+            if comment_uids:
+                votes.delete(votes.table.columns.target_uid.in_(comment_uids), target_type="comment")
+        delete_engagement(target_type, [item["uid"]])
+        if comment_uids:
+            delete_engagement("comment", comment_uids)
+        if target_type == "project":
+            from devplacepy.project_files import delete_all_project_files
+            delete_all_project_files(item["uid"])
+        if inline_image_field:
+            delete_inline_image(item.get(inline_image_field))
+        table.delete(id=item["id"])
+        logger.info(f"{table_name} {item['uid']} deleted by {user['username']}")
+    return action_result(request, redirect_url)
+
+
+def load_detail(table_name: str, target_type: str, slug: str, user: dict | None) -> dict | None:
+    item = resolve_by_slug(get_table(table_name), slug)
+    if not item:
+        return None
+    author = get_users_by_uids([item["user_uid"]]).get(item["user_uid"])
+    ups, downs = get_vote_counts([item["uid"]])
+    reactions = get_reactions_by_targets(target_type, [item["uid"]], user).get(item["uid"], {"counts": {}, "mine": []}) if target_type in REACTABLE_TYPES else {"counts": {}, "mine": []}
+    bookmarked = bool(user) and target_type in BOOKMARKABLE_TYPES and item["uid"] in get_user_bookmarks(user["uid"], target_type, [item["uid"]])
+    return {
+        "item": item,
+        "author": author,
+        "is_owner": bool(user and user["uid"] == item["user_uid"]),
+        "star_count": ups.get(item["uid"], 0) - downs.get(item["uid"], 0),
+        "my_vote": get_user_votes(user["uid"], [item["uid"]]).get(item["uid"], 0) if user else 0,
+        "comments": load_comments(target_type, item["uid"], user),
+        "attachments": get_attachments(target_type, item["uid"]),
+        "time_ago": time_ago(item["created_at"]),
+        "reactions": reactions,
+        "bookmarked": bookmarked,
+        "poll": get_poll_for_post(item["uid"], user) if target_type == "post" else None,
+    }
+
+
+def enrich_items(items: list, key: str, authors: dict, extra_maps: dict[str, Any] | None = None, ts_field: str = "created_at", user: dict | None = None) -> list:
+    extra_maps = extra_maps or {}
+    my_votes = get_user_votes(user["uid"], [item["uid"] for item in items]) if user else {}
+    enriched = []
+    for item in items:
+        entry = {
+            key: item,
+            "author": authors.get(item["user_uid"]),
+            "time_ago": time_ago(item[ts_field]),
+            "my_vote": my_votes.get(item["uid"], 0),
+        }
+        for name, source in extra_maps.items():
+            entry[name] = source(item) if callable(source) else source.get(item["uid"], 0)
+        enriched.append(entry)
+    return enriched

devplacepy/docs_examples.py

@@ -0,0 +1,81 @@
+from typing import Any, Union, get_args, get_origin
+
+from pydantic import BaseModel
+
+_PLACEHOLDERS = {
+    "uid": "UID",
+    "username": "username",
+    "slug": "slug",
+    "url": "/path",
+    "email": "user@example.com",
+    "title": "Title",
+    "content": "text",
+    "description": "text",
+    "bio": "text",
+    "language": "python",
+    "topic": "random",
+    "status": "published",
+    "next_cursor": "2026-01-01T00:00:00+00:00",
+    "created_at": "2026-01-01T00:00:00+00:00",
+    "updated_at": "2026-01-01T00:00:00+00:00",
+    "synced_at": "2026-01-01T00:00:00+00:00",
+    "time_ago": "2 hours ago",
+}
+
+
+def _placeholder(name: str) -> str:
+    for token, value in _PLACEHOLDERS.items():
+        if token in name:
+            return value
+    return "string"
+
+
+def _unwrap_optional(annotation):
+    if get_origin(annotation) is Union:
+        args = [arg for arg in get_args(annotation) if arg is not type(None)]
+        return args[0] if args else Any
+    return annotation
+
+
+def _value(name, annotation, stack):
+    annotation = _unwrap_optional(annotation)
+    origin = get_origin(annotation)
+    if origin in (list, set, tuple):
+        args = get_args(annotation)
+        inner = args[0] if args else str
+        if isinstance(inner, type) and issubclass(inner, BaseModel) and inner in stack:
+            return []
+        return [_value(name, inner, stack)]
+    if origin is dict or annotation is dict:
+        return {}
+    if annotation in (list, set, tuple):
+        return []
+    if isinstance(annotation, type) and issubclass(annotation, BaseModel):
+        return _from_model(annotation, stack)
+    if annotation is bool:
+        return False
+    if annotation is int:
+        return 0
+    if annotation is float:
+        return 0.0
+    if annotation is Any:
+        return None
+    return _placeholder(name)
+
+
+def _from_model(model, stack):
+    if model in stack:
+        return {}
+    stack = stack | {model}
+    example = {}
+    for name, info in model.model_fields.items():
+        example[name] = _value(name, info.annotation, stack)
+    return example
+
+
+def schema_example(model):
+    return _from_model(model, frozenset())
+
+
+def action_example(redirect, data=None):
+    return {"ok": True, "redirect": redirect, "data": data}

devplacepy/docs_export.py

@@ -0,0 +1,159 @@
+import base64
+import json
+import re
+from functools import lru_cache
+from pathlib import Path
+
+from devplacepy.config import STATIC_DIR
+from devplacepy import docs_api
+
+
+@lru_cache(maxsize=None)
+def _vendor(name: str) -> str:
+    return (Path(STATIC_DIR) / "vendor" / name).read_text(encoding="utf-8")
+
+
+def _prose_markdown(slug: str, ctx: dict) -> str:
+    from devplacepy.templating import templates
+    try:
+        rendered = templates.env.get_template(f"docs/{slug}.html").render(**ctx)
+    except Exception:
+        return ""
+    rendered = re.sub(r"^\s*<div[^>]*>", "", rendered.strip())
+    rendered = re.sub(r"</div>\s*$", "", rendered)
+    return rendered.strip()
+
+
+def _params_table(params: list) -> str:
+    if not params:
+        return ""
+    rows = ["| Name | In | Type | Required | Description |",
+            "|------|----|------|----------|-------------|"]
+    for p in params:
+        desc = (p.get("description", "") or "").replace("|", "\\|")
+        if p["type"] == "enum" and p.get("options"):
+            allowed = ", ".join(str(option) for option in p["options"]).replace("|", "\\|")
+            desc = f"{desc} Allowed: {allowed}.".strip()
+        rows.append(f"| `{p['name']}` | {p['location']} | {p['type']} | "
+                    f"{'yes' if p['required'] else 'no'} | {desc} |")
+    return "\n".join(rows)
+
+
+def _endpoint_markdown(ep: dict) -> str:
+    lines = [f"### `{ep['method']} {ep['path']}` - {ep['title']}", "", ep["summary"], "",
+             f"*Minimal role:* {ep.get('min_role', ep['auth'])}"]
+    if ep.get("params"):
+        lines += ["", "**Parameters**", "", _params_table(ep["params"])]
+    for note in ep.get("notes", []) or []:
+        lines += ["", f"> {note}"]
+    if ep.get("sample_response") is not None:
+        lines += ["", "**Sample response**", "", "```json",
+                  json.dumps(ep["sample_response"], indent=2), "```"]
+    return "\n".join(lines)
+
+
+def _group_markdown(group: dict) -> str:
+    parts = [(group.get("intro", "") or "").strip(), ""]
+    for ep in group.get("endpoints", []) or []:
+        parts.append(_endpoint_markdown(ep))
+        parts.append("")
+    return "\n".join(parts).strip()
+
+
+def build_markdown(is_admin: bool, base: str, username: str = "", api_key: str = "") -> str:
+    from devplacepy.routers.docs import DOCS_PAGES
+    from devplacepy.services.manager import service_manager
+
+    user_ctx = {"role": "Admin"} if is_admin else None
+    replacements = {
+        "{{ base }}": base,
+        "{{ username }}": username or "YOUR_USERNAME",
+        "{{ api_key }}": api_key or "YOUR_API_KEY",
+    }
+    pages = [p for p in DOCS_PAGES if (not p.get("admin") or is_admin) and p["kind"] != "live"]
+    slugs = {p["slug"] for p in pages}
+
+    out = [f"# DevPlace Documentation", "",
+           f"Complete developer documentation for `{base}`.", "", "## Contents", ""]
+    for page in pages:
+        out.append(f"- [{page['title']}](#doc-{page['slug']})")
+    out += ["", "---", ""]
+
+    for page in pages:
+        if page["kind"] == "prose":
+            body = _prose_markdown(page["slug"], {"base": base, "user": user_ctx})
+        elif page.get("dynamic"):
+            group = docs_api.build_services_group(service_manager.describe_all(), base)
+            body = _group_markdown(docs_api._substitute(group, replacements))
+        else:
+            group = docs_api.render_group(page["slug"], base, username, api_key) or {}
+            body = _group_markdown(group)
+        out += [f'<a id="doc-{page["slug"]}"></a>', body, "", "---", ""]
+
+    text = "\n".join(out).strip() + "\n"
+    return re.sub(
+        r"\(/docs/([a-z0-9-]+)\.html(?:#[^)]*)?\)",
+        lambda m: f"(#doc-{m.group(1)})" if m.group(1) in slugs else m.group(0),
+        text,
+    )
+
+
+_LAYOUT_CSS = """
+:root { color-scheme: light dark; }
+* { box-sizing: border-box; }
+body { margin: 0; font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
+  line-height: 1.65; color: #1f2328; background: #fff; }
+.doc-header { position: sticky; top: 0; z-index: 5; background: #0d1117; color: #fff;
+  padding: 0.9rem 1.5rem; font-weight: 700; letter-spacing: 0.02em; box-shadow: 0 1px 0 rgba(0,0,0,0.15); }
+.doc-header span { opacity: 0.6; font-weight: 400; }
+main.doc { max-width: 860px; margin: 0 auto; padding: 2rem 1.5rem 4rem; }
+main.doc h1 { font-size: 1.9rem; margin: 2.4rem 0 1rem; padding-bottom: 0.3rem; border-bottom: 2px solid #e1e4e8; }
+main.doc h1:first-child { margin-top: 0; }
+main.doc h2 { font-size: 1.4rem; margin: 2rem 0 0.8rem; }
+main.doc h3 { font-size: 1.08rem; margin: 1.6rem 0 0.5rem; }
+main.doc h3 code { background: #eef1f4; padding: 0.1em 0.4em; border-radius: 5px; font-size: 0.95em; }
+main.doc a { color: #0969da; text-decoration: none; }
+main.doc a:hover { text-decoration: underline; }
+main.doc code { font-family: "SF Mono", Monaco, Consolas, monospace; font-size: 0.9em; }
+main.doc :not(pre) > code { background: #eef1f4; padding: 0.12em 0.4em; border-radius: 5px; }
+main.doc pre { background: #282c34; border-radius: 8px; padding: 1rem; overflow-x: auto; }
+main.doc pre code { color: #abb2bf; font-size: 0.85rem; line-height: 1.5; }
+main.doc table { border-collapse: collapse; width: 100%; margin: 1rem 0; font-size: 0.92rem; }
+main.doc th, main.doc td { border: 1px solid #d0d7de; padding: 0.45rem 0.7rem; text-align: left; }
+main.doc th { background: #f6f8fa; }
+main.doc blockquote { margin: 0.8rem 0; padding: 0.2rem 1rem; border-left: 4px solid #d0d7de; color: #57606a; }
+main.doc hr { border: none; border-top: 1px solid #e1e4e8; margin: 2.5rem 0; }
+@media (prefers-color-scheme: dark) {
+  body { background: #0d1117; color: #c9d1d9; }
+  main.doc h1, main.doc h2 { border-color: #30363d; }
+  main.doc :not(pre) > code, main.doc h3 code { background: #21262d; }
+  main.doc th { background: #161b22; }
+  main.doc th, main.doc td, main.doc hr, main.doc blockquote { border-color: #30363d; }
+  main.doc a { color: #58a6ff; }
+}
+"""
+
+
+def build_html(markdown: str) -> str:
+    marked = _vendor("marked.umd.js")
+    hljs = _vendor("highlight.min.js")
+    hl_css = _vendor("atom-one-dark.min.css")
+    payload = base64.b64encode(markdown.encode("utf-8")).decode("ascii")
+    return (
+        "<!doctype html>\n<html lang=\"en\">\n<head>\n<meta charset=\"utf-8\">\n"
+        "<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n"
+        "<title>DevPlace Documentation</title>\n"
+        f"<style>{hl_css}</style>\n<style>{_LAYOUT_CSS}</style>\n"
+        f"<script>{marked}</script>\n<script>{hljs}</script>\n"
+        "</head>\n<body>\n"
+        "<header class=\"doc-header\">DevPlace <span>Documentation</span></header>\n"
+        "<main id=\"doc\" class=\"doc\"></main>\n"
+        f"<script id=\"docmd\" type=\"application/x-markdown-base64\">{payload}</script>\n"
+        "<script>\n(function(){\n"
+        "  var raw = atob(document.getElementById('docmd').textContent);\n"
+        "  var md = decodeURIComponent(escape(raw));\n"
+        "  marked.setOptions({ gfm: true, breaks: false });\n"
+        "  document.getElementById('doc').innerHTML = marked.parse(md);\n"
+        "  document.querySelectorAll('#doc pre code').forEach(function(el){ try { hljs.highlightElement(el); } catch (e) {} });\n"
+        "})();\n</script>\n</body>\n</html>\n"
+    )

devplacepy/docs_live.py

@@ -0,0 +1,147 @@
+from devplacepy.database import (
+    db,
+    get_table,
+    get_setting,
+    get_site_stats,
+    get_top_authors,
+    get_featured_news,
+    get_gist_languages,
+    get_daily_topic,
+    get_streaks,
+    get_user_stars,
+    get_user_rank,
+)
+from devplacepy.utils import format_date
+
+
+def _count(table: str, **criteria) -> int:
+    if table not in db.tables:
+        return 0
+    return get_table(table).count(**criteria)
+
+
+def _recent(table: str, user_uid: str, limit: int = 5) -> list:
+    if table not in db.tables:
+        return []
+    return list(get_table(table).find(user_uid=user_uid, order_by=["-created_at"], _limit=limit))
+
+
+def _user_facts(user: dict) -> dict:
+    uid = user["uid"]
+
+    posts = _recent("posts", uid)
+    gists = _recent("gists", uid)
+    projects = _recent("projects", uid)
+    badges = [b.get("badge_name", "") for b in get_table("badges").find(user_uid=uid)] if "badges" in db.tables else []
+    languages = sorted({
+        g.get("language") for g in (get_table("gists").find(user_uid=uid) if "gists" in db.tables else [])
+        if g.get("language")
+    })
+
+    return {
+        "username": user.get("username", ""),
+        "role": user.get("role", "Member"),
+        "member_since": format_date(user.get("created_at", "")) if user.get("created_at") else "",
+        "level": user.get("level", 1),
+        "xp": user.get("xp", 0),
+        "stars": get_user_stars(uid),
+        "rank": get_user_rank(uid),
+        "streak": get_streaks(uid),
+        "badges": badges,
+        "languages": languages,
+        "counts": {
+            "posts": _count("posts", user_uid=uid),
+            "gists": _count("gists", user_uid=uid),
+            "projects": _count("projects", user_uid=uid),
+            "comments": _count("comments", user_uid=uid),
+            "followers": _count("follows", following_uid=uid),
+            "following": _count("follows", follower_uid=uid),
+            "bookmarks": _count("bookmarks", user_uid=uid),
+            "unread_notifications": _count("notifications", user_uid=uid, read=False),
+            "unread_messages": _count("messages", receiver_uid=uid, read=False),
+        },
+        "recent_posts": [
+            {"title": (p.get("title") or (p.get("content") or "")[:60] or "Untitled"),
+             "url": f"/posts/{p.get('slug') or p['uid']}"}
+            for p in posts
+        ],
+        "recent_gists": [
+            {"title": g.get("title", "Untitled"), "language": g.get("language", ""),
+             "url": f"/gists/{g.get('slug') or g['uid']}"}
+            for g in gists
+        ],
+        "recent_projects": [
+            {"title": p.get("title", "Untitled"), "status": p.get("status", ""),
+             "url": f"/projects/{p.get('slug') or p['uid']}"}
+            for p in projects
+        ],
+    }
+
+
+def build_live_facts(user: dict | None) -> dict:
+    stats = get_site_stats()
+    facts = {
+        "site": {
+            "name": get_setting("site_name", "DevPlace"),
+            "tagline": get_setting("site_tagline", ""),
+            "total_members": stats.get("total_members", 0),
+            "posts_today": stats.get("posts_today", 0),
+            "total_projects": stats.get("total_projects", 0),
+            "total_gists": stats.get("total_gists", 0),
+        },
+        "top_authors": [
+            {"username": a.get("username", ""), "stars": a.get("stars", 0)}
+            for a in get_top_authors(10)
+        ],
+        "languages": sorted(get_gist_languages()),
+        "news": get_featured_news(5),
+        "topic": get_daily_topic(),
+        "user": _user_facts(user) if user else None,
+    }
+    return facts
+
+
+def live_text(facts: dict) -> str:
+    parts: list[str] = []
+    site = facts["site"]
+    parts += [
+        site["name"], site["tagline"],
+        "members", str(site["total_members"]),
+        "posts today", str(site["posts_today"]),
+        "projects", str(site["total_projects"]),
+        "gists", str(site["total_gists"]),
+        "top authors leaderboard",
+    ]
+    parts += [f"{a['username']} {a['stars']} stars" for a in facts["top_authors"]]
+    parts += ["trending languages"] + facts["languages"]
+    parts += ["developer news"] + [n.get("title", "") for n in facts["news"]]
+    parts += [facts["topic"].get("title", "")]
+
+    user = facts.get("user")
+    if user:
+        counts = user["counts"]
+        parts += [
+            "your dashboard account stats",
+            user["username"], user["role"], "member since", user["member_since"],
+            "level", str(user["level"]), "xp", str(user["xp"]),
+            "stars", str(user["stars"]), "rank", str(user["rank"]),
+            "current streak", str(user["streak"].get("current", 0)),
+            "longest streak", str(user["streak"].get("longest", 0)),
+            "posts", str(counts["posts"]),
+            "gists", str(counts["gists"]),
+            "projects", str(counts["projects"]),
+            "comments", str(counts["comments"]),
+            "followers", str(counts["followers"]),
+            "following", str(counts["following"]),
+            "bookmarks saved", str(counts["bookmarks"]),
+            "unread notifications", str(counts["unread_notifications"]),
+            "unread messages", str(counts["unread_messages"]),
+            "badges",
+        ]
+        parts += user["badges"]
+        parts += ["languages you use"] + user["languages"]
+        parts += ["your recent posts"] + [p["title"] for p in user["recent_posts"]]
+        parts += ["your gists"] + [g["title"] for g in user["recent_gists"]]
+        parts += ["your projects"] + [p["title"] for p in user["recent_projects"]]
+
+    return " ".join(str(part) for part in parts if part)

devplacepy/docs_prose.py

@@ -0,0 +1,28 @@
+# retoor <retoor@molodetz.nl>
+
+import html
+import re
+
+import mistune
+
+_markdown = mistune.create_markdown(
+    escape=False,
+    hard_wrap=True,
+    plugins=["table", "strikethrough", "url"],
+)
+
+_RENDER_BLOCK = re.compile(
+    r'<div class="docs-content" data-render>(.*?)</div>', re.DOTALL
+)
+
+
+def _convert(match: re.Match) -> str:
+    source = html.unescape(match.group(1)).strip()
+    return f'<div class="docs-content">{_markdown(source)}</div>'
+
+
+def render_prose(slug: str, context: dict) -> str:
+    from devplacepy.templating import templates
+
+    rendered = templates.env.get_template(f"docs/{slug}.html").render(**context)
+    return _RENDER_BLOCK.sub(_convert, rendered, count=1)

devplacepy/docs_search.py

@@ -0,0 +1,188 @@
+import html
+import math
+import re
+from pathlib import Path
+
+from devplacepy.config import TEMPLATES_DIR
+from devplacepy import docs_api
+
+K1 = 1.5
+B = 0.75
+_TOKEN = re.compile(r"[a-z0-9]+")
+_static_docs = None
+
+
+def _tokenize(text: str) -> list:
+    return [t for t in _TOKEN.findall(text.lower()) if len(t) > 1]
+
+
+def _demarkdown(text: str) -> str:
+    text = re.sub(r"(?m)^[ \t]*```.*$", " ", text)
+    text = re.sub(r"!\[([^\]]*)\]\([^)]*\)", r"\1", text)
+    text = re.sub(r"\[([^\]]+)\]\([^)]*\)", r"\1", text)
+    text = re.sub(r"(?m)^[ \t]{0,3}#{1,6}[ \t]*", "", text)
+    text = re.sub(r"(?m)^[ \t]{0,3}>[ \t]?", "", text)
+    text = re.sub(r"(?m)^[ \t]*(?:[-*+]|\d+\.)[ \t]+", "", text)
+    text = re.sub(r"(?m)^[ \t]*\|?(?:[ \t]*:?-{2,}:?[ \t]*\|)+[ \t]*:?-{0,}:?[ \t]*$", " ", text)
+    text = text.replace("|", " ")
+    text = re.sub(r"[`*~]", "", text)
+    return text
+
+
+def _strip(raw: str) -> str:
+    raw = re.sub(r"(?is)<script\b.*?</script>", " ", raw)
+    raw = re.sub(r"(?is)<style\b.*?</style>", " ", raw)
+    raw = re.sub(r"{#.*?#}", " ", raw, flags=re.S)
+    raw = re.sub(r"{%.*?%}", " ", raw, flags=re.S)
+    raw = re.sub(r"{{.*?}}", " ", raw, flags=re.S)
+    raw = html.unescape(raw)
+    raw = re.sub(r"<[^>]+>", " ", raw)
+    raw = _demarkdown(raw)
+    return re.sub(r"\s+", " ", raw).strip()
+
+
+def _prose_text(slug: str) -> str:
+    path = Path(TEMPLATES_DIR) / "docs" / f"{slug}.html"
+    try:
+        return _strip(path.read_text(encoding="utf-8"))
+    except OSError:
+        return ""
+
+
+def _group_text(group: dict) -> str:
+    parts = [group.get("intro", "")]
+    for ep in group.get("endpoints", []) or []:
+        parts += [ep.get("method", ""), ep.get("path", ""), ep.get("title", ""), ep.get("summary", "")]
+        parts += [str(note) for note in ep.get("notes", []) or []]
+        for param in ep.get("params", []) or []:
+            parts += [param.get("name", ""), param.get("description", "")]
+    return _strip("\n".join(str(p) for p in parts))
+
+
+def _collect_static_docs() -> list:
+    global _static_docs
+    if _static_docs is not None:
+        return _static_docs
+    from devplacepy.routers.docs import DOCS_PAGES
+    from devplacepy.services.manager import service_manager
+
+    docs = []
+    for page in DOCS_PAGES:
+        if page["kind"] == "live":
+            continue
+        slug, title = page["slug"], page["title"]
+        if page["kind"] == "prose":
+            body = _prose_text(slug)
+        elif page.get("dynamic"):
+            body = _group_text(docs_api.build_services_group(service_manager.describe_all(), ""))
+        else:
+            body = _group_text(docs_api.get_group(slug) or {})
+        docs.append({
+            "slug": slug,
+            "title": title,
+            "admin": page.get("admin", False),
+            "text": f"{title}\n{body}",
+        })
+    _static_docs = docs
+    return docs
+
+
+def _live_docs(user) -> list:
+    from devplacepy import docs_live
+
+    facts = docs_live.build_live_facts(user)
+    return [{
+        "slug": "dashboard",
+        "title": "Your dashboard",
+        "admin": False,
+        "text": "Your dashboard live data stats\n" + docs_live.live_text(facts),
+    }]
+
+
+def _build(docs: list) -> dict:
+    postings: dict = {}
+    doc_len: list = []
+    doc_text: list = []
+    df: dict = {}
+    for i, doc in enumerate(docs):
+        tokens = _tokenize(doc["text"])
+        doc_len.append(len(tokens) or 1)
+        doc_text.append(doc["text"])
+        freqs: dict = {}
+        for token in tokens:
+            freqs[token] = freqs.get(token, 0) + 1
+        for token, count in freqs.items():
+            postings.setdefault(token, []).append((i, count))
+            df[token] = df.get(token, 0) + 1
+    n = len(docs) or 1
+    avgdl = (sum(doc_len) / n) if doc_len else 1.0
+    idf = {t: math.log(1 + (n - d + 0.5) / (d + 0.5)) for t, d in df.items()}
+    return {
+        "docs": docs,
+        "postings": postings,
+        "doc_len": doc_len,
+        "doc_text": doc_text,
+        "avgdl": avgdl or 1.0,
+        "idf": idf,
+    }
+
+
+def get_index(user=None) -> dict:
+    return _build(list(_collect_static_docs()) + _live_docs(user))
+
+
+def reset_index() -> None:
+    global _static_docs
+    _static_docs = None
+
+
+def _snippet(text: str, terms: list, width: int = 280) -> str:
+    low = text.lower()
+    pos = -1
+    for term in terms:
+        found = low.find(term)
+        if found != -1 and (pos == -1 or found < pos):
+            pos = found
+    if pos == -1:
+        pos = 0
+    start = max(0, pos - width // 3)
+    end = min(len(text), start + width)
+    fragment = html.escape(text[start:end])
+    for term in sorted(set(terms), key=len, reverse=True):
+        fragment = re.sub(r"(?i)(" + re.escape(html.escape(term)) + r")", r"<mark>\1</mark>", fragment)
+    prefix = "… " if start > 0 else ""
+    suffix = " …" if end < len(text) else ""
+    return prefix + fragment + suffix
+
+
+def search(query: str, user=None, is_admin: bool = False, limit: int = 20) -> list:
+    terms = _tokenize(query or "")
+    if not terms:
+        return []
+    idx = get_index(user)
+    scores: dict = {}
+    for term in set(terms):
+        plist = idx["postings"].get(term)
+        if not plist:
+            continue
+        idf_t = idx["idf"][term]
+        for i, tf in plist:
+            dl = idx["doc_len"][i]
+            denom = tf + K1 * (1 - B + B * dl / idx["avgdl"])
+            scores[i] = scores.get(i, 0.0) + idf_t * (tf * (K1 + 1)) / denom
+    ranked = sorted(scores.items(), key=lambda kv: kv[1], reverse=True)
+    results = []
+    for i, score in ranked:
+        doc = idx["docs"][i]
+        if doc["admin"] and not is_admin:
+            continue
+        results.append({
+            "slug": doc["slug"],
+            "title": doc["title"],
+            "score": round(score, 3),
+            "url": f"/docs/{doc['slug']}.html",
+            "snippet": _snippet(idx["doc_text"][i], terms),
+        })
+        if len(results) >= limit:
+            break
+    return results

devplacepy/main.py

@@ -1,19 +1,29 @@
-import asyncio
+import fcntl
 import logging
 import os
 import time
 from collections import defaultdict
+from contextlib import contextmanager
+from pathlib import Path
 from fastapi import FastAPI, Request
 from fastapi.responses import HTMLResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
-from devplacepy.config import STATIC_DIR, PORT
-from devplacepy.database import init_db, get_table, db, get_users_by_uids, get_comment_counts_by_post_uids, get_vote_counts
+from fastapi.exceptions import RequestValidationError
+from devplacepy.config import STATIC_DIR, PORT, SERVICE_LOCK_FILE, INIT_LOCK_FILE
+from devplacepy.database import init_db, get_table, db, refresh_snapshot, get_users_by_uids, get_comment_counts_by_post_uids, get_vote_counts, get_news_images_by_uids, get_setting, get_int_setting
 from devplacepy.templating import templates
+from devplacepy.responses import wants_json, json_error
+from devplacepy.schemas import ValidationErrorOut
+from fastapi.responses import JSONResponse
 from devplacepy.utils import get_current_user, time_ago
-from devplacepy.seo import base_seo_context, site_url, website_schema, breadcrumb_schema, combine
-from devplacepy.routers import auth, feed, posts, comments, projects, profile, messages, notifications, votes, avatar, follow, admin, seo, bugs, news, gists, services as services_router, uploads
+from devplacepy.seo import base_seo_context, site_url, website_schema
+from devplacepy.routers import auth, feed, posts, comments, projects, project_files, profile, messages, notifications, votes, avatar, follow, admin, seo, bugs, news, gists, services as services_router, uploads, push, leaderboard, reactions, bookmarks, polls, docs, openai_gateway, devii, zips
 from devplacepy.services.manager import service_manager
 from devplacepy.services.news import NewsService
+from devplacepy.services.bot import BotsService
+from devplacepy.services.openai_gateway import GatewayService
+from devplacepy.services.devii import DeviiService
+from devplacepy.services.jobs.zip_service import ZipService
 
 logging.basicConfig(
     level=logging.INFO,
@@ -22,42 +32,160 @@ logging.basicConfig(
 logger = logging.getLogger(__name__)
 
 _rate_limit_store = defaultdict(list)
-RATE_LIMIT = 60
+RATE_LIMIT = int(os.environ.get("DEVPLACE_RATE_LIMIT", "60"))
 RATE_WINDOW = 60
+WEB_WORKERS = max(1, int(os.environ.get("DEVPLACE_WEB_WORKERS", "1")))
 
-app = FastAPI(title="DevPlace")
+
+def _worker_rate_limit(limit: int) -> int:
+    return max(1, -(-limit // WEB_WORKERS))
+
+
+_service_lock_handle = None
+
+
+@contextmanager
+def init_lock():
+    handle = open(INIT_LOCK_FILE, "w")
+    try:
+        fcntl.flock(handle, fcntl.LOCK_EX)
+        yield
+    finally:
+        fcntl.flock(handle, fcntl.LOCK_UN)
+        handle.close()
+
+
+def acquire_service_lock() -> bool:
+    global _service_lock_handle
+    handle = open(SERVICE_LOCK_FILE, "w")
+    try:
+        fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except OSError:
+        handle.close()
+        return False
+    _service_lock_handle = handle
+    return True
+
+INLINE_MEDIA_EXTENSIONS = {
+    ".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".tiff",
+    ".mp4", ".webm", ".ogv", ".mov", ".m4v", ".mp3",
+}
+
+
+class UploadStaticFiles(StaticFiles):
+    async def get_response(self, path, scope):
+        response = await super().get_response(path, scope)
+        disposition = "inline" if Path(path).suffix.lower() in INLINE_MEDIA_EXTENSIONS else "attachment"
+        response.headers["Content-Disposition"] = disposition
+        return response
+
+
+app = FastAPI(title="DevPlace", docs_url="/swagger", redoc_url="/redoc", openapi_url="/openapi.json")
+app.mount("/static/uploads", UploadStaticFiles(directory=str(STATIC_DIR / "uploads"), check_dir=False), name="uploads")
 app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
 
 
 @app.exception_handler(404)
 async def not_found(request: Request, exc):
-    seo_ctx = base_seo_context(request, title="Not Found — DevPlace", description="The page you requested does not exist.", robots="noindex")
-    return templates.TemplateResponse("error.html", {**seo_ctx, "request": request, "error_code": 404, "error_message": "Page not found"}, status_code=404)
+    if wants_json(request):
+        return json_error(404, "Not found")
+    seo_ctx = base_seo_context(request, title="Not Found - DevPlace", description="The page you requested does not exist.", robots="noindex")
+    return templates.TemplateResponse(request, "error.html", {**seo_ctx, "request": request, "error_code": 404, "error_message": "Page not found"}, status_code=404)
 
 
 @app.exception_handler(500)
 async def server_error(request: Request, exc):
-    seo_ctx = base_seo_context(request, title="Server Error — DevPlace", description="Something went wrong.", robots="noindex")
-    return templates.TemplateResponse("error.html", {**seo_ctx, "request": request, "error_code": 500, "error_message": "Internal server error"}, status_code=500)
+    logger.exception("500 error on %s %s", request.method, request.url.path)
+    if wants_json(request):
+        return json_error(500, "Internal server error")
+    seo_ctx = base_seo_context(request, title="Server Error - DevPlace", description="Something went wrong.", robots="noindex")
+    return templates.TemplateResponse(request, "error.html", {**seo_ctx, "request": request, "error_code": 500, "error_message": "Internal server error"}, status_code=500)
+
+
+_AUTH_FORM_PAGES = {
+    "/auth/signup": ("signup.html", "Join DevPlace"),
+    "/auth/login": ("login.html", "Sign In"),
+    "/auth/forgot-password": ("forgot_password.html", "Reset Password"),
+}
+
+_FRIENDLY_ERRORS = {
+    ("username", "too_short"): "Username must be between 3 and 32 characters",
+    ("username", "too_long"): "Username must be between 3 and 32 characters",
+    ("password", "too_short"): "Password must be at least 6 characters",
+}
+
+
+def _friendly_error(err):
+    field = err["loc"][-1] if err.get("loc") else ""
+    key = (field, err.get("type", "").replace("string_", ""))
+    if key in _FRIENDLY_ERRORS:
+        return _FRIENDLY_ERRORS[key]
+    msg = err.get("msg", "Invalid input")
+    prefix = "Value error, "
+    return msg[len(prefix):] if msg.startswith(prefix) else msg
+
+
+@app.exception_handler(RequestValidationError)
+async def on_validation_error(request: Request, exc: RequestValidationError):
+    errors = [_friendly_error(e) for e in exc.errors()]
+    if wants_json(request):
+        fields: dict = {}
+        for raw, friendly in zip(exc.errors(), errors):
+            name = raw["loc"][-1] if raw.get("loc") else "_"
+            fields.setdefault(str(name), []).append(friendly)
+        return JSONResponse(ValidationErrorOut(fields=fields, messages=errors).model_dump(mode="json"), status_code=422)
+    path = request.url.path
+    page = _AUTH_FORM_PAGES.get(path)
+    if page is None and path.startswith("/auth/reset-password/"):
+        page = ("reset_password.html", "Set New Password")
+    if page:
+        template_name, title = page
+        context = {**base_seo_context(request, title=title, robots="noindex,nofollow"), "request": request, "errors": errors}
+        try:
+            form = await request.form()
+            context.update({k: v for k, v in form.items() if isinstance(v, str)})
+        except Exception:
+            pass
+        if "token" in request.path_params:
+            context["token"] = request.path_params["token"]
+        return templates.TemplateResponse(request, template_name, context, status_code=400)
+    referer = request.headers.get("referer") or "/feed"
+    return RedirectResponse(url=referer, status_code=303)
 
 app.include_router(auth.router, prefix="/auth")
 app.include_router(feed.router, prefix="/feed")
 app.include_router(posts.router, prefix="/posts")
 app.include_router(comments.router, prefix="/comments")
 app.include_router(projects.router, prefix="/projects")
+app.include_router(project_files.router, prefix="/projects")
 app.include_router(profile.router, prefix="/profile")
 app.include_router(messages.router, prefix="/messages")
 app.include_router(notifications.router, prefix="/notifications")
 app.include_router(votes.router, prefix="/votes")
+app.include_router(reactions.router, prefix="/reactions")
+app.include_router(bookmarks.router, prefix="/bookmarks")
+app.include_router(polls.router, prefix="/polls")
 app.include_router(avatar.router, prefix="/avatar")
 app.include_router(follow.router, prefix="/follow")
+app.include_router(leaderboard.router, prefix="/leaderboard")
 app.include_router(admin.router, prefix="/admin")
 app.include_router(seo.router)
+app.include_router(push.router)
+app.include_router(docs.router)
+app.include_router(openai_gateway.router, prefix="/openai")
+app.include_router(devii.router, prefix="/devii")
 app.include_router(bugs.router, prefix="/bugs")
 app.include_router(gists.router, prefix="/gists")
 app.include_router(news.router, prefix="/news")
 app.include_router(services_router.router, prefix="/admin/services")
 app.include_router(uploads.router, prefix="/uploads")
+app.include_router(zips.router, prefix="/zips")
+
+
+@app.middleware("http")
+async def refresh_db_snapshot(request: Request, call_next):
+    refresh_snapshot()
+    return await call_next(request)
 
 
 @app.middleware("http")
@@ -71,31 +199,68 @@ async def add_security_headers(request: Request, call_next):
 
 @app.middleware("http")
 async def rate_limit_middleware(request: Request, call_next):
-    if request.method in ("POST", "PUT", "DELETE", "PATCH"):
-        ip = request.client.host if request.client else "unknown"
+    if request.method in ("POST", "PUT", "DELETE", "PATCH") and not request.url.path.startswith("/openai"):
+        limit = _worker_rate_limit(max(1, get_int_setting("rate_limit_per_minute", RATE_LIMIT)))
+        window = max(1, get_int_setting("rate_limit_window_seconds", RATE_WINDOW))
+        ip = request.headers.get("X-Real-IP") or (request.client.host if request.client else "unknown")
         now = time.time()
-        window_start = now - RATE_WINDOW
+        window_start = now - window
         _rate_limit_store[ip] = [t for t in _rate_limit_store[ip] if t > window_start]
-        if len(_rate_limit_store[ip]) >= RATE_LIMIT:
-            return HTMLResponse("Rate limit exceeded. Try again later.", status_code=429)
+        if len(_rate_limit_store[ip]) >= limit:
+            retry_after = {"Retry-After": str(window)}
+            if wants_json(request):
+                response = json_error(429, "Rate limit exceeded. Try again later.")
+                response.headers["Retry-After"] = str(window)
+                return response
+            return HTMLResponse("Rate limit exceeded. Try again later.", status_code=429, headers=retry_after)
         _rate_limit_store[ip].append(now)
     return await call_next(request)
 
 
+_MAINTENANCE_ALLOWED_PREFIXES = ("/static", "/avatar", "/auth", "/admin", "/openai")
+
+
+@app.middleware("http")
+async def maintenance_middleware(request: Request, call_next):
+    if get_setting("maintenance_mode", "0") != "1":
+        return await call_next(request)
+    if request.url.path.startswith(_MAINTENANCE_ALLOWED_PREFIXES):
+        return await call_next(request)
+    user = get_current_user(request)
+    if user and user.get("role") == "Admin":
+        return await call_next(request)
+    message = get_setting("maintenance_message", "DevPlace is undergoing scheduled maintenance. Please check back shortly.")
+    if wants_json(request):
+        return json_error(503, message)
+    seo_ctx = base_seo_context(request, title="Maintenance - DevPlace", description=message, robots="noindex")
+    return templates.TemplateResponse(request, "error.html", {**seo_ctx, "request": request, "error_code": 503, "error_message": message}, status_code=503)
+
+
 @app.on_event("startup")
 async def startup():
-    init_db()
+    with init_lock():
+        init_db()
+        from devplacepy.push import ensure_certificates
+        ensure_certificates()
+    service_manager.register(NewsService())
+    service_manager.register(BotsService())
+    service_manager.register(GatewayService())
+    service_manager.register(DeviiService())
+    service_manager.register(ZipService())
     if not os.environ.get("DEVPLACE_DISABLE_SERVICES"):
-        news_service = NewsService()
-        service_manager.register(news_service)
-        asyncio.create_task(service_manager.start_all())
+        if acquire_service_lock():
+            service_manager.set_lock_owner(True)
+            service_manager.supervise()
+            logger.info(f"Background services started in worker pid {os.getpid()}")
+        else:
+            logger.info(f"Worker pid {os.getpid()} declined service lock; another worker owns background services")
     logger.info(f"DevPlace started on port {PORT}")
 
 
 @app.on_event("shutdown")
 async def shutdown():
     logger.info("Shutting down services...")
-    await service_manager.stop_all()
+    await service_manager.shutdown_all()
 
 
 @app.get("/")
@@ -107,14 +272,9 @@ async def landing(request: Request):
     landing_articles = []
     if "news" in db.tables:
         news_table = get_table("news")
-        raw = list(news_table.find(order_by=["-synced_at"], _limit=6))
-        images_table = get_table("news_images") if "news_images" in db.tables else None
+        raw = list(news_table.find(show_on_landing=1, order_by=["-synced_at"], _limit=6))
+        images_by_news = get_news_images_by_uids([a["uid"] for a in raw])
         for a in raw:
-            image_url = ""
-            if images_table:
-                img = images_table.find_one(news_uid=a["uid"])
-                if img:
-                    image_url = img["url"]
             landing_articles.append({
                 "uid": a["uid"],
                 "slug": a.get("slug", ""),
@@ -125,7 +285,7 @@ async def landing(request: Request):
                 "grade": a.get("grade", 0),
                 "synced_at": a.get("synced_at", "") or "",
                 "time_ago": time_ago(a["synced_at"]),
-                "image_url": image_url,
+                "image_url": images_by_news.get(a["uid"], ""),
             })
 
     landing_posts = []
@@ -151,12 +311,12 @@ async def landing(request: Request):
     base = site_url(request)
     seo_ctx = base_seo_context(
         request,
-        title="DevPlace — The Developer Social Network",
+        title="DevPlace - The Developer Social Network",
         description="Track industry shifts. Discover bold releases. Share what you're building in an open, uncensored environment.",
         breadcrumbs=[],
         schemas=[website_schema(base)],
     )
-    return templates.TemplateResponse("landing.html", {
+    return templates.TemplateResponse(request, "landing.html", {
         **seo_ctx, "request": request,
         "landing_articles": landing_articles,
         "landing_posts": landing_posts,

devplacepy/models.py

@@ -1,49 +1,260 @@
-from pydantic import BaseModel, Field
-from typing import Optional
-from datetime import date
+from datetime import datetime
+from typing import Literal
+from pydantic import BaseModel, Field, field_validator, model_validator
+from devplacepy.constants import TOPICS, REACTION_EMOJI
 
 
-class SignupRequest(BaseModel):
-    username: str = Field(min_length=3, max_length=32, pattern=r"^[a-zA-Z0-9_-]+$")
-    email: str = Field(min_length=5, max_length=255)
+def normalize_european_date(value):
+    if not value:
+        return ""
+    text = str(value).strip()
+    if not text:
+        return ""
+    for fmt in ("%d/%m/%Y", "%Y-%m-%d"):
+        try:
+            return datetime.strptime(text, fmt).strftime("%Y-%m-%d")
+        except ValueError:
+            continue
+    raise ValueError("Date must be in DD/MM/YYYY format")
+
+
+def normalize_poll_options(value):
+    if value is None:
+        return []
+    if isinstance(value, str):
+        value = [value]
+    if not isinstance(value, list):
+        return value
+    if len(value) == 1 and isinstance(value[0], str):
+        single = value[0]
+        separator = "\n" if "\n" in single else ("," if "," in single else "")
+        if separator:
+            return [part.strip() for part in single.split(separator) if part.strip()]
+    return value
+
+
+class SignupForm(BaseModel):
+    username: str = Field(min_length=3, max_length=32)
+    email: str = Field(min_length=1, max_length=255)
     password: str = Field(min_length=6, max_length=128)
-    confirm_password: str = Field(min_length=6, max_length=128)
+    confirm_password: str = Field(min_length=1, max_length=128)
+
+    @field_validator("username")
+    @classmethod
+    def username_chars(cls, value):
+        if not value.isascii() or not all(c.isalnum() or c in ("-", "_") for c in value):
+            raise ValueError("Username can only contain letters, numbers, hyphens, and underscores")
+        return value
+
+    @field_validator("email")
+    @classmethod
+    def email_has_at(cls, value):
+        if "@" not in value:
+            raise ValueError("Valid email is required")
+        return value
+
+    @model_validator(mode="after")
+    def passwords_match(self):
+        if self.password != self.confirm_password:
+            raise ValueError("Passwords do not match")
+        return self
 
 
-class LoginRequest(BaseModel):
-    email: str = Field(min_length=5, max_length=255)
+class LoginForm(BaseModel):
+    email: str = Field(min_length=1, max_length=255)
+    password: str = Field(min_length=1, max_length=128)
+    remember_me: str = ""
+    next: str = ""
+
+
+class ForgotPasswordForm(BaseModel):
+    email: str = Field(min_length=1, max_length=255)
+
+    @field_validator("email")
+    @classmethod
+    def email_has_at(cls, value):
+        if "@" not in value:
+            raise ValueError("Valid email is required")
+        return value
+
+
+class ResetPasswordForm(BaseModel):
     password: str = Field(min_length=6, max_length=128)
-    remember_me: bool = False
+    confirm_password: str = Field(min_length=1, max_length=128)
+
+    @model_validator(mode="after")
+    def passwords_match(self):
+        if self.password != self.confirm_password:
+            raise ValueError("Passwords do not match")
+        return self
 
 
-class PostCreate(BaseModel):
-    content: str = Field(min_length=1, max_length=2000)
-    title: Optional[str] = Field(default=None, max_length=500)
-    topic: str = Field(pattern=r"^(devlog|showcase|question|rant|fun|random)$")
-    project_uid: Optional[str] = None
+class PostForm(BaseModel):
+    content: str = Field(min_length=10, max_length=2000)
+    title: str = Field(default="", max_length=500)
+    topic: str = "random"
+    project_uid: str = ""
+    attachment_uids: list[str] = []
+    poll_question: str = Field(default="", max_length=200)
+    poll_options: list[str] = []
+
+    @field_validator("topic")
+    @classmethod
+    def valid_topic(cls, value):
+        return value if value in TOPICS else "random"
+
+    @field_validator("poll_options", mode="before")
+    @classmethod
+    def split_poll_options(cls, value):
+        return normalize_poll_options(value)
 
 
-class CommentCreate(BaseModel):
-    content: str = Field(min_length=1, max_length=1000)
-    parent_uid: Optional[str] = None
+class PostEditForm(BaseModel):
+    content: str = Field(min_length=10, max_length=2000)
+    title: str = Field(default="", max_length=500)
+    topic: str = "random"
+    poll_question: str = Field(default="", max_length=200)
+    poll_options: list[str] = []
+
+    @field_validator("topic")
+    @classmethod
+    def valid_topic(cls, value):
+        return value if value in TOPICS else "random"
+
+    @field_validator("poll_options", mode="before")
+    @classmethod
+    def split_poll_options(cls, value):
+        return normalize_poll_options(value)
 
 
-class ProjectCreate(BaseModel):
+class CommentForm(BaseModel):
+    content: str = Field(min_length=3, max_length=1000)
+    target_uid: str = ""
+    post_uid: str = ""
+    target_type: Literal["post", "project", "news", "bug", "gist"] = "post"
+    parent_uid: str = ""
+    attachment_uids: list[str] = []
+
+    @model_validator(mode="after")
+    def require_target(self):
+        if not (self.target_uid or self.post_uid):
+            raise ValueError("A target is required")
+        return self
+
+
+class ProjectForm(BaseModel):
     title: str = Field(min_length=1, max_length=200)
     description: str = Field(min_length=1, max_length=5000)
-    release_date: Optional[date] = None
-    demo_date: Optional[date] = None
-    project_type: str = Field(pattern=r"^(game|game_asset|software|mobile_app|website)$")
+    release_date: str = ""
+    demo_date: str = ""
+    project_type: Literal["game", "game_asset", "software", "mobile_app", "website"] = "software"
     platforms: str = Field(default="", max_length=500)
+    status: str = Field(default="In Development", max_length=100)
+    attachment_uids: list[str] = []
+
+    @field_validator("release_date", "demo_date", mode="before")
+    @classmethod
+    def normalize_dates(cls, value):
+        return normalize_european_date(value)
 
 
-class MessageCreate(BaseModel):
+class ProjectFileWriteForm(BaseModel):
+    path: str = Field(min_length=1, max_length=1024)
+    content: str = Field(default="", max_length=400000)
+
+
+class ProjectFileMkdirForm(BaseModel):
+    path: str = Field(min_length=1, max_length=1024)
+
+
+class ProjectFileMoveForm(BaseModel):
+    from_path: str = Field(min_length=1, max_length=1024)
+    to_path: str = Field(min_length=1, max_length=1024)
+
+
+class ProjectFileDeleteForm(BaseModel):
+    path: str = Field(min_length=1, max_length=1024)
+
+
+class MessageForm(BaseModel):
     content: str = Field(min_length=1, max_length=2000)
-    receiver_uid: str
+    receiver_uid: str = Field(min_length=1)
+    attachment_uids: list[str] = []
 
 
-class ProfileUpdate(BaseModel):
-    bio: Optional[str] = Field(default=None, max_length=500)
-    location: Optional[str] = Field(default=None, max_length=200)
-    git_link: Optional[str] = Field(default=None, max_length=500)
-    website: Optional[str] = Field(default=None, max_length=500)
+class ProfileForm(BaseModel):
+    bio: str = Field(default="", max_length=500)
+    location: str = Field(default="", max_length=200)
+    git_link: str = Field(default="", max_length=500)
+    website: str = Field(default="", max_length=500)
+
+
+class GistForm(BaseModel):
+    title: str = Field(min_length=1, max_length=200)
+    description: str = Field(default="", max_length=5000)
+    source_code: str = Field(min_length=1, max_length=400000)
+    language: str = Field(default="plaintext", max_length=50)
+    attachment_uids: list[str] = []
+
+
+class GistEditForm(BaseModel):
+    title: str = Field(min_length=1, max_length=200)
+    description: str = Field(default="", max_length=5000)
+    source_code: str = Field(min_length=1, max_length=400000)
+    language: str = Field(default="plaintext", max_length=50)
+
+
+class BugForm(BaseModel):
+    title: str = Field(min_length=1, max_length=200)
+    description: str = Field(min_length=1, max_length=5000)
+    attachment_uids: list[str] = []
+
+
+class VoteForm(BaseModel):
+    value: int
+
+    @field_validator("value")
+    @classmethod
+    def valid_value(cls, value):
+        if value not in (1, -1):
+            raise ValueError("value must be 1 or -1")
+        return value
+
+
+class ReactionForm(BaseModel):
+    emoji: str = Field(min_length=1, max_length=16)
+
+    @field_validator("emoji")
+    @classmethod
+    def valid_emoji(cls, value):
+        if value not in REACTION_EMOJI:
+            raise ValueError("Invalid reaction")
+        return value
+
+
+class PollVoteForm(BaseModel):
+    option_uid: str = Field(min_length=1)
+
+
+class AdminRoleForm(BaseModel):
+    role: Literal["member", "admin"]
+
+
+class AdminPasswordForm(BaseModel):
+    password: str = Field(min_length=6, max_length=128)
+
+
+class AdminSettingsForm(BaseModel):
+    site_name: str = Field(default="", max_length=200)
+    site_description: str = Field(default="", max_length=500)
+    site_tagline: str = Field(default="", max_length=500)
+    max_upload_size_mb: str = Field(default="", max_length=10)
+    allowed_file_types: str = Field(default="", max_length=1000)
+    max_attachments_per_resource: str = Field(default="", max_length=10)
+    rate_limit_per_minute: str = Field(default="", max_length=10)
+    rate_limit_window_seconds: str = Field(default="", max_length=10)
+    session_max_age_days: str = Field(default="", max_length=10)
+    session_remember_days: str = Field(default="", max_length=10)
+    registration_open: str = Field(default="", max_length=1)
+    maintenance_mode: str = Field(default="", max_length=1)
+    maintenance_message: str = Field(default="", max_length=300)

devplacepy/project_files.py

@@ -0,0 +1,394 @@
+# retoor <retoor@molodetz.nl>
+
+import logging
+import shutil
+from datetime import datetime, timezone
+from pathlib import Path
+
+from devplacepy.database import get_table, db, get_int_setting
+from devplacepy.config import STATIC_DIR
+from devplacepy.utils import generate_uid
+from devplacepy.attachments import _directory_for, _detect_mime
+
+logger = logging.getLogger(__name__)
+
+PROJECT_FILES_DIR = STATIC_DIR / "uploads" / "project_files"
+MAX_TEXT_CHARS = 400_000
+MAX_FILES_PER_PROJECT = 5000
+MAX_PATH_LENGTH = 1024
+MAX_SEGMENT_LENGTH = 255
+MAX_DEPTH = 32
+
+TEXT_EXTENSIONS = {
+    ".txt", ".md", ".markdown", ".rst", ".log", ".csv", ".tsv",
+    ".py", ".pyi", ".ipynb", ".js", ".mjs", ".cjs", ".jsx", ".ts", ".tsx",
+    ".json", ".jsonc", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".conf", ".env",
+    ".html", ".htm", ".xml", ".svg", ".css", ".scss", ".sass", ".less",
+    ".c", ".h", ".cpp", ".cc", ".hpp", ".hh", ".cs", ".java", ".kt", ".kts",
+    ".go", ".rs", ".rb", ".php", ".pl", ".pm", ".lua", ".r", ".dart", ".swift",
+    ".scala", ".clj", ".ex", ".exs", ".erl", ".hs", ".ml", ".vue", ".svelte",
+    ".sh", ".bash", ".zsh", ".fish", ".ps1", ".bat", ".cmd",
+    ".sql", ".graphql", ".gql", ".proto", ".tf", ".hcl", ".dockerfile",
+    ".gitignore", ".gitattributes", ".editorconfig", ".properties",
+}
+TEXT_FILENAMES = {
+    "dockerfile", "makefile", "license", "readme", "changelog", "authors",
+    "procfile", "rakefile", "gemfile", ".gitignore", ".gitattributes",
+    ".env", ".editorconfig", ".dockerignore", ".npmrc", ".prettierrc", ".babelrc",
+}
+
+
+class ProjectFileError(ValueError):
+    pass
+
+
+def normalize_path(raw) -> str:
+    if raw is None:
+        raise ProjectFileError("path is required")
+    text = str(raw).strip().replace("\\", "/")
+    if "\x00" in text:
+        raise ProjectFileError("path contains a null byte")
+    parts = []
+    for segment in text.split("/"):
+        segment = segment.strip()
+        if segment in ("", "."):
+            continue
+        if segment == "..":
+            raise ProjectFileError("path may not contain '..'")
+        if len(segment) > MAX_SEGMENT_LENGTH:
+            raise ProjectFileError("path segment is too long")
+        if any(ord(ch) < 32 for ch in segment):
+            raise ProjectFileError("path contains control characters")
+        parts.append(segment)
+    if not parts:
+        raise ProjectFileError("path is empty")
+    if len(parts) > MAX_DEPTH:
+        raise ProjectFileError("path is too deep")
+    normalized = "/".join(parts)
+    if len(normalized) > MAX_PATH_LENGTH:
+        raise ProjectFileError("path is too long")
+    return normalized
+
+
+def _name_of(path: str) -> str:
+    return path.rsplit("/", 1)[-1]
+
+
+def _parent_of(path: str) -> str:
+    return path.rsplit("/", 1)[0] if "/" in path else ""
+
+
+def is_text_name(name: str) -> bool:
+    lower = name.lower()
+    if lower in TEXT_FILENAMES:
+        return True
+    return Path(name).suffix.lower() in TEXT_EXTENSIONS
+
+
+def _table():
+    return get_table("project_files")
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _file_url(directory: str, stored_name: str) -> str:
+    return f"/static/uploads/project_files/{directory}/{stored_name}"
+
+
+def get_node(project_uid: str, path: str):
+    return _table().find_one(project_uid=project_uid, path=path)
+
+
+def _count(project_uid: str) -> int:
+    if "project_files" not in db.tables:
+        return 0
+    return _table().count(project_uid=project_uid)
+
+
+def _guard_capacity(project_uid: str) -> None:
+    if _count(project_uid) >= MAX_FILES_PER_PROJECT:
+        raise ProjectFileError(f"project reached the {MAX_FILES_PER_PROJECT}-node limit")
+
+
+def _insert_dir(project_uid: str, user: dict, path: str) -> None:
+    _table().insert({
+        "uid": generate_uid(),
+        "project_uid": project_uid,
+        "user_uid": user["uid"],
+        "path": path,
+        "name": _name_of(path),
+        "parent_path": _parent_of(path),
+        "type": "dir",
+        "content": None,
+        "is_binary": 0,
+        "stored_name": None,
+        "directory": None,
+        "mime_type": None,
+        "size": 0,
+        "created_at": _now(),
+        "updated_at": _now(),
+    })
+
+
+def ensure_dirs(project_uid: str, user: dict, dir_path: str) -> None:
+    if not dir_path:
+        return
+    current = ""
+    for segment in dir_path.split("/"):
+        current = f"{current}/{segment}" if current else segment
+        existing = get_node(project_uid, current)
+        if existing is None:
+            _guard_capacity(project_uid)
+            _insert_dir(project_uid, user, current)
+        elif existing["type"] != "dir":
+            raise ProjectFileError(f"'{current}' exists and is a file")
+
+
+def make_dir(project_uid: str, user: dict, raw_path: str) -> dict:
+    path = normalize_path(raw_path)
+    existing = get_node(project_uid, path)
+    if existing is not None:
+        if existing["type"] != "dir":
+            raise ProjectFileError(f"'{path}' exists and is a file")
+        return existing
+    ensure_dirs(project_uid, user, path)
+    return get_node(project_uid, path)
+
+
+def write_text_file(project_uid: str, user: dict, raw_path: str, content: str) -> dict:
+    path = normalize_path(raw_path)
+    content = content or ""
+    if len(content) > MAX_TEXT_CHARS:
+        raise ProjectFileError(f"file content exceeds the {MAX_TEXT_CHARS}-character limit")
+    ensure_dirs(project_uid, user, _parent_of(path))
+    existing = get_node(project_uid, path)
+    if existing is not None and existing["type"] == "dir":
+        raise ProjectFileError(f"'{path}' is a directory")
+    if existing is not None:
+        if existing.get("is_binary"):
+            _unlink_blob(existing)
+        _table().update({
+            "uid": existing["uid"],
+            "content": content,
+            "is_binary": 0,
+            "stored_name": None,
+            "directory": None,
+            "mime_type": "text/plain",
+            "size": len(content.encode("utf-8")),
+            "updated_at": _now(),
+        }, ["uid"])
+        return get_node(project_uid, path)
+    _guard_capacity(project_uid)
+    _table().insert({
+        "uid": generate_uid(),
+        "project_uid": project_uid,
+        "user_uid": user["uid"],
+        "path": path,
+        "name": _name_of(path),
+        "parent_path": _parent_of(path),
+        "type": "file",
+        "content": content,
+        "is_binary": 0,
+        "stored_name": None,
+        "directory": None,
+        "mime_type": "text/plain",
+        "size": len(content.encode("utf-8")),
+        "created_at": _now(),
+        "updated_at": _now(),
+    })
+    return get_node(project_uid, path)
+
+
+def _store_blob(data: bytes, filename: str) -> tuple[str, str, str]:
+    uid = generate_uid()
+    ext = Path(filename).suffix.lower()
+    stored_name = f"{uid}{ext}"
+    directory = _directory_for(uid)
+    file_dir = PROJECT_FILES_DIR / directory
+    file_dir.mkdir(parents=True, exist_ok=True)
+    (file_dir / stored_name).write_bytes(data)
+    return stored_name, directory, _detect_mime(data, filename)
+
+
+def _unlink_blob(node: dict) -> None:
+    stored_name = node.get("stored_name")
+    directory = node.get("directory")
+    if not (stored_name and directory):
+        return
+    try:
+        (PROJECT_FILES_DIR / directory / stored_name).unlink(missing_ok=True)
+    except OSError as exc:
+        logger.warning("Failed to delete project blob %s/%s: %s", directory, stored_name, exc)
+
+
+def store_upload(project_uid: str, user: dict, dir_path: str, filename: str, data: bytes) -> dict:
+    max_bytes = get_int_setting("max_upload_size_mb", 10) * 1024 * 1024
+    if len(data) > max_bytes:
+        raise ProjectFileError(f"file exceeds the {get_int_setting('max_upload_size_mb', 10)}MB limit")
+    base = _name_of(normalize_path(filename))
+    path = normalize_path(f"{dir_path}/{base}" if dir_path else base)
+
+    if is_text_name(base):
+        try:
+            text = data.decode("utf-8")
+            if len(text) <= MAX_TEXT_CHARS:
+                return write_text_file(project_uid, user, path, text)
+        except UnicodeDecodeError:
+            pass
+
+    ensure_dirs(project_uid, user, _parent_of(path))
+    existing = get_node(project_uid, path)
+    if existing is not None and existing["type"] == "dir":
+        raise ProjectFileError(f"'{path}' is a directory")
+    stored_name, directory, mime = _store_blob(data, base)
+    if existing is not None:
+        if existing.get("is_binary"):
+            _unlink_blob(existing)
+        _table().update({
+            "uid": existing["uid"], "content": None, "is_binary": 1,
+            "stored_name": stored_name, "directory": directory, "mime_type": mime,
+            "size": len(data), "updated_at": _now(),
+        }, ["uid"])
+        return get_node(project_uid, path)
+    _guard_capacity(project_uid)
+    _table().insert({
+        "uid": generate_uid(),
+        "project_uid": project_uid,
+        "user_uid": user["uid"],
+        "path": path,
+        "name": base,
+        "parent_path": _parent_of(path),
+        "type": "file",
+        "content": None,
+        "is_binary": 1,
+        "stored_name": stored_name,
+        "directory": directory,
+        "mime_type": mime,
+        "size": len(data),
+        "created_at": _now(),
+        "updated_at": _now(),
+    })
+    return get_node(project_uid, path)
+
+
+def _descendants(project_uid: str, path: str) -> list:
+    prefix = f"{path}/"
+    return [
+        row for row in _table().find(project_uid=project_uid)
+        if row["path"] == path or row["path"].startswith(prefix)
+    ]
+
+
+def move_node(project_uid: str, user: dict, raw_from: str, raw_to: str) -> dict:
+    src_path = normalize_path(raw_from)
+    dst_path = normalize_path(raw_to)
+    if src_path == dst_path:
+        raise ProjectFileError("source and target are the same")
+    if dst_path == src_path or dst_path.startswith(f"{src_path}/"):
+        raise ProjectFileError("cannot move a node into itself")
+    src = get_node(project_uid, src_path)
+    if src is None:
+        raise ProjectFileError(f"'{src_path}' does not exist")
+    if get_node(project_uid, dst_path) is not None:
+        raise ProjectFileError(f"'{dst_path}' already exists")
+    ensure_dirs(project_uid, user, _parent_of(dst_path))
+
+    if src["type"] == "file":
+        _table().update({
+            "uid": src["uid"], "path": dst_path, "name": _name_of(dst_path),
+            "parent_path": _parent_of(dst_path), "updated_at": _now(),
+        }, ["uid"])
+        return get_node(project_uid, dst_path)
+
+    for row in _descendants(project_uid, src_path):
+        new_path = dst_path + row["path"][len(src_path):]
+        _table().update({
+            "uid": row["uid"], "path": new_path, "name": _name_of(new_path),
+            "parent_path": _parent_of(new_path), "updated_at": _now(),
+        }, ["uid"])
+    return get_node(project_uid, dst_path)
+
+
+def delete_node(project_uid: str, raw_path: str) -> None:
+    path = normalize_path(raw_path)
+    node = get_node(project_uid, path)
+    if node is None:
+        raise ProjectFileError(f"'{path}' does not exist")
+    for row in _descendants(project_uid, path):
+        if row.get("is_binary"):
+            _unlink_blob(row)
+        _table().delete(uid=row["uid"])
+
+
+def node_to_dict(row: dict) -> dict:
+    is_binary = bool(row.get("is_binary"))
+    url = None
+    if is_binary and row.get("stored_name") and row.get("directory"):
+        url = _file_url(row["directory"], row["stored_name"])
+    return {
+        "uid": row["uid"],
+        "path": row["path"],
+        "name": row["name"],
+        "type": row["type"],
+        "is_binary": is_binary,
+        "mime_type": row.get("mime_type"),
+        "size": row.get("size", 0),
+        "url": url,
+        "updated_at": row.get("updated_at"),
+    }
+
+
+def list_files(project_uid: str) -> list:
+    if "project_files" not in db.tables:
+        return []
+    rows = sorted(_table().find(project_uid=project_uid), key=lambda r: r["path"])
+    return [node_to_dict(row) for row in rows]
+
+
+def read_file(project_uid: str, raw_path: str) -> dict:
+    path = normalize_path(raw_path)
+    node = get_node(project_uid, path)
+    if node is None:
+        raise ProjectFileError(f"'{path}' does not exist")
+    payload = node_to_dict(node)
+    payload["content"] = node.get("content") if node["type"] == "file" and not node.get("is_binary") else None
+    return payload
+
+
+def export_to_dir(project_uid: str, subpath: str, dest_dir) -> int:
+    dest = Path(dest_dir).resolve()
+    dest.mkdir(parents=True, exist_ok=True)
+    if subpath:
+        base = normalize_path(subpath)
+        rows = _descendants(project_uid, base)
+        strip = _parent_of(base)
+    else:
+        rows = list(_table().find(project_uid=project_uid))
+        strip = ""
+    written = 0
+    for row in sorted(rows, key=lambda r: r["path"]):
+        relative = row["path"][len(strip):].lstrip("/") if strip else row["path"]
+        target = (dest / relative).resolve()
+        if target != dest and not target.is_relative_to(dest):
+            raise ProjectFileError("path escapes export directory")
+        if row["type"] == "dir":
+            target.mkdir(parents=True, exist_ok=True)
+            continue
+        target.parent.mkdir(parents=True, exist_ok=True)
+        if row.get("is_binary") and row.get("stored_name") and row.get("directory"):
+            shutil.copyfile(PROJECT_FILES_DIR / row["directory"] / row["stored_name"], target)
+        else:
+            target.write_text(row.get("content") or "", encoding="utf-8")
+        written += 1
+    return written
+
+
+def delete_all_project_files(project_uid: str) -> None:
+    if "project_files" not in db.tables:
+        return
+    for row in _table().find(project_uid=project_uid):
+        if row.get("is_binary"):
+            _unlink_blob(row)
+    _table().delete(project_uid=project_uid)

devplacepy/push.py

@@ -0,0 +1,286 @@
+# retoor <retoor@molodetz.nl>
+
+import base64
+import fcntl
+import json
+import logging
+import os
+import random
+import time
+from datetime import datetime, timezone
+from typing import Any
+from urllib.parse import urlparse
+
+import httpx
+import jwt
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric import ec
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+from cryptography.hazmat.primitives.hashes import SHA256
+from cryptography.hazmat.primitives.kdf.hkdf import HKDF
+
+from devplacepy.config import (
+    SECONDS_PER_DAY,
+    VAPID_PRIVATE_KEY_FILE,
+    VAPID_PRIVATE_KEY_PKCS8_FILE,
+    VAPID_PUBLIC_KEY_FILE,
+    VAPID_SUB,
+)
+from devplacepy.database import get_table
+from devplacepy.utils import generate_uid
+
+logger = logging.getLogger(__name__)
+
+JWT_LIFETIME_SECONDS = 60 * 60
+PUSH_TTL_SECONDS = str(SECONDS_PER_DAY)
+DEAD_SUBSCRIPTION_STATUSES = (404, 410)
+ACCEPTED_STATUSES = (200, 201)
+
+
+def generate_private_key() -> None:
+    if not VAPID_PRIVATE_KEY_FILE.exists():
+        private_key = ec.generate_private_key(ec.SECP256R1(), default_backend())
+        pem = private_key.private_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PrivateFormat.TraditionalOpenSSL,
+            encryption_algorithm=serialization.NoEncryption(),
+        )
+        VAPID_PRIVATE_KEY_FILE.write_bytes(pem)
+        logger.info("Generated VAPID private key at %s", VAPID_PRIVATE_KEY_FILE)
+
+
+def generate_pkcs8_private_key() -> None:
+    if not VAPID_PRIVATE_KEY_PKCS8_FILE.exists():
+        private_key = serialization.load_pem_private_key(
+            VAPID_PRIVATE_KEY_FILE.read_bytes(), password=None, backend=default_backend()
+        )
+        pem = private_key.private_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PrivateFormat.PKCS8,
+            encryption_algorithm=serialization.NoEncryption(),
+        )
+        VAPID_PRIVATE_KEY_PKCS8_FILE.write_bytes(pem)
+        logger.info("Generated VAPID PKCS8 private key at %s", VAPID_PRIVATE_KEY_PKCS8_FILE)
+
+
+def generate_public_key() -> None:
+    if not VAPID_PUBLIC_KEY_FILE.exists():
+        private_key = serialization.load_pem_private_key(
+            VAPID_PRIVATE_KEY_FILE.read_bytes(), password=None, backend=default_backend()
+        )
+        pem = private_key.public_key().public_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PublicFormat.SubjectPublicKeyInfo,
+        )
+        VAPID_PUBLIC_KEY_FILE.write_bytes(pem)
+        logger.info("Generated VAPID public key at %s", VAPID_PUBLIC_KEY_FILE)
+
+
+def ensure_certificates() -> None:
+    if VAPID_PRIVATE_KEY_FILE.exists() and VAPID_PRIVATE_KEY_PKCS8_FILE.exists() and VAPID_PUBLIC_KEY_FILE.exists():
+        return
+    lock_path = VAPID_PRIVATE_KEY_FILE.parent / ".vapid.lock"
+    with open(lock_path, "w") as lock:
+        fcntl.flock(lock, fcntl.LOCK_EX)
+        try:
+            generate_private_key()
+            generate_pkcs8_private_key()
+            generate_public_key()
+        finally:
+            fcntl.flock(lock, fcntl.LOCK_UN)
+
+
+def hkdf(input_key: bytes, salt: bytes, info: bytes, length: int) -> bytes:
+    return HKDF(
+        algorithm=SHA256(),
+        length=length,
+        salt=salt,
+        info=info,
+        backend=default_backend(),
+    ).derive(input_key)
+
+
+def browser_base64(data: bytes) -> str:
+    return base64.urlsafe_b64encode(data).decode("utf-8").rstrip("=")
+
+
+_keys: dict[str, Any] = {}
+
+
+def _load_keys() -> dict[str, Any]:
+    if _keys:
+        return _keys
+    ensure_certificates()
+    private_key = serialization.load_pem_private_key(
+        VAPID_PRIVATE_KEY_FILE.read_bytes(), password=None, backend=default_backend()
+    )
+    public_key = serialization.load_pem_public_key(
+        VAPID_PUBLIC_KEY_FILE.read_bytes(), backend=default_backend()
+    )
+    uncompressed_point = public_key.public_bytes(
+        encoding=serialization.Encoding.X962,
+        format=serialization.PublicFormat.UncompressedPoint,
+    )
+    _keys["private_key_pem"] = private_key.private_bytes(
+        encoding=serialization.Encoding.PEM,
+        format=serialization.PrivateFormat.TraditionalOpenSSL,
+        encryption_algorithm=serialization.NoEncryption(),
+    )
+    _keys["public_key_point"] = uncompressed_point
+    _keys["public_key_base64"] = browser_base64(uncompressed_point)
+    logger.debug("Loaded VAPID key material into cache")
+    return _keys
+
+
+def public_key_standard_b64() -> str:
+    point = _load_keys()["public_key_point"]
+    return base64.b64encode(point).decode("utf-8").rstrip("=")
+
+
+def create_notification_authorization(push_url: str) -> str:
+    target = urlparse(push_url)
+    audience = f"{target.scheme}://{target.netloc}"
+    issued_at = int(time.time())
+    return jwt.encode(
+        {
+            "sub": VAPID_SUB,
+            "aud": audience,
+            "exp": issued_at + JWT_LIFETIME_SECONDS,
+            "nbf": issued_at,
+            "iat": issued_at,
+            "jti": generate_uid(),
+        },
+        _load_keys()["private_key_pem"],
+        algorithm="ES256",
+    )
+
+
+def create_notification_info_with_payload(
+    endpoint: str, auth: str, p256dh: str, payload: str
+) -> dict[str, Any]:
+    message_private_key = ec.generate_private_key(ec.SECP256R1(), default_backend())
+    message_public_key_bytes = message_private_key.public_key().public_bytes(
+        encoding=serialization.Encoding.X962,
+        format=serialization.PublicFormat.UncompressedPoint,
+    )
+
+    salt = os.urandom(16)
+
+    user_key_bytes = base64.urlsafe_b64decode(p256dh + "==")
+    shared_secret = message_private_key.exchange(
+        ec.ECDH(),
+        ec.EllipticCurvePublicKey.from_encoded_point(ec.SECP256R1(), user_key_bytes),
+    )
+
+    encryption_key = hkdf(
+        shared_secret,
+        base64.urlsafe_b64decode(auth + "=="),
+        b"Content-Encoding: auth\x00",
+        32,
+    )
+
+    context = (
+        b"P-256\x00"
+        + len(user_key_bytes).to_bytes(2, "big")
+        + user_key_bytes
+        + len(message_public_key_bytes).to_bytes(2, "big")
+        + message_public_key_bytes
+    )
+
+    nonce = hkdf(encryption_key, salt, b"Content-Encoding: nonce\x00" + context, 12)
+    content_encryption_key = hkdf(
+        encryption_key, salt, b"Content-Encoding: aesgcm\x00" + context, 16
+    )
+
+    padding_length = random.randint(0, 16)
+    padding = padding_length.to_bytes(2, "big") + b"\x00" * padding_length
+    data = AESGCM(content_encryption_key).encrypt(
+        nonce, padding + payload.encode("utf-8"), None
+    )
+
+    return {
+        "headers": {
+            "Authorization": f"WebPush {create_notification_authorization(endpoint)}",
+            "Crypto-Key": f"dh={browser_base64(message_public_key_bytes)}; p256ecdsa={_load_keys()['public_key_base64']}",
+            "Encryption": f"salt={browser_base64(salt)}",
+            "Content-Encoding": "aesgcm",
+            "Content-Length": str(len(data)),
+            "Content-Type": "application/octet-stream",
+        },
+        "data": data,
+    }
+
+
+def _mark_subscription_dead(subscription_id: int) -> None:
+    get_table("push_registration").update(
+        {"id": subscription_id, "deleted_at": datetime.now(timezone.utc).isoformat()},
+        ["id"],
+    )
+    logger.info("Soft-deleted dead push subscription id=%s", subscription_id)
+
+
+async def notify_user(user_uid: str, payload: dict[str, Any]) -> None:
+    registrations = list(
+        get_table("push_registration").find(user_uid=user_uid, deleted_at=None)
+    )
+    if not registrations:
+        logger.debug("No active push subscriptions for user %s", user_uid)
+        return
+
+    body = json.dumps(payload)
+    async with httpx.AsyncClient(timeout=10.0) as client:
+        for subscription in registrations:
+            endpoint = subscription["endpoint"]
+            try:
+                notification_info = create_notification_info_with_payload(
+                    endpoint,
+                    subscription["key_auth"],
+                    subscription["key_p256dh"],
+                    body,
+                )
+                headers = {**notification_info["headers"], "TTL": PUSH_TTL_SECONDS}
+                response = await client.post(
+                    endpoint, headers=headers, content=notification_info["data"]
+                )
+            except (httpx.HTTPError, ValueError) as exc:
+                logger.warning("Push error for %s via %s: %s", user_uid, endpoint, exc)
+                continue
+
+            if response.status_code in ACCEPTED_STATUSES:
+                logger.debug("Push delivered to %s via %s", user_uid, endpoint)
+            elif response.status_code in DEAD_SUBSCRIPTION_STATUSES:
+                _mark_subscription_dead(subscription["id"])
+            else:
+                logger.warning(
+                    "Push rejected (%s) for %s via %s", response.status_code, user_uid, endpoint
+                )
+
+
+async def register(
+    user_uid: str, endpoint: str, key_auth: str, key_p256dh: str
+) -> tuple[dict[str, Any], bool]:
+    table = get_table("push_registration")
+    existing = table.find_one(
+        user_uid=user_uid,
+        endpoint=endpoint,
+        key_auth=key_auth,
+        key_p256dh=key_p256dh,
+        deleted_at=None,
+    )
+    if existing:
+        logger.debug("Push subscription already registered for user %s", user_uid)
+        return existing, False
+
+    record = {
+        "uid": generate_uid(),
+        "user_uid": user_uid,
+        "endpoint": endpoint,
+        "key_auth": key_auth,
+        "key_p256dh": key_p256dh,
+        "created_at": datetime.now(timezone.utc).isoformat(),
+        "deleted_at": None,
+    }
+    table.insert(record)
+    logger.info("Registered push subscription for user %s", user_uid)
+    return record, True

devplacepy/responses.py

@@ -0,0 +1,42 @@
+import logging
+
+from fastapi import Request
+from fastapi.responses import JSONResponse, RedirectResponse
+
+from devplacepy.schemas import ActionResult
+
+logger = logging.getLogger(__name__)
+
+
+def wants_json(request: Request) -> bool:
+    # NOTE: `X-Requested-With: fetch` is intentionally NOT a trigger here. The browser
+    # frontend sends it on form/engagement fetches and relies on the legacy HTML/redirect
+    # behaviour (the four engagement endpoints handle that header themselves). JSON is
+    # requested via the Accept or Content-Type header so existing flows are unchanged.
+    content_type = request.headers.get("content-type", "")
+    if content_type.startswith("application/json"):
+        return True
+    accept = request.headers.get("accept", "")
+    return "application/json" in accept and "text/html" not in accept
+
+
+def respond(request: Request, template: str, context: dict, *, model):
+    if wants_json(request):
+        try:
+            payload = model.model_validate(context).model_dump(mode="json")
+        except Exception as exc:
+            logger.warning("JSON serialization failed for %s: %s", template, exc)
+            return json_error(500, "Could not serialize response")
+        return JSONResponse(payload)
+    from devplacepy.templating import templates
+    return templates.TemplateResponse(request, template, context)
+
+
+def action_result(request: Request, redirect_url: str, *, data=None, status_code: int = 302):
+    if wants_json(request):
+        return JSONResponse(ActionResult(ok=True, redirect=redirect_url, data=data).model_dump(mode="json"))
+    return RedirectResponse(url=redirect_url, status_code=status_code)
+
+
+def json_error(status_code: int, message: str, **extra) -> JSONResponse:
+    return JSONResponse({"error": {"status": status_code, "message": message, **extra}}, status_code=status_code)

devplacepy/routers/admin.py

@@ -1,11 +1,17 @@
 import logging
-from datetime import datetime
-from fastapi import APIRouter, Request
-from fastapi.responses import HTMLResponse, RedirectResponse
-from devplacepy.database import get_table, db, build_pagination
+from typing import Annotated
+from fastapi import APIRouter, Request, Form
+from devplacepy.models import AdminRoleForm, AdminPasswordForm, AdminSettingsForm
+from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse
+from devplacepy.database import get_table, build_pagination, get_post_counts_by_user_uids, get_news_images_by_uids, clear_settings_cache, get_platform_analytics
 from devplacepy.templating import templates
-from devplacepy.utils import require_admin, hash_password, generate_uid, time_ago
+from devplacepy.utils import require_admin, hash_password, generate_uid, time_ago, clear_user_cache, get_current_user, is_admin
 from devplacepy.seo import base_seo_context, site_url, website_schema
+from devplacepy.responses import respond, action_result
+from devplacepy.schemas import AdminUsersOut, AdminNewsOut, AdminSettingsOut, GatewayUsageOut
+from devplacepy.services.manager import service_manager
+from devplacepy.services.openai_gateway.analytics import build_analytics, build_user_usage
+from devplacepy.services.openai_gateway.usage import pricing_from_cfg
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
@@ -14,24 +20,81 @@ router = APIRouter()
 @router.get("", response_class=HTMLResponse)
 async def admin_index(request: Request):
     require_admin(request)
-    return RedirectResponse(url="/admin/users", status_code=302)
+    return action_result(request, "/admin/users")
+
+
+@router.get("/analytics")
+async def admin_analytics(request: Request, top_n: int = 10):
+    user = get_current_user(request)
+    if not user:
+        return JSONResponse({"error": "Authentication required"}, status_code=401)
+    if not is_admin(user):
+        return JSONResponse({"error": "Admin access required"}, status_code=403)
+    return JSONResponse(get_platform_analytics(top_n))
+
+
+@router.get("/ai-usage", response_class=HTMLResponse)
+async def admin_ai_usage(request: Request):
+    admin = require_admin(request)
+    base = site_url(request)
+    seo_ctx = base_seo_context(
+        request,
+        title="AI usage - Admin",
+        description="AI gateway token usage, cost, latency, and reliability metrics.",
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {"name": "Admin", "url": "/admin"},
+            {"name": "AI usage", "url": "/admin/ai-usage"},
+        ],
+        schemas=[website_schema(base)],
+    )
+    return respond(request, "ai_usage.html", {
+        **seo_ctx,
+        "request": request,
+        "user": admin,
+        "admin_section": "ai-usage",
+    }, model=GatewayUsageOut)
+
+
+@router.get("/ai-usage/data")
+async def admin_ai_usage_data(request: Request, hours: int = 48, top_n: int = 10):
+    user = get_current_user(request)
+    if not user:
+        return JSONResponse({"error": "Authentication required"}, status_code=401)
+    if not is_admin(user):
+        return JSONResponse({"error": "Admin access required"}, status_code=403)
+    service = service_manager.get_service("openai")
+    pricing = pricing_from_cfg(service.get_config()) if service is not None else None
+    return JSONResponse(build_analytics(hours, top_n=top_n, pricing=pricing))
+
+
+@router.get("/users/{uid}/ai-usage")
+async def admin_user_ai_usage(request: Request, uid: str, hours: int = 24):
+    user = get_current_user(request)
+    if not user:
+        return JSONResponse({"error": "Authentication required"}, status_code=401)
+    if not is_admin(user):
+        return JSONResponse({"error": "Admin access required"}, status_code=403)
+    service = service_manager.get_service("openai")
+    pricing = pricing_from_cfg(service.get_config()) if service is not None else None
+    return JSONResponse(build_user_usage(uid, hours=hours, pricing=pricing))
 
 
 @router.get("/users", response_class=HTMLResponse)
 async def admin_users(request: Request, page: int = 1):
     admin = require_admin(request)
     users_table = get_table("users")
-    total = len(list(users_table.all()))
+    total = users_table.count()
     pagination = build_pagination(page, total)
     offset = (pagination["page"] - 1) * pagination["per_page"]
     page_users = list(users_table.find(order_by=["-created_at"], _limit=pagination["per_page"], _offset=offset))
+    post_counts = get_post_counts_by_user_uids([u["uid"] for u in page_users])
     for u in page_users:
-        posts_count = len(list(get_table("posts").find(user_uid=u["uid"])))
-        u["posts_count"] = posts_count
+        u["posts_count"] = post_counts.get(u["uid"], 0)
     base = site_url(request)
     seo_ctx = base_seo_context(
         request,
-        title="Users — Admin",
+        title="Users - Admin",
         description="Manage DevPlace users.",
         breadcrumbs=[
             {"name": "Home", "url": "/feed"},
@@ -40,56 +103,78 @@ async def admin_users(request: Request, page: int = 1):
         ],
         schemas=[website_schema(base)],
     )
-    return templates.TemplateResponse("admin_users.html", {
+    return respond(request, "admin_users.html", {
         **seo_ctx,
         "request": request,
         "user": admin,
         "users": page_users,
         "pagination": pagination,
         "admin_section": "users",
-    })
+    }, model=AdminUsersOut)
 
 
 @router.post("/users/{uid}/role")
-async def admin_user_role(request: Request, uid: str):
+async def admin_user_role(request: Request, uid: str, data: Annotated[AdminRoleForm, Form()]):
     admin = require_admin(request)
-    form = await request.form()
-    role = form.get("role", "").strip().capitalize()
-    if role not in ("Member", "Admin"):
-        return RedirectResponse(url="/admin/users", status_code=302)
+    role = data.role.capitalize()
     if uid == admin["uid"]:
-        return RedirectResponse(url="/admin/users", status_code=302)
+        return action_result(request, "/admin/users")
     users = get_table("users")
     users.update({"uid": uid, "role": role}, ["uid"])
+    clear_user_cache(uid)
     logger.info(f"Admin {admin['username']} set user {uid} role to {role}")
-    return RedirectResponse(url="/admin/users", status_code=302)
+    return action_result(request, "/admin/users")
 
 
 @router.post("/users/{uid}/password")
-async def admin_user_password(request: Request, uid: str):
+async def admin_user_password(request: Request, uid: str, data: Annotated[AdminPasswordForm, Form()]):
     admin = require_admin(request)
-    form = await request.form()
-    password = form.get("password", "")
-    if len(password) < 6:
-        return RedirectResponse(url="/admin/users", status_code=302)
     users = get_table("users")
-    users.update({"uid": uid, "password_hash": hash_password(password)}, ["uid"])
+    users.update({"uid": uid, "password_hash": hash_password(data.password)}, ["uid"])
     logger.info(f"Admin {admin['username']} changed password for user {uid}")
-    return RedirectResponse(url="/admin/users", status_code=302)
+    return action_result(request, "/admin/users")
 
 
 @router.post("/users/{uid}/toggle")
 async def admin_user_toggle(request: Request, uid: str):
     admin = require_admin(request)
     if uid == admin["uid"]:
-        return RedirectResponse(url="/admin/users", status_code=302)
+        return action_result(request, "/admin/users")
     users = get_table("users")
     user = users.find_one(uid=uid)
     if user:
         new_state = not user.get("is_active", True)
         users.update({"uid": uid, "is_active": new_state}, ["uid"])
+        clear_user_cache(uid)
         logger.info(f"Admin {admin['username']} {'disabled' if not new_state else 'enabled'} user {uid}")
-    return RedirectResponse(url="/admin/users", status_code=302)
+    return action_result(request, "/admin/users")
+
+
+@router.post("/users/{uid}/reset-ai-quota")
+async def admin_user_reset_ai_quota(request: Request, uid: str):
+    admin = require_admin(request)
+    devii = service_manager.get_service("devii")
+    removed = devii.reset_quota("user", uid) if devii is not None else 0
+    logger.info(f"Admin {admin['username']} reset AI quota for user {uid} ({removed} ledger rows)")
+    return action_result(request, "/admin/users")
+
+
+@router.post("/ai-quota/reset-guests")
+async def admin_reset_guest_ai_quota(request: Request):
+    admin = require_admin(request)
+    devii = service_manager.get_service("devii")
+    removed = devii.reset_guest_quotas() if devii is not None else 0
+    logger.info(f"Admin {admin['username']} reset all guest AI quotas ({removed} ledger rows)")
+    return action_result(request, "/admin/ai-usage")
+
+
+@router.post("/ai-quota/reset-all")
+async def admin_reset_all_ai_quota(request: Request):
+    admin = require_admin(request)
+    devii = service_manager.get_service("devii")
+    removed = devii.reset_all_quotas() if devii is not None else 0
+    logger.info(f"Admin {admin['username']} reset ALL AI quotas ({removed} ledger rows)")
+    return action_result(request, "/admin/ai-usage")
 
 
 @router.get("/settings", response_class=HTMLResponse)
@@ -100,7 +185,7 @@ async def admin_settings(request: Request):
     base = site_url(request)
     seo_ctx = base_seo_context(
         request,
-        title="Settings — Admin",
+        title="Settings - Admin",
         description="Manage DevPlace site settings.",
         breadcrumbs=[
             {"name": "Home", "url": "/feed"},
@@ -109,42 +194,39 @@ async def admin_settings(request: Request):
         ],
         schemas=[website_schema(base)],
     )
-    return templates.TemplateResponse("admin_settings.html", {
+    return respond(request, "admin_settings.html", {
         **seo_ctx,
         "request": request,
         "user": admin,
         "settings": all_settings,
         "admin_section": "settings",
-    })
+    }, model=AdminSettingsOut)
 
 
 @router.get("/news", response_class=HTMLResponse)
 async def admin_news(request: Request, page: int = 1):
     admin = require_admin(request)
     news_table = get_table("news")
-    total = len(list(news_table.all()))
+    total = news_table.count()
     pagination = build_pagination(page, total)
     offset = (pagination["page"] - 1) * pagination["per_page"]
     page_articles = list(news_table.find(order_by=["-synced_at"], _limit=pagination["per_page"], _offset=offset))
-    images_table = get_table("news_images")
+    images_by_news = get_news_images_by_uids([a["uid"] for a in page_articles])
 
     enriched = []
     for a in page_articles:
-        has_image = False
-        if "news_images" in db.tables:
-            has_image = images_table.find_one(news_uid=a["uid"]) is not None
         enriched.append({
             "article": a,
             "time_ago": time_ago(a["synced_at"]),
             "synced_at": a.get("synced_at", ""),
             "grade": a.get("grade", 0),
-            "has_image": has_image,
+            "has_image": a["uid"] in images_by_news,
         })
 
     base = site_url(request)
     seo_ctx = base_seo_context(
         request,
-        title="News — Admin",
+        title="News - Admin",
         description="Manage DevPlace news articles.",
         breadcrumbs=[
             {"name": "Home", "url": "/feed"},
@@ -153,14 +235,14 @@ async def admin_news(request: Request, page: int = 1):
         ],
         schemas=[website_schema(base)],
     )
-    return templates.TemplateResponse("admin_news.html", {
+    return respond(request, "admin_news.html", {
         **seo_ctx,
         "request": request,
         "user": admin,
         "articles": enriched,
         "pagination": pagination,
         "admin_section": "news",
-    })
+    }, model=AdminNewsOut)
 
 
 @router.post("/news/{uid}/toggle")
@@ -172,7 +254,7 @@ async def admin_news_toggle(request: Request, uid: str):
         current = article.get("featured", 0)
         news_table.update({"uid": uid, "featured": 0 if current else 1}, ["uid"])
         logger.info(f"Admin {admin['username']} toggled news {uid} featured={'off' if current else 'on'}")
-    return RedirectResponse(url="/admin/news", status_code=302)
+    return action_result(request, "/admin/news")
 
 
 @router.post("/news/{uid}/publish")
@@ -185,7 +267,7 @@ async def admin_news_publish(request: Request, uid: str):
         new_status = "draft" if current == "published" else "published"
         news_table.update({"uid": uid, "status": new_status}, ["uid"])
         logger.info(f"Admin {admin['username']} set news {uid} status={new_status}")
-    return RedirectResponse(url="/admin/news", status_code=302)
+    return action_result(request, "/admin/news")
 
 
 @router.post("/news/{uid}/landing")
@@ -197,7 +279,7 @@ async def admin_news_landing(request: Request, uid: str):
         current = article.get("show_on_landing", 0)
         news_table.update({"uid": uid, "show_on_landing": 0 if current else 1}, ["uid"])
         logger.info(f"Admin {admin['username']} toggled news {uid} landing={'on' if not current else 'off'}")
-    return RedirectResponse(url="/admin/news", status_code=302)
+    return action_result(request, "/admin/news")
 
 
 @router.post("/news/{uid}/delete")
@@ -211,19 +293,21 @@ async def admin_news_delete(request: Request, uid: str):
             images_table.delete(id=img["id"])
         news_table.delete(id=article["id"])
         logger.info(f"Admin {admin['username']} deleted news {uid}")
-    return RedirectResponse(url="/admin/news", status_code=302)
+    return action_result(request, "/admin/news")
 
 
 @router.post("/settings")
-async def admin_settings_save(request: Request):
+async def admin_settings_save(request: Request, data: Annotated[AdminSettingsForm, Form()]):
     admin = require_admin(request)
-    form = await request.form()
     settings = get_table("site_settings")
-    for key, value in form.multi_items():
+    for key, value in data.model_dump().items():
         existing = settings.find_one(key=key)
         if existing:
+            if value == "":
+                continue
             settings.update({"id": existing["id"], "key": key, "value": value}, ["id"])
         else:
             settings.insert({"uid": generate_uid(), "key": key, "value": value})
+    clear_settings_cache()
     logger.info(f"Admin {admin['username']} updated settings")
-    return RedirectResponse(url="/admin/settings", status_code=302)
+    return action_result(request, "/admin/settings")

devplacepy/routers/auth.py

@@ -1,13 +1,18 @@
 import hashlib
 import secrets
 import logging
-from datetime import datetime, timedelta
-from fastapi import APIRouter, Request
-from fastapi.responses import RedirectResponse, HTMLResponse
-from devplacepy.database import get_table
+from typing import Annotated
+from datetime import datetime, timedelta, timezone
+from fastapi import APIRouter, Request, Form
+from fastapi.responses import RedirectResponse, HTMLResponse, JSONResponse
+from devplacepy.database import get_table, get_setting, get_int_setting
 from devplacepy.templating import templates
-from devplacepy.utils import hash_password, verify_password, create_session, generate_uid, get_current_user
+from devplacepy.utils import hash_password, verify_password, create_session, generate_uid, get_current_user, award_badge, clear_session_cache, safe_next
 from devplacepy.seo import base_seo_context
+from devplacepy.models import SignupForm, LoginForm, ForgotPasswordForm, ResetPasswordForm
+from devplacepy.config import SECONDS_PER_DAY
+from devplacepy.responses import respond, action_result, wants_json, json_error
+from devplacepy.schemas import AuthPageOut
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
@@ -17,50 +22,47 @@ router = APIRouter()
 async def signup_page(request: Request):
     user = get_current_user(request)
     if user:
-        return RedirectResponse(url="/feed", status_code=302)
+        return action_result(request, "/feed")
     seo_ctx = base_seo_context(
         request,
         title="Join DevPlace",
         description="Create your DevPlace account and start connecting with developers.",
         robots="noindex,nofollow",
     )
-    return templates.TemplateResponse("signup.html", {**seo_ctx, "request": request})
+    registration_closed = get_setting("registration_open", "1") != "1"
+    return respond(request, "signup.html", {**seo_ctx, "request": request, "page": "signup", "registration_closed": registration_closed}, model=AuthPageOut)
 
 
 @router.get("/login", response_class=HTMLResponse)
-async def login_page(request: Request):
+async def login_page(request: Request, next: str | None = None):
     user = get_current_user(request)
     if user:
-        return RedirectResponse(url="/feed", status_code=302)
+        return action_result(request, safe_next(next))
     seo_ctx = base_seo_context(
         request,
         title="Sign In",
         description="Sign in to DevPlace to connect with developers.",
         robots="noindex,nofollow",
     )
-    return templates.TemplateResponse("login.html", {**seo_ctx, "request": request})
+    return respond(request, "login.html", {**seo_ctx, "request": request, "page": "login", "next_url": safe_next(next, "")}, model=AuthPageOut)
 
 
 @router.post("/signup")
-async def signup(request: Request):
-    form = await request.form()
-    username = form.get("username", "").strip()
-    email = form.get("email", "").strip().lower()
-    password = form.get("password", "")
-    confirm_password = form.get("confirm_password", "")
+async def signup(request: Request, data: Annotated[SignupForm, Form()]):
+    username = data.username
+    email = data.email.strip().lower()
+    password = data.password
+
+    if get_setting("registration_open", "1") != "1":
+        if wants_json(request):
+            return json_error(403, "Registration is closed")
+        seo_ctx = base_seo_context(request, title="Join DevPlace", robots="noindex,nofollow")
+        return templates.TemplateResponse(
+            request, "signup.html",
+            {**seo_ctx, "request": request, "registration_closed": True},
+        )
 
     errors = []
-    if len(username) < 3 or len(username) > 32:
-        errors.append("Username must be between 3 and 32 characters")
-    if not username.isascii() or not all(c.isalnum() or c in ("-", "_") for c in username):
-        errors.append("Username can only contain letters, numbers, hyphens, and underscores")
-    if not email or "@" not in email or len(email) > 255:
-        errors.append("Valid email is required")
-    if len(password) < 6:
-        errors.append("Password must be at least 6 characters")
-    if password != confirm_password:
-        errors.append("Passwords do not match")
-
     users = get_table("users")
     if users.find_one(username=username):
         errors.append("Username already taken")
@@ -68,18 +70,21 @@ async def signup(request: Request):
         errors.append("Email already registered")
 
     if errors:
+        if wants_json(request):
+            return json_error(400, "; ".join(errors), errors=errors)
         seo_ctx = base_seo_context(request, title="Join DevPlace", robots="noindex,nofollow")
         return templates.TemplateResponse(
-            "signup.html",
+            request, "signup.html",
             {**seo_ctx, "request": request, "errors": errors, "username": username, "email": email},
         )
 
     uid = generate_uid()
-    is_first = len(list(users.all())) == 0
+    is_first = users.count() == 0
     users.insert({
         "uid": uid,
         "username": username,
         "email": email,
+        "api_key": generate_uid(),
         "password_hash": hash_password(password),
         "bio": "",
         "location": "",
@@ -90,51 +95,45 @@ async def signup(request: Request):
         "level": 1,
         "xp": 0,
         "stars": 0,
-        "created_at": datetime.utcnow().isoformat(),
+        "created_at": datetime.now(timezone.utc).isoformat(),
     })
 
-    badges = get_table("badges")
-    badges.insert({
-        "uid": generate_uid(),
-        "user_uid": uid,
-        "badge_name": "Member",
-        "created_at": datetime.utcnow().isoformat(),
-    })
+    award_badge(uid, "Member")
 
-    token = create_session(uid)
-    response = RedirectResponse(url="/feed", status_code=302)
-    response.set_cookie(key="session", value=token, max_age=86400 * 7, httponly=True, samesite="lax")
+    max_age = max(1, get_int_setting("session_max_age_days", 7)) * SECONDS_PER_DAY
+    token = create_session(uid, max_age)
+    response = action_result(request, "/feed", data={"username": username})
+    response.set_cookie(key="session", value=token, max_age=max_age, httponly=True, samesite="lax")
     logger.info(f"User {username} signed up")
     return response
 
 
 @router.post("/login")
-async def login(request: Request):
-    form = await request.form()
-    email = form.get("email", "").strip().lower()
-    password = form.get("password", "")
-    remember_me = form.get("remember_me") == "on"
+async def login(request: Request, data: Annotated[LoginForm, Form()]):
+    email = data.email.strip().lower()
+    password = data.password
+    remember_me = data.remember_me == "on"
 
     errors = []
-    if not email or not password:
-        errors.append("Email and password are required")
-
     users = get_table("users")
-    user = users.find_one(email=email) if not errors else None
+    user = users.find_one(email=email)
 
     if not user or not verify_password(password, user["password_hash"]):
         errors.append("Invalid email or password")
 
     if errors:
+        if wants_json(request):
+            return json_error(401, "; ".join(errors), errors=errors)
         seo_ctx = base_seo_context(request, title="Sign In", robots="noindex,nofollow")
         return templates.TemplateResponse(
-            "login.html",
-            {**seo_ctx, "request": request, "errors": errors, "email": email},
+            request, "login.html",
+            {**seo_ctx, "request": request, "errors": errors, "email": email, "next_url": safe_next(data.next, "")},
         )
 
-    token = create_session(user["uid"])
-    max_age = 86400 * 30 if remember_me else 86400 * 7
-    response = RedirectResponse(url="/feed", status_code=302)
+    days = get_int_setting("session_remember_days", 30) if remember_me else get_int_setting("session_max_age_days", 7)
+    max_age = max(1, days) * SECONDS_PER_DAY
+    token = create_session(user["uid"], max_age)
+    response = action_result(request, safe_next(data.next), data={"username": user["username"]})
     response.set_cookie(key="session", value=token, max_age=max_age, httponly=True, samesite="lax")
     logger.info(f"User {user['username']} logged in")
     return response
@@ -147,22 +146,13 @@ async def forgot_password_page(request: Request):
         title="Reset Password",
         robots="noindex,nofollow",
     )
-    return templates.TemplateResponse("forgot_password.html", {**seo_ctx, "request": request})
+    return respond(request, "forgot_password.html", {**seo_ctx, "request": request, "page": "forgot-password"}, model=AuthPageOut)
 
 
 @router.post("/forgot-password")
-async def forgot_password(request: Request):
-    form = await request.form()
-    email = form.get("email", "").strip().lower()
-    errors = []
-    if not email or "@" not in email:
-        errors.append("Valid email is required")
-
+async def forgot_password(request: Request, data: Annotated[ForgotPasswordForm, Form()]):
+    email = data.email.strip().lower()
     seo_ctx = base_seo_context(request, title="Reset Password", robots="noindex,nofollow")
-    if errors:
-        return templates.TemplateResponse("forgot_password.html", {
-            **seo_ctx, "request": request, "errors": errors,
-        })
 
     users = get_table("users")
     user = users.find_one(email=email)
@@ -174,13 +164,15 @@ async def forgot_password(request: Request):
             "uid": generate_uid(),
             "user_uid": user["uid"],
             "token": token_hash,
-            "expires_at": (datetime.utcnow() + timedelta(hours=1)).isoformat(),
+            "expires_at": (datetime.now(timezone.utc) + timedelta(hours=1)).isoformat(),
             "used": False,
-            "created_at": datetime.utcnow().isoformat(),
+            "created_at": datetime.now(timezone.utc).isoformat(),
         })
         logger.info(f"Password reset requested for {email}")
 
-    return templates.TemplateResponse("forgot_password.html", {
+    if wants_json(request):
+        return JSONResponse({"ok": True, "sent": True})
+    return templates.TemplateResponse(request, "forgot_password.html", {
         **seo_ctx, "request": request, "sent": True,
     })
 
@@ -192,44 +184,40 @@ async def reset_password_page(request: Request, token: str):
         title="Set New Password",
         robots="noindex,nofollow",
     )
-    return templates.TemplateResponse("reset_password.html", {
-        **seo_ctx, "request": request, "token": token,
-    })
+    return respond(request, "reset_password.html", {
+        **seo_ctx, "request": request, "page": "reset-password", "token": token,
+    }, model=AuthPageOut)
 
 
 @router.post("/reset-password/{token}")
-async def reset_password(request: Request, token: str):
-    form = await request.form()
-    password = form.get("password", "")
-    confirm = form.get("confirm_password", "")
-
+async def reset_password(request: Request, token: str, data: Annotated[ResetPasswordForm, Form()]):
+    password = data.password
     errors = []
-    if len(password) < 6:
-        errors.append("Password must be at least 6 characters")
-    if password != confirm:
-        errors.append("Passwords do not match")
-
-    if errors:
-        seo_ctx = base_seo_context(request, title="Set New Password", robots="noindex,nofollow")
-        return templates.TemplateResponse("reset_password.html", {
-            **seo_ctx, "request": request, "token": token, "errors": errors,
-        })
 
     resets = get_table("password_resets")
     users = get_table("users")
     token_hash = hashlib.sha256(token.encode()).hexdigest()
     matched = resets.find_one(token=token_hash, used=False)
 
-    if not matched or datetime.fromisoformat(matched["expires_at"]) < datetime.utcnow():
+    expired = False
+    if matched:
+        expires = datetime.fromisoformat(matched["expires_at"])
+        if expires.tzinfo is None:
+            expires = expires.replace(tzinfo=timezone.utc)
+        expired = expires < datetime.now(timezone.utc)
+
+    if not matched or expired:
         errors.append("Invalid or expired reset token")
-        return templates.TemplateResponse("reset_password.html", {
+        if wants_json(request):
+            return json_error(400, errors[0], errors=errors)
+        return templates.TemplateResponse(request, "reset_password.html", {
             "request": request, "token": token, "errors": errors,
         })
 
     users.update({"uid": matched["user_uid"], "password_hash": hash_password(password)}, ["uid"])
     resets.update({"id": matched["id"], "used": True}, ["id"])
     logger.info(f"Password reset completed for user {matched['user_uid']}")
-    return RedirectResponse(url="/auth/login", status_code=302)
+    return action_result(request, "/auth/login")
 
 
 @router.get("/logout")
@@ -240,6 +228,7 @@ async def logout(request: Request):
         session = sessions.find_one(session_token=token)
         if session:
             sessions.delete(id=session["id"])
-    response = RedirectResponse(url="/", status_code=302)
+        clear_session_cache(token)
+    response = action_result(request, "/")
     response.delete_cookie("session")
     return response

devplacepy/routers/avatar.py

@@ -1,20 +1,29 @@
+import hashlib
 import logging
 from fastapi import APIRouter, Request
 from fastapi.responses import Response
 from devplacepy.avatar import generate_avatar_svg
+from devplacepy.cache import TTLCache
+from devplacepy.config import SECONDS_PER_DAY
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
-_cache = {}
+_cache = TTLCache(ttl=SECONDS_PER_DAY, max_size=4096)
+_CACHE_CONTROL = f"public, max-age={SECONDS_PER_DAY}, immutable"
 
 
 @router.get("/{style}/{seed}")
 async def avatar_proxy(request: Request, style: str, seed: str, size: int = 128):
     cache_key = f"{seed}:{size}"
-    if cache_key in _cache:
-        return Response(content=_cache[cache_key], media_type="image/svg+xml")
+    etag = '"' + hashlib.md5(cache_key.encode("utf-8")).hexdigest() + '"'
+    headers = {"ETag": etag, "Cache-Control": _CACHE_CONTROL}
 
-    svg = generate_avatar_svg(seed)
-    _cache[cache_key] = svg
-    return Response(content=svg, media_type="image/svg+xml")
+    if request.headers.get("if-none-match") == etag:
+        return Response(status_code=304, headers=headers)
+
+    svg = _cache.get(cache_key)
+    if svg is None:
+        svg = generate_avatar_svg(seed)
+        _cache.set(cache_key, svg)
+    return Response(content=svg, media_type="image/svg+xml", headers=headers)

devplacepy/routers/bookmarks.py

@@ -0,0 +1,89 @@
+# retoor <retoor@molodetz.nl>
+import logging
+from datetime import datetime, timezone
+from fastapi import APIRouter, Request
+from fastapi.responses import RedirectResponse, JSONResponse, HTMLResponse
+from devplacepy.database import get_table, db, paginate, resolve_object_url
+from devplacepy.templating import templates
+from devplacepy.utils import generate_uid, require_user, time_ago
+from devplacepy.seo import base_seo_context
+from devplacepy.responses import respond
+from devplacepy.schemas import SavedOut
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+BOOKMARKABLE: set[str] = {"post", "gist", "project", "news"}
+
+TABLE_BY_TYPE: dict[str, str] = {"post": "posts", "gist": "gists", "project": "projects", "news": "news"}
+
+LABEL_BY_TYPE: dict[str, str] = {"post": "Post", "gist": "Gist", "project": "Project", "news": "Article"}
+
+
+@router.get("/saved", response_class=HTMLResponse)
+async def saved_page(request: Request, before: str = None):
+    user = require_user(request)
+    bookmarks = get_table("bookmarks")
+    rows, next_cursor = paginate(bookmarks, before=before, user_uid=user["uid"])
+
+    uids_by_type: dict[str, list] = {}
+    for row in rows:
+        uids_by_type.setdefault(row["target_type"], []).append(row["target_uid"])
+
+    resolved: dict[tuple, dict] = {}
+    for target_type, uids in uids_by_type.items():
+        table_name = TABLE_BY_TYPE.get(target_type)
+        if not table_name or table_name not in db.tables:
+            continue
+        table = get_table(table_name)
+        for obj in table.find(table.table.columns.uid.in_(uids)):
+            resolved[(target_type, obj["uid"])] = obj
+
+    items = []
+    for row in rows:
+        obj = resolved.get((row["target_type"], row["target_uid"]))
+        if not obj:
+            continue
+        title = obj.get("title") or (obj.get("content", "") or "")[:80] or "Untitled"
+        items.append({
+            "target_type": row["target_type"],
+            "type_label": LABEL_BY_TYPE.get(row["target_type"], row["target_type"].title()),
+            "title": title,
+            "url": resolve_object_url(row["target_type"], row["target_uid"]),
+            "time_ago": time_ago(row["created_at"]),
+        })
+
+    seo_ctx = base_seo_context(request, title="Saved", description="Your saved content on DevPlace.", robots="noindex,nofollow")
+    return respond(request, "saved.html", {
+        **seo_ctx,
+        "request": request,
+        "user": user,
+        "items": items,
+        "next_cursor": next_cursor,
+    }, model=SavedOut)
+
+
+@router.post("/{target_type}/{target_uid}")
+async def toggle_bookmark(request: Request, target_type: str, target_uid: str):
+    user = require_user(request)
+    if target_type not in BOOKMARKABLE:
+        return JSONResponse({"error": "Invalid target"}, status_code=400)
+
+    bookmarks = get_table("bookmarks")
+    existing = bookmarks.find_one(user_uid=user["uid"], target_uid=target_uid, target_type=target_type)
+    if existing:
+        bookmarks.delete(id=existing["id"])
+        saved = False
+    else:
+        bookmarks.insert({
+            "uid": generate_uid(),
+            "user_uid": user["uid"],
+            "target_uid": target_uid,
+            "target_type": target_type,
+            "created_at": datetime.now(timezone.utc).isoformat(),
+        })
+        saved = True
+
+    if request.headers.get("x-requested-with") == "fetch":
+        return JSONResponse({"saved": saved})
+    return RedirectResponse(url=request.headers.get("Referer", "/feed"), status_code=302)

devplacepy/routers/bugs.py

@@ -1,11 +1,16 @@
 import logging
-from datetime import datetime
-from fastapi import APIRouter, Request
+from typing import Annotated
+from datetime import datetime, timezone
+from fastapi import APIRouter, Request, Form
 from fastapi.responses import HTMLResponse, RedirectResponse
-from devplacepy.database import get_table, load_comments, get_attachments_by_type
+from devplacepy.models import BugForm
+from devplacepy.database import get_table, load_comments
+from devplacepy.attachments import get_attachments_batch, link_attachments
 from devplacepy.templating import templates
 from devplacepy.utils import generate_uid, require_user, time_ago, get_current_user, create_mention_notifications
 from devplacepy.seo import base_seo_context
+from devplacepy.responses import respond, action_result
+from devplacepy.schemas import BugsOut
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
@@ -23,13 +28,13 @@ async def bugs_page(request: Request):
 
     bug_list = []
     bug_uids = [b["uid"] for b in all_bugs]
-    attachments_map = get_attachments_by_type("bug", bug_uids) if bug_uids else {}
+    attachments_map = get_attachments_batch("bug", bug_uids) if bug_uids else {}
     for b in all_bugs:
         bug_list.append({
             "bug": b,
             "author": users_map.get(b["user_uid"]),
             "time_ago": time_ago(b["created_at"]),
-            "comments": load_comments("bug", b["uid"]),
+            "comments": load_comments("bug", b["uid"], user),
             "attachments": attachments_map.get(b["uid"], []),
         })
 
@@ -42,23 +47,19 @@ async def bugs_page(request: Request):
             {"name": "Bug Reports", "url": "/bugs"},
         ],
     )
-    return templates.TemplateResponse("bugs.html", {
+    return respond(request, "bugs.html", {
         **seo_ctx,
         "request": request,
         "user": user,
         "bugs": bug_list,
-    })
+    }, model=BugsOut)
 
 
 @router.post("/create")
-async def create_bug(request: Request):
+async def create_bug(request: Request, data: Annotated[BugForm, Form()]):
     user = require_user(request)
-    form = await request.form()
-    title = form.get("title", "").strip()
-    description = form.get("description", "").strip()
-
-    if not title or not description:
-        return RedirectResponse(url="/bugs", status_code=302)
+    title = data.title.strip()
+    description = data.description.strip()
 
     bugs_table = get_table("bug_reports")
     bug_uid = generate_uid()
@@ -68,11 +69,12 @@ async def create_bug(request: Request):
         "title": title,
         "description": description,
         "status": "open",
-        "created_at": datetime.utcnow().isoformat(),
+        "created_at": datetime.now(timezone.utc).isoformat(),
     })
 
-    attachment_uids = form.getlist("attachment_uids") if hasattr(form, "getlist") else []
+    if data.attachment_uids:
+        link_attachments(data.attachment_uids, "bug", bug_uid)
 
     create_mention_notifications(description, user["uid"], f"/bugs?highlight={bug_uid}")
     logger.info(f"Bug report created by {user['username']}: {title}")
-    return RedirectResponse(url="/bugs", status_code=302)
+    return action_result(request, "/bugs", data={"uid": bug_uid})

devplacepy/routers/comments.py

@@ -1,51 +1,28 @@
 import logging
-from datetime import datetime
-from fastapi import APIRouter, Request
+from typing import Annotated
+from datetime import datetime, timezone
+from fastapi import APIRouter, Request, Form
 from fastapi.responses import RedirectResponse
-from devplacepy.database import get_table, delete_attachments
-from devplacepy.utils import generate_uid, require_user, create_mention_notifications
+from devplacepy.database import get_table, resolve_object_url, delete_engagement
+from devplacepy.attachments import link_attachments, delete_target_attachments
+from devplacepy.content import is_owner
+from devplacepy.utils import generate_uid, require_user, create_mention_notifications, create_notification, award_rewards, XP_COMMENT
+from devplacepy.models import CommentForm
+from devplacepy.responses import action_result, wants_json, json_error
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
 
-def resolve_target_redirect(target_type, target_uid):
-    if target_type == "post":
-        post = resolve_by_slug(get_table("posts"), target_uid)
-        return f"/posts/{post['slug'] or post['uid']}" if post else "/feed"
-    if target_type == "project":
-        project = resolve_by_slug(get_table("projects"), target_uid)
-        return f"/projects/{project['slug'] or project['uid']}" if project else "/projects"
-    if target_type == "news":
-        article = resolve_by_slug(get_table("news"), target_uid)
-        if article:
-            return f"/news/{article.get('slug', '') or article['uid']}"
-        return "/news"
-    if target_type == "bug":
-        return f"/bugs?highlight={target_uid}"
-    if target_type == "gist":
-        gist = resolve_by_slug(get_table("gists"), target_uid)
-        return f"/gists/{gist['slug'] or gist['uid']}" if gist else "/gists"
-    return "/bugs"
-
-
 @router.post("/create")
-async def create_comment(request: Request):
+async def create_comment(request: Request, data: Annotated[CommentForm, Form()]):
     user = require_user(request)
-    form = await request.form()
-    content = form.get("content", "").strip()
-    target_uid = form.get("target_uid") or form.get("post_uid", "")
-    target_type = form.get("target_type", "post")
-    parent_uid = form.get("parent_uid", "")
+    content = data.content.strip()
+    target_uid = data.target_uid or data.post_uid
+    target_type = data.target_type
+    parent_uid = data.parent_uid
 
-    redirect_url = resolve_target_redirect(target_type, target_uid)
-
-    if not content or not target_uid:
-        return RedirectResponse(url=redirect_url, status_code=302)
-    if len(content) < 3:
-        return RedirectResponse(url=redirect_url, status_code=302)
-    if len(content) > 1000:
-        return RedirectResponse(url=redirect_url, status_code=302)
+    redirect_url = resolve_object_url(target_type, target_uid)
 
     comment_uid = generate_uid()
     insert = {
@@ -55,62 +32,35 @@ async def create_comment(request: Request):
         "user_uid": user["uid"],
         "content": content,
         "parent_uid": parent_uid or None,
-        "created_at": datetime.utcnow().isoformat(),
+        "created_at": datetime.now(timezone.utc).isoformat(),
     }
     if target_type == "post":
         insert["post_uid"] = target_uid
     get_table("comments").insert(insert)
 
-    attachment_uids = form.getlist("attachment_uids") if hasattr(form, "getlist") else []
-    for auid in attachment_uids:
-        if auid.strip():
-            get_table("attachments").update({"uid": auid.strip(), "resource_uid": insert["uid"], "resource_type": "comment"}, ["uid"])
+    if data.attachment_uids:
+        link_attachments(data.attachment_uids, "comment", comment_uid)
 
-    badges = get_table("badges")
-    existing = badges.find_one(user_uid=user["uid"], badge_name="First Comment")
-    if not existing:
-        badges.insert({
-            "uid": generate_uid(),
-            "user_uid": user["uid"],
-            "badge_name": "First Comment",
-            "created_at": datetime.utcnow().isoformat(),
-        })
+    award_rewards(user["uid"], XP_COMMENT, "First Comment")
+
+    comment_url = f"{redirect_url}#comment-{comment_uid}"
 
     if target_type == "post":
         if parent_uid:
-            comments_table = get_table("comments")
-            parent = comments_table.find_one(uid=parent_uid)
+            parent = get_table("comments").find_one(uid=parent_uid)
             if parent and parent["user_uid"] != user["uid"]:
-                notifications = get_table("notifications")
-                notifications.insert({
-                    "uid": generate_uid(),
-                    "user_uid": parent["user_uid"],
-                    "type": "reply",
-                    "message": f"{user['username']} replied to your comment",
-                    "related_uid": user["uid"],
-                    "read": False,
-                    "created_at": datetime.utcnow().isoformat(),
-                })
+                create_notification(parent["user_uid"], "reply", f"{user['username']} replied to your comment", user["uid"], comment_url)
         else:
             posts = get_table("posts")
             post = posts.find_one(uid=target_uid)
             if not post:
                 post = posts.find_one(slug=target_uid)
             if post and post["user_uid"] != user["uid"]:
-                notifications = get_table("notifications")
-                notifications.insert({
-                    "uid": generate_uid(),
-                    "user_uid": post["user_uid"],
-                    "type": "comment",
-                    "message": f"{user['username']} commented on your post",
-                    "related_uid": user["uid"],
-                    "read": False,
-                    "created_at": datetime.utcnow().isoformat(),
-                })
+                create_notification(post["user_uid"], "comment", f"{user['username']} commented on your post", user["uid"], comment_url)
 
-    create_mention_notifications(content, user["uid"], redirect_url)
+    create_mention_notifications(content, user["uid"], comment_url)
     logger.info(f"Comment by {user['username']} on {target_type} {target_uid}")
-    return RedirectResponse(url=redirect_url, status_code=302)
+    return action_result(request, comment_url, data={"uid": comment_uid, "url": comment_url})
 
 
 @router.post("/delete/{comment_uid}")
@@ -118,15 +68,16 @@ async def delete_comment(request: Request, comment_uid: str):
     user = require_user(request)
     comments = get_table("comments")
     comment = comments.find_one(uid=comment_uid)
-    if not comment:
-        return RedirectResponse(url="/feed", status_code=302)
-    if comment["user_uid"] != user["uid"]:
+    if not is_owner(comment, user):
+        if wants_json(request):
+            return json_error(403, "Not allowed")
         return RedirectResponse(url="/feed", status_code=302)
     target_type = comment.get("target_type", "post")
     target_uid = comment.get("target_uid") or comment.get("post_uid", "")
-    delete_attachments("comment", comment_uid)
+    delete_target_attachments("comment", comment_uid)
     get_table("votes").delete(target_uid=comment_uid, target_type="comment")
+    delete_engagement("comment", [comment_uid])
     comments.delete(id=comment["id"])
     logger.info(f"Comment {comment_uid} deleted by {user['username']}")
-    redirect_url = resolve_target_redirect(target_type, target_uid)
-    return RedirectResponse(url=redirect_url, status_code=302)
+    redirect_url = resolve_object_url(target_type, target_uid)
+    return action_result(request, redirect_url)

devplacepy/routers/devii.py

@@ -0,0 +1,196 @@
+# retoor <retoor@molodetz.nl>
+
+import logging
+
+import httpx
+import uuid_utils
+from fastapi import APIRouter, Request, WebSocket, WebSocketDisconnect
+from fastapi.responses import JSONResponse, RedirectResponse
+
+from devplacepy.database import get_int_setting
+from devplacepy.services.manager import service_manager
+from devplacepy.templating import templates
+from devplacepy.utils import _user_from_api_key, _user_from_session, get_current_user, is_admin
+
+logger = logging.getLogger("devii.router")
+
+router = APIRouter()
+
+GUEST_COOKIE = "devii_guest"
+GUEST_COOKIE_MAX_AGE = 31536000
+
+
+def _service():
+    return service_manager.get_service("devii")
+
+
+def _bearer_key(headers) -> str:
+    key = headers.get("x-api-key", "")
+    if key:
+        return key.strip()
+    scheme, _, credentials = headers.get("authorization", "").partition(" ")
+    if scheme.lower() == "bearer":
+        return credentials.strip()
+    return ""
+
+
+def _resolve_ws_owner(websocket: WebSocket):
+    user = _user_from_session(websocket)
+    if not user:
+        key = _bearer_key(websocket.headers)
+        if key:
+            user = _user_from_api_key(key)
+    if user:
+        return "user", user["uid"], user.get("username", ""), user.get("api_key", ""), is_admin(user)
+    guest_id = websocket.cookies.get(GUEST_COOKIE) or uuid_utils.uuid7().hex
+    return "guest", guest_id, "guest", "", False
+
+
+def _owner_from_request(request: Request):
+    user = get_current_user(request)
+    if user:
+        return "user", user["uid"], user.get("username", "")
+    guest_id = request.cookies.get(GUEST_COOKIE) or ""
+    return "guest", guest_id, "guest"
+
+
+@router.get("/")
+async def devii_page(request: Request):
+    user = get_current_user(request)
+    response = templates.TemplateResponse(
+        request, "devii.html", {"request": request, "user": user, "page_title": "Devii", "meta_robots": "noindex,nofollow"}
+    )
+    if not user and not request.cookies.get(GUEST_COOKIE):
+        response.set_cookie(GUEST_COOKIE, uuid_utils.uuid7().hex, httponly=True, samesite="lax", max_age=GUEST_COOKIE_MAX_AGE)
+    return response
+
+
+@router.get("/session")
+async def devii_session(request: Request):
+    user = get_current_user(request)
+    svc = _service()
+    payload = {
+        "loggedIn": bool(user),
+        "username": user.get("username") if user else "guest",
+        "enabled": bool(svc and svc.is_enabled()),
+        "guestsEnabled": bool(svc and svc.guests_enabled()),
+    }
+    response = JSONResponse(payload)
+    if not user and not request.cookies.get(GUEST_COOKIE):
+        response.set_cookie(GUEST_COOKIE, uuid_utils.uuid7().hex, httponly=True, samesite="lax", max_age=GUEST_COOKIE_MAX_AGE)
+    return response
+
+
+def _safe_next(value: str) -> str:
+    if value.startswith("/") and not value.startswith("//"):
+        return value
+    return "/"
+
+
+@router.get("/adopt")
+async def devii_adopt(request: Request):
+    # The terminal's agent logged in; adopt the real session it minted into the browser so both
+    # share one session, then return to where the user was.
+    target = _safe_next(request.query_params.get("next", "/"))
+    response = RedirectResponse(target, status_code=303)
+    svc = _service()
+    if svc is None:
+        return response
+    owner_kind, owner_id, _ = _owner_from_request(request)
+    session = svc.hub().find(owner_kind, owner_id) if owner_id else None
+    token = session.take_pending_session() if session else None
+    if token:
+        max_age = max(1, get_int_setting("session_remember_days", 30)) * 86400
+        response.set_cookie("session", token, max_age=max_age, httponly=True, samesite="lax")
+    return response
+
+
+@router.get("/usage")
+async def devii_usage(request: Request):
+    svc = _service()
+    owner_kind, owner_id, _ = _owner_from_request(request)
+    if svc is None:
+        return JSONResponse({"used_pct": 0.0, "turns_today": 0})
+    spent = svc.spent_24h(owner_kind, owner_id) if owner_id else 0.0
+    limit = svc.daily_limit_for(owner_kind)
+    turns = svc.hub().ledger.turns_24h(owner_kind, owner_id) if owner_id else 0
+    used_pct = round(min(100.0, spent / limit * 100), 1) if limit > 0 else 0.0
+    payload = {
+        "used_pct": used_pct,
+        "turns_today": turns,
+        "owner_kind": owner_kind,
+    }
+    if is_admin(get_current_user(request)):
+        payload["spent_24h"] = round(spent, 6)
+        payload["limit"] = limit
+    return JSONResponse(payload)
+
+
+@router.post("/clippy/ai/chat")
+async def clippy_proxy(request: Request):
+    svc = _service()
+    if svc is None or not svc.is_enabled():
+        return JSONResponse({"error": "Devii is unavailable"}, status_code=503)
+    cfg = svc.effective_config()
+    body = await request.body()
+    headers = {"Content-Type": "application/json"}
+    if cfg.get("devii_ai_key"):
+        headers["Authorization"] = f"Bearer {cfg['devii_ai_key']}"
+    async with httpx.AsyncClient(timeout=45.0) as client:
+        upstream = await client.post(cfg["devii_ai_url"], content=body, headers=headers)
+    return JSONResponse(
+        content=upstream.json() if upstream.headers.get("content-type", "").startswith("application/json") else {"raw": upstream.text},
+        status_code=upstream.status_code,
+    )
+
+
+@router.websocket("/ws")
+async def devii_ws(websocket: WebSocket):
+    await websocket.accept()
+    svc = _service()
+    if svc is None or not svc.is_enabled():
+        await websocket.close(code=1013)
+        return
+    if not service_manager.owns_lock():
+        await websocket.close(code=1013)
+        return
+    owner_kind, owner_id, username, api_key, owner_is_admin = _resolve_ws_owner(websocket)
+    if owner_kind == "guest" and not svc.guests_enabled():
+        await websocket.send_json({"type": "error", "text": "Guest access to Devii is disabled."})
+        await websocket.close(code=1008)
+        return
+
+    session = svc.hub().get_or_create(
+        owner_kind, owner_id, username, api_key, svc.instance_base_url(), is_admin=owner_is_admin
+    )
+    session.attach(websocket)
+    try:
+        await session.send_bootstrap(websocket)
+        while True:
+            data = await websocket.receive_json()
+            kind = data.get("type")
+            if kind == "input":
+                text = str(data.get("text", "")).strip()
+                if not text:
+                    continue
+                limit = svc.daily_limit_for(owner_kind, owner_is_admin)
+                spent = svc.spent_24h(owner_kind, owner_id)
+                if limit > 0 and spent >= limit:
+                    await websocket.send_json({
+                        "type": "error",
+                        "text": "Daily AI quota reached (100%). Try again later.",
+                    })
+                    continue
+                session.spawn_turn(text)
+            elif kind == "reset":
+                await session.reset_conversation()
+            elif kind == "visibility":
+                session.set_visibility(websocket, bool(data.get("visible", True)), bool(data.get("focused", False)))
+            elif kind in ("avatar_result", "client_result"):
+                session.resolve_query(str(data.get("id", "")), data.get("result"))
+    except WebSocketDisconnect:
+        pass
+    except Exception:  # noqa: BLE001 - never let the socket loop crash the worker
+        logger.exception("Devii websocket loop failed for %s/%s", owner_kind, owner_id)
+    finally:
+        session.detach(websocket)

devplacepy/routers/docs.py

@@ -0,0 +1,219 @@
+import logging
+from fastapi import APIRouter, Request
+from fastapi.responses import HTMLResponse, RedirectResponse, Response
+from devplacepy.templating import templates
+from devplacepy.utils import get_current_user, not_found, is_admin as user_is_admin
+from devplacepy.seo import base_seo_context, site_url, website_schema
+from devplacepy.docs_api import api_doc_pages, render_group, build_services_group
+from devplacepy.services.manager import service_manager
+from devplacepy import docs_search, docs_export, docs_live, docs_prose
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+SECTION_GENERAL = "General"
+SECTION_COMPONENTS = "Components"
+SECTION_API = "API"
+SECTION_ADMIN = "Administration"
+SECTION_DEVII = "Devii internals"
+SECTION_SERVICES = "Services"
+SECTION_ARCH = "Architecture"
+SECTION_TESTING = "Testing"
+SECTION_PROD = "Production"
+
+DOCS_PAGES = [
+    # General - how to use the site and Devii (everyone)
+    {"slug": "index", "title": "Overview", "kind": "prose", "section": SECTION_GENERAL},
+    {"slug": "devii", "title": "Devii Assistant", "kind": "prose", "section": SECTION_GENERAL},
+    {"slug": "dashboard", "title": "Your dashboard", "kind": "live", "section": SECTION_GENERAL},
+    # Components - custom HTML web components with live examples (everyone)
+    {"slug": "components", "title": "Components overview", "kind": "prose", "section": SECTION_COMPONENTS},
+    {"slug": "component-dp-avatar", "title": "dp-avatar", "kind": "prose", "section": SECTION_COMPONENTS},
+    {"slug": "component-dp-code", "title": "dp-code", "kind": "prose", "section": SECTION_COMPONENTS},
+    {"slug": "component-dp-content", "title": "dp-content", "kind": "prose", "section": SECTION_COMPONENTS},
+    {"slug": "component-dp-upload", "title": "dp-upload", "kind": "prose", "section": SECTION_COMPONENTS},
+    {"slug": "component-dp-toast", "title": "dp-toast", "kind": "prose", "section": SECTION_COMPONENTS},
+    {"slug": "component-dp-dialog", "title": "dp-dialog", "kind": "prose", "section": SECTION_COMPONENTS},
+    {"slug": "component-dp-context-menu", "title": "dp-context-menu", "kind": "prose", "section": SECTION_COMPONENTS},
+    {"slug": "component-devii-terminal", "title": "devii-terminal", "kind": "prose", "section": SECTION_COMPONENTS},
+    {"slug": "component-devii-avatar", "title": "devii-avatar", "kind": "prose", "section": SECTION_COMPONENTS},
+    {"slug": "component-emoji-picker", "title": "emoji-picker", "kind": "prose", "section": SECTION_COMPONENTS},
+    # API - developer reference (everyone); admin-only groups are routed to Administration below
+    {"slug": "authentication", "title": "Authentication", "kind": "prose", "section": SECTION_API},
+    *[{**page, "section": (SECTION_ADMIN if page.get("admin") else SECTION_API)} for page in api_doc_pages()],
+    # Devii internals - technical reference for the Devii assistant (admins only)
+    {"slug": "devii-internals", "title": "Devii internals", "kind": "prose", "admin": True, "section": SECTION_DEVII},
+    {"slug": "devii-architecture", "title": "Architecture and sessions", "kind": "prose", "admin": True, "section": SECTION_DEVII},
+    {"slug": "devii-tools", "title": "Tools and scopes", "kind": "prose", "admin": True, "section": SECTION_DEVII},
+    {"slug": "devii-data", "title": "Data and persistence", "kind": "prose", "admin": True, "section": SECTION_DEVII},
+    {"slug": "devii-security", "title": "Security and limits", "kind": "prose", "admin": True, "section": SECTION_DEVII},
+    {"slug": "devii-config", "title": "Configuration and CLI", "kind": "prose", "admin": True, "section": SECTION_DEVII},
+    # Services - the background service framework and every service in detail (admins only)
+    {"slug": "services-overview", "title": "Services overview", "kind": "prose", "admin": True, "section": SECTION_SERVICES},
+    {"slug": "services-framework", "title": "Service framework", "kind": "prose", "admin": True, "section": SECTION_SERVICES},
+    {"slug": "services-data", "title": "Reading service data", "kind": "prose", "admin": True, "section": SECTION_SERVICES},
+    {"slug": "services-gateway", "title": "GatewayService (AI)", "kind": "prose", "admin": True, "section": SECTION_SERVICES},
+    {"slug": "services-devii", "title": "DeviiService", "kind": "prose", "admin": True, "section": SECTION_SERVICES},
+    {"slug": "services-news", "title": "NewsService", "kind": "prose", "admin": True, "section": SECTION_SERVICES},
+    {"slug": "services-bots", "title": "BotsService", "kind": "prose", "admin": True, "section": SECTION_SERVICES},
+    {"slug": "services-zip", "title": "ZipService", "kind": "prose", "admin": True, "section": SECTION_SERVICES},
+    # Architecture - platform design, structure, and development process (admins only)
+    {"slug": "architecture", "title": "Architecture overview", "kind": "prose", "admin": True, "section": SECTION_ARCH},
+    {"slug": "architecture-backend", "title": "Backend and request pipeline", "kind": "prose", "admin": True, "section": SECTION_ARCH},
+    {"slug": "architecture-frontend", "title": "Frontend and components", "kind": "prose", "admin": True, "section": SECTION_ARCH},
+    {"slug": "architecture-styling", "title": "Styling and templates", "kind": "prose", "admin": True, "section": SECTION_ARCH},
+    {"slug": "architecture-conventions", "title": "Conventions and rules", "kind": "prose", "admin": True, "section": SECTION_ARCH},
+    {"slug": "architecture-workflow", "title": "Development workflow", "kind": "prose", "admin": True, "section": SECTION_ARCH},
+    {"slug": "architecture-jobs", "title": "Async job services", "kind": "prose", "admin": True, "section": SECTION_ARCH},
+    # Testing - test framework, load testing, and make targets (admins only)
+    {"slug": "testing", "title": "Testing overview", "kind": "prose", "admin": True, "section": SECTION_TESTING},
+    {"slug": "testing-framework", "title": "Test framework and rules", "kind": "prose", "admin": True, "section": SECTION_TESTING},
+    {"slug": "testing-locust", "title": "Load testing with Locust", "kind": "prose", "admin": True, "section": SECTION_TESTING},
+    {"slug": "testing-make", "title": "Running tests via make", "kind": "prose", "admin": True, "section": SECTION_TESTING},
+    {"slug": "testing-cicd", "title": "Coverage and CI/CD", "kind": "prose", "admin": True, "section": SECTION_TESTING},
+    # Production - deployment and operations reference (admins only)
+    {"slug": "production", "title": "Production overview", "kind": "prose", "admin": True, "section": SECTION_PROD},
+    {"slug": "production-deploy", "title": "Deploy and update", "kind": "prose", "admin": True, "section": SECTION_PROD},
+    {"slug": "production-nginx", "title": "nginx and networking", "kind": "prose", "admin": True, "section": SECTION_PROD},
+    {"slug": "production-concurrency", "title": "Multi-worker and concurrency", "kind": "prose", "admin": True, "section": SECTION_PROD},
+]
+_PAGES_BY_SLUG = {page["slug"]: page for page in DOCS_PAGES}
+
+
+@router.get("/docs")
+async def docs_root(request: Request):
+    return RedirectResponse(url="/docs/index.html", status_code=302)
+
+
+def _export_context(request: Request):
+    user = get_current_user(request)
+    is_admin = user_is_admin(user)
+    return (
+        is_admin,
+        site_url(request),
+        user.get("username") if user else "",
+        user.get("api_key") if user else "",
+    )
+
+
+@router.get("/docs/download.md")
+async def docs_download_md(request: Request):
+    is_admin, base, username, api_key = _export_context(request)
+    markdown = docs_export.build_markdown(is_admin, base, username, api_key)
+    return Response(
+        content=markdown,
+        media_type="text/markdown; charset=utf-8",
+        headers={"Content-Disposition": 'attachment; filename="devplace-docs.md"'},
+    )
+
+
+@router.get("/docs/download.html")
+async def docs_download_html(request: Request):
+    is_admin, base, username, api_key = _export_context(request)
+    markdown = docs_export.build_markdown(is_admin, base, username, api_key)
+    return HTMLResponse(
+        content=docs_export.build_html(markdown),
+        headers={"Content-Disposition": 'attachment; filename="devplace-docs.html"'},
+    )
+
+
+@router.get("/docs/{slug}.html", response_class=HTMLResponse)
+async def docs_page(request: Request, slug: str):
+    user = get_current_user(request)
+    is_admin = user_is_admin(user)
+    base = site_url(request)
+    visible_pages = [p for p in DOCS_PAGES if not p.get("admin") or is_admin]
+
+    if slug == "search":
+        query = request.query_params.get("q", "")
+        results = docs_search.search(query, user=user, is_admin=is_admin)
+        seo_ctx = base_seo_context(
+            request,
+            title="Search - Documentation",
+            description="Search the DevPlace developer documentation.",
+            robots="noindex",
+            breadcrumbs=[
+                {"name": "Home", "url": "/feed"},
+                {"name": "Docs", "url": "/docs/index.html"},
+                {"name": "Search", "url": "/docs/search.html"},
+            ],
+            schemas=[website_schema(base)],
+        )
+        return templates.TemplateResponse(request, "docs_base.html", {
+            **seo_ctx,
+            "request": request,
+            "user": user,
+            "pages": visible_pages,
+            "current": "search",
+            "kind": "search",
+            "page_title_doc": "Search",
+            "search_query": query,
+            "search_results": results,
+            "base": base,
+        })
+
+    page = _PAGES_BY_SLUG.get(slug)
+    if not page:
+        raise not_found("Documentation page not found")
+    if page.get("admin") and not is_admin:
+        raise not_found("Documentation page not found")
+    if page["kind"] == "live":
+        seo_ctx = base_seo_context(
+            request,
+            title=f"{page['title']} - Documentation",
+            description="Your live DevPlace stats, activity, and site data.",
+            robots="noindex",
+            breadcrumbs=[
+                {"name": "Home", "url": "/feed"},
+                {"name": "Docs", "url": "/docs/index.html"},
+                {"name": page["title"], "url": f"/docs/{slug}.html"},
+            ],
+            schemas=[website_schema(base)],
+        )
+        return templates.TemplateResponse(request, "docs_base.html", {
+            **seo_ctx,
+            "request": request,
+            "user": user,
+            "pages": visible_pages,
+            "current": slug,
+            "kind": "live",
+            "facts": docs_live.build_live_facts(user),
+            "page_title_doc": page["title"],
+            "base": base,
+        })
+    prose_html = None
+    group = None
+    if page["kind"] == "prose":
+        prose_html = docs_prose.render_prose(slug, {
+            "base": base,
+            "user": user,
+            "username": user.get("username") if user else "",
+            "api_key": user.get("api_key") if user else "",
+        })
+    elif page.get("dynamic"):
+        group = build_services_group(service_manager.describe_all(), base)
+    else:
+        group = render_group(slug, base, user.get("username") if user else "", user.get("api_key") if user else "")
+    seo_ctx = base_seo_context(
+        request,
+        title=f"{page['title']} - Documentation",
+        description="DevPlace developer documentation.",
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {"name": "Docs", "url": "/docs/index.html"},
+            {"name": page["title"], "url": f"/docs/{slug}.html"},
+        ],
+        schemas=[website_schema(base)],
+    )
+    return templates.TemplateResponse(request, "docs_base.html", {
+        **seo_ctx,
+        "request": request,
+        "user": user,
+        "pages": visible_pages,
+        "current": slug,
+        "kind": page["kind"],
+        "group": group,
+        "prose_html": prose_html,
+        "page_title_doc": page["title"],
+        "base": base,
+    })

devplacepy/routers/feed.py

@@ -1,25 +1,22 @@
 import logging
-from datetime import datetime
 from fastapi import APIRouter, Request
 from fastapi.responses import HTMLResponse
-from devplacepy.database import get_table, get_daily_topic, get_users_by_uids, get_comment_counts_by_post_uids
+from devplacepy.database import get_table, get_daily_topic, get_users_by_uids, get_comment_counts_by_post_uids, get_recent_comments_by_post_uids, get_site_stats, get_top_authors, get_reactions_by_targets, get_user_bookmarks, get_polls_by_post_uids, paginate
 from devplacepy.attachments import get_attachments_batch
+from devplacepy.content import enrich_items
 from devplacepy.templating import templates
-from devplacepy.utils import get_current_user, time_ago
-from devplacepy.seo import base_seo_context, site_url, website_schema, breadcrumb_schema, combine
+from devplacepy.utils import get_current_user
+from devplacepy.seo import list_page_seo, next_page_url
+from devplacepy.responses import respond
+from devplacepy.schemas import FeedOut
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
-PAGE_SIZE = 25
-
 
 def get_feed_posts(user, tab: str = "all", topic: str = None, before: str = None):
     posts_table = get_table("posts")
-
-    filters = {}
-    if topic:
-        filters["topic"] = topic
+    order = ["-stars", "-created_at"] if tab == "trending" else ["-created_at"]
 
     if tab == "following":
         if not user:
@@ -28,40 +25,18 @@ def get_feed_posts(user, tab: str = "all", topic: str = None, before: str = None
         following = [f["following_uid"] for f in follows.find(follower_uid=user["uid"])]
         if not following:
             return [], None
-        posts = list(posts_table.find(posts_table.table.columns.user_uid.in_(following), order_by=["-created_at"], _limit=PAGE_SIZE))
+        posts, next_cursor = paginate(posts_table, posts_table.table.columns.user_uid.in_(following), before=before, order=order)
     else:
-        order = ["-created_at"]
-        if tab == "trending":
-            order = ["-stars", "-created_at"]
-        if before:
-            posts = list(posts_table.find(**filters, order_by=order, _limit=PAGE_SIZE + 1))
-            posts = [p for p in posts if p["created_at"] < before][:PAGE_SIZE]
-        else:
-            posts = list(posts_table.find(**filters, order_by=order, _limit=PAGE_SIZE + 1))
-
-    has_more = len(posts) > PAGE_SIZE
-    posts = posts[:PAGE_SIZE]
-
-    next_cursor = None
-    if has_more and posts:
-        next_cursor = posts[-1]["created_at"]
+        filters = {"topic": topic} if topic else {}
+        posts, next_cursor = paginate(posts_table, before=before, order=order, **filters)
 
     if not posts:
         return [], next_cursor
 
-    uids = [p["user_uid"] for p in posts]
-    post_uids = [p["uid"] for p in posts]
-    authors = get_users_by_uids(uids)
-    counts = get_comment_counts_by_post_uids(post_uids)
+    authors = get_users_by_uids([p["user_uid"] for p in posts])
+    counts = get_comment_counts_by_post_uids([p["uid"] for p in posts])
 
-    result = []
-    for post in posts:
-        result.append({
-            "post": post,
-            "author": authors.get(post["user_uid"]),
-            "time_ago": time_ago(post["created_at"]),
-            "comment_count": counts.get(post["uid"], 0),
-        })
+    result = enrich_items(posts, "post", authors, {"comment_count": counts}, user=user)
     return result, next_cursor
 
 
@@ -69,41 +44,43 @@ def get_feed_posts(user, tab: str = "all", topic: str = None, before: str = None
 async def feed_page(request: Request, tab: str = "all", topic: str = None, before: str = None):
     user = get_current_user(request)
     posts, next_cursor = get_feed_posts(user, tab, topic, before)
-    users_table = get_table("users")
-    total_members = len(list(users_table.all()))
-    posts_table = get_table("posts")
-    today_start = datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0).isoformat()
-    posts_today = len(list(posts_table.find(created_at={">=": today_start})))
-    total_projects = len(list(get_table("projects").all()))
-    total_gists = len(list(get_table("gists").all()))
-    top_authors = list(users_table.find(stars={">": 0}, order_by=["-stars"], _limit=5))
+    stats = get_site_stats()
+    top_authors = get_top_authors(5)
     daily_topic = get_daily_topic()
 
     post_uids_list = [item["post"]["uid"] for item in posts]
     attachments_map = get_attachments_batch("post", post_uids_list)
+    recent_comments = get_recent_comments_by_post_uids(post_uids_list, 3, user)
+    reactions_map = get_reactions_by_targets("post", post_uids_list, user)
+    bookmark_set = get_user_bookmarks(user["uid"], "post", post_uids_list) if user else set()
+    polls_map = get_polls_by_post_uids(post_uids_list, user)
     for item in posts:
-        item["attachments"] = attachments_map.get(item["post"]["uid"], [])
+        uid = item["post"]["uid"]
+        item["attachments"] = attachments_map.get(uid, [])
+        item["recent_comments"] = recent_comments.get(uid, [])
+        item["reactions"] = reactions_map.get(uid, {"counts": {}, "mine": []})
+        item["bookmarked"] = uid in bookmark_set
+        item["poll"] = polls_map.get(uid)
 
-    base = site_url(request)
-    seo_ctx = base_seo_context(
+    seo_ctx = list_page_seo(
         request,
         title="Feed",
         description="Discover the latest developer discussions, projects, and community activity on DevPlace.",
         breadcrumbs=[{"name": "Home", "url": "/feed"}, {"name": "Feed", "url": "/feed"}],
-        schemas=[website_schema(base)],
+        next_url=next_page_url(request, next_cursor),
     )
-    return templates.TemplateResponse("feed.html", {
+    return respond(request, "feed.html", {
         **seo_ctx,
         "request": request,
         "user": user,
         "posts": posts,
         "current_tab": tab,
         "current_topic": topic,
-        "total_members": total_members,
-        "posts_today": posts_today,
-        "total_projects": total_projects,
-        "total_gists": total_gists,
+        "total_members": stats["total_members"],
+        "posts_today": stats["posts_today"],
+        "total_projects": stats["total_projects"],
+        "total_gists": stats["total_gists"],
         "top_authors": top_authors,
         "daily_topic": daily_topic,
         "next_cursor": next_cursor,
-    })
+    }, model=FeedOut)

devplacepy/routers/follow.py

@@ -1,9 +1,10 @@
 import logging
-from datetime import datetime
+from datetime import datetime, timezone
 from fastapi import APIRouter, Request
 from fastapi.responses import RedirectResponse
 from devplacepy.database import get_table
-from devplacepy.utils import generate_uid, require_user
+from devplacepy.utils import generate_uid, require_user, create_notification, award_rewards, XP_FOLLOW
+from devplacepy.responses import action_result
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
@@ -15,33 +16,25 @@ async def follow_user(request: Request, username: str):
     users = get_table("users")
     target = users.find_one(username=username)
     if not target or target["uid"] == user["uid"]:
-        return RedirectResponse(url=f"/profile/{username}", status_code=302)
+        return action_result(request, f"/profile/{username}")
 
     follows = get_table("follows")
     existing = follows.find_one(follower_uid=user["uid"], following_uid=target["uid"])
     if existing:
-        return RedirectResponse(url=f"/profile/{username}", status_code=302)
+        return action_result(request, f"/profile/{username}")
 
     follows.insert({
         "uid": generate_uid(),
         "follower_uid": user["uid"],
         "following_uid": target["uid"],
-        "created_at": datetime.utcnow().isoformat(),
+        "created_at": datetime.now(timezone.utc).isoformat(),
     })
 
-    notifications = get_table("notifications")
-    notifications.insert({
-        "uid": generate_uid(),
-        "user_uid": target["uid"],
-        "type": "follow",
-        "message": f"{user['username']} started following you",
-        "related_uid": user["uid"],
-        "read": False,
-        "created_at": datetime.utcnow().isoformat(),
-    })
+    create_notification(target["uid"], "follow", f"{user['username']} started following you", user["uid"], f"/profile/{user['username']}")
+    award_rewards(target["uid"], XP_FOLLOW)
 
     logger.info(f"{user['username']} followed {username}")
-    return RedirectResponse(url=f"/profile/{username}", status_code=302)
+    return action_result(request, f"/profile/{username}")
 
 
 @router.post("/unfollow/{username}")
@@ -50,7 +43,7 @@ async def unfollow_user(request: Request, username: str):
     users = get_table("users")
     target = users.find_one(username=username)
     if not target:
-        return RedirectResponse(url=f"/profile/{username}", status_code=302)
+        return action_result(request, f"/profile/{username}")
 
     follows = get_table("follows")
     existing = follows.find_one(follower_uid=user["uid"], following_uid=target["uid"])
@@ -58,4 +51,4 @@ async def unfollow_user(request: Request, username: str):
         follows.delete(id=existing["id"])
         logger.info(f"{user['username']} unfollowed {username}")
 
-    return RedirectResponse(url=f"/profile/{username}", status_code=302)
+    return action_result(request, f"/profile/{username}")

devplacepy/routers/gists.py

@@ -1,11 +1,15 @@
 import logging
-from datetime import datetime
-from fastapi import APIRouter, Request, HTTPException
+from typing import Annotated
+from fastapi import APIRouter, Request, Form
+from devplacepy.models import GistForm, GistEditForm
 from fastapi.responses import HTMLResponse, RedirectResponse
-from devplacepy.database import get_table, load_comments, get_vote_counts, get_attachments, get_attachments_by_type, delete_attachments, resolve_by_slug
+from devplacepy.database import get_table, get_users_by_uids, get_gist_languages, paginate
+from devplacepy.content import load_detail, edit_content_item, delete_content_item, enrich_items, create_content_item, detail_context, canonical_redirect, first_image_url
 from devplacepy.templating import templates
-from devplacepy.utils import generate_uid, get_current_user, require_user, time_ago, make_combined_slug, create_mention_notifications
-from devplacepy.seo import base_seo_context, site_url, website_schema, breadcrumb_schema, combine
+from devplacepy.utils import get_current_user, require_user, not_found, XP_GIST
+from devplacepy.seo import base_seo_context, site_url, website_schema, software_source_code_schema, list_page_seo, next_page_url
+from devplacepy.responses import respond, action_result
+from devplacepy.schemas import GistsOut, GistDetailOut
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
@@ -21,7 +25,7 @@ LANGUAGES = [
 ]
 
 
-def get_gists_list(user_uid=None, language=None):
+def get_gists_list(user_uid=None, language=None, before=None, viewer=None):
     gists_table = get_table("gists")
     filters = {}
     if user_uid:
@@ -29,111 +33,110 @@ def get_gists_list(user_uid=None, language=None):
     if language:
         filters["language"] = language
 
-    all_gists = list(gists_table.find(**filters, order_by=["-created_at"]))
+    total = gists_table.count(**filters)
+    gists, next_cursor = paginate(gists_table, before=before, **filters)
 
-    if not all_gists:
-        return []
+    if not gists:
+        return [], next_cursor, total
 
-    from devplacepy.database import get_users_by_uids
-    uids = [g["user_uid"] for g in all_gists]
-    users_map = get_users_by_uids(uids)
+    users_map = get_users_by_uids([g["user_uid"] for g in gists])
+    return enrich_items(gists, "gist", users_map, user=viewer), next_cursor, total
 
-    result = []
-    for g in all_gists:
-        author = users_map.get(g["user_uid"])
-        result.append({
-            "gist": g,
-            "author": author,
-            "time_ago": time_ago(g["created_at"]),
-        })
-    return result
+
+@router.get("", response_class=HTMLResponse)
+async def gists_page(request: Request, language: str = None, user_uid: str = None, before: str = None):
+    user = get_current_user(request)
+    gists_data, next_cursor, total_count = get_gists_list(user_uid, language, before, viewer=user)
+    seo_ctx = list_page_seo(
+        request,
+        title="Gists",
+        description=f"Browse {total_count} code snippets on DevPlace.",
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {"name": "Gists", "url": "/gists"},
+        ],
+        next_url=next_page_url(request, next_cursor),
+    )
+    return respond(request, "gists.html", {
+        **seo_ctx,
+        "request": request,
+        "user": user,
+        "gists": gists_data,
+        "total_count": total_count,
+        "next_cursor": next_cursor,
+        "current_language": language,
+        "languages": LANGUAGES,
+        "gist_language_codes": get_gist_languages(),
+    }, model=GistsOut)
 
 
 @router.get("/{gist_slug}", response_class=HTMLResponse)
 async def gist_detail(request: Request, gist_slug: str):
     user = get_current_user(request)
-    gists = get_table("gists")
-    gist = resolve_by_slug(gists, gist_slug)
-    if not gist:
-        raise HTTPException(status_code=404, detail="Gist not found")
-
-    from devplacepy.database import get_users_by_uids
-    users_map = get_users_by_uids([gist["user_uid"]])
-    author = users_map.get(gist["user_uid"])
-
-    is_owner = user and user["uid"] == gist["user_uid"]
-
-    ups, downs = get_vote_counts([gist["uid"]])
-    star_count = ups.get(gist["uid"], 0) - downs.get(gist["uid"], 0)
-
-    comments = load_comments("gist", gist["uid"])
-
-    gist_attachments = get_attachments("gist", gist["uid"])
+    detail = load_detail("gists", "gist", gist_slug, user)
+    if not detail:
+        raise not_found("Gist not found")
+    gist = detail["item"]
+    redirect = canonical_redirect("gists", gist, gist_slug)
+    if redirect:
+        return redirect
 
     base = site_url(request)
     seo_ctx = base_seo_context(
         request,
         title=gist.get("title", "Gist"),
         description=gist.get("description", "")[:160],
+        og_type="article",
+        og_image=first_image_url(gist, detail["attachments"]),
         breadcrumbs=[
             {"name": "Home", "url": "/feed"},
             {"name": "Gists", "url": "/gists"},
             {"name": gist.get("title", "Gist"), "url": f"/gists/{gist['slug'] or gist['uid']}"},
         ],
-        schemas=[website_schema(base)],
+        schemas=[website_schema(base), software_source_code_schema(gist, base)],
     )
-    return templates.TemplateResponse("gist_detail.html", {
-        **seo_ctx,
-        "request": request,
-        "user": user,
-        "gist": gist,
-        "author": author,
-        "is_owner": is_owner,
-        "star_count": star_count,
-        "time_ago": time_ago(gist["created_at"]),
-        "comments": comments,
+    return respond(request, "gist_detail.html", detail_context(request, user, detail, "gist", seo_ctx, {
         "languages": LANGUAGES,
-        "attachments": gist_attachments,
-    })
+    }), model=GistDetailOut)
 
 
 @router.post("/create")
-async def create_gist(request: Request):
+async def create_gist(request: Request, data: Annotated[GistForm, Form()]):
     user = require_user(request)
-    form = await request.form()
-    title = form.get("title", "").strip()
-    description = form.get("description", "").strip()
-    source_code = form.get("source_code", "").strip()
-    language = form.get("language", "plaintext")
-
-    if not title:
-        return RedirectResponse(url="/gists", status_code=302)
-    if len(title) > 200:
-        return RedirectResponse(url="/gists", status_code=302)
-    if not source_code:
-        return RedirectResponse(url="/gists", status_code=302)
-    if len(source_code) > 50000:
-        return RedirectResponse(url="/gists", status_code=302)
-    if len(description) > 5000:
-        return RedirectResponse(url="/gists", status_code=302)
+    title = data.title.strip()
+    description = data.description.strip()
+    source_code = data.source_code.strip()
+    language = data.language
 
     valid_languages = {l[0] for l in LANGUAGES}
     if language not in valid_languages:
         language = "plaintext"
 
-    gists = get_table("gists")
-    uid = generate_uid()
-    gist_slug = make_combined_slug(title, uid)
-    gists.insert({
-        "uid": uid,
-        "user_uid": user["uid"],
+    uid, gist_slug = create_content_item("gists", "gist", user, {
         "title": title,
-        "slug": gist_slug,
         "description": description or None,
         "source_code": source_code,
         "language": language,
-        "stars": 0,
-        "created_at": datetime.utcnow().isoformat(),
-    })
+    }, title, XP_GIST, "First Gist", description or "", data.attachment_uids)
+    url = f"/gists/{gist_slug}"
+    return action_result(request, url, data={"uid": uid, "slug": gist_slug, "url": url})
 
-    attachment_uids = form.getlist("attachment_uids") if hasattr(form, "getlist") else []
+
+@router.post("/edit/{gist_slug}")
+async def edit_gist(request: Request, gist_slug: str, data: Annotated[GistEditForm, Form()]):
+    user = require_user(request)
+    language = data.language
+    if language not in {l[0] for l in LANGUAGES}:
+        language = "plaintext"
+    return edit_content_item(request, "gists", user, gist_slug, {
+        "title": data.title.strip(),
+        "description": data.description.strip() or None,
+        "source_code": data.source_code.strip(),
+        "language": language,
+    }, "/gists")
+
+
+@router.post("/delete/{gist_slug}")
+async def delete_gist(request: Request, gist_slug: str):
+    user = require_user(request)
+    return delete_content_item(request, "gists", "gist", user, gist_slug, "/gists")

devplacepy/routers/leaderboard.py

@@ -0,0 +1,46 @@
+import logging
+from fastapi import APIRouter, Request
+from fastapi.responses import HTMLResponse
+from devplacepy.database import get_leaderboard, get_user_rank, get_site_stats, get_top_authors, get_featured_news
+from devplacepy.templating import templates
+from devplacepy.utils import get_current_user
+from devplacepy.seo import list_page_seo
+from devplacepy.responses import respond
+from devplacepy.schemas import LeaderboardOut
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+TOP_LIMIT = 50
+
+
+@router.get("", response_class=HTMLResponse)
+async def leaderboard_page(request: Request):
+    entries = get_leaderboard(TOP_LIMIT, 0)
+
+    user = get_current_user(request)
+    user_rank = get_user_rank(user["uid"]) if user else None
+
+    stats = get_site_stats()
+    top_authors = get_top_authors(5)
+    featured_news = get_featured_news(5)
+
+    seo_ctx = list_page_seo(
+        request,
+        title="Leaderboard",
+        description="Top contributors on DevPlace ranked by the stars their posts, projects, and gists have earned.",
+        breadcrumbs=[{"name": "Home", "url": "/feed"}, {"name": "Leaderboard", "url": "/leaderboard"}],
+    )
+    return respond(request, "leaderboard.html", {
+        **seo_ctx,
+        "request": request,
+        "user": user,
+        "entries": entries,
+        "user_rank": user_rank,
+        "total_members": stats["total_members"],
+        "posts_today": stats["posts_today"],
+        "total_projects": stats["total_projects"],
+        "total_gists": stats["total_gists"],
+        "top_authors": top_authors,
+        "featured_news": featured_news,
+    }, model=LeaderboardOut)

devplacepy/routers/messages.py

@@ -1,11 +1,16 @@
 import logging
-from datetime import datetime
-from fastapi import APIRouter, Request
+from typing import Annotated
+from datetime import datetime, timezone
+from fastapi import APIRouter, Request, Form
+from devplacepy.models import MessageForm
 from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse
-from devplacepy.database import get_table, db, get_attachments_by_type
-from devplacepy.templating import templates
+from devplacepy.database import get_table, db
+from devplacepy.attachments import get_attachments_batch
+from devplacepy.templating import templates, clear_unread_cache, clear_messages_cache
 from devplacepy.utils import generate_uid, require_user, time_ago, create_mention_notifications
 from devplacepy.seo import base_seo_context
+from devplacepy.responses import respond, action_result
+from devplacepy.schemas import MessagesOut
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
@@ -65,9 +70,10 @@ def get_conversation_messages(user_uid: str, other_uid: str):
             msgs.append(m)
     msgs.sort(key=lambda m: m["created_at"])
 
-    for msg in msgs:
-        if msg["receiver_uid"] == user_uid and not msg["read"]:
-            messages_table.update({"id": msg["id"], "read": True}, ["id"])
+    if "messages" in db.tables:
+        with db:
+            db.query("UPDATE messages SET read = 1 WHERE receiver_uid = :me AND sender_uid = :other AND read = 0", me=user_uid, other=other_uid)
+        clear_messages_cache(user_uid)
 
     from devplacepy.database import get_users_by_uids
     user_ids = list({m["sender_uid"] for m in msgs} | {other_uid})
@@ -76,7 +82,7 @@ def get_conversation_messages(user_uid: str, other_uid: str):
 
     result = []
     msg_uids = [m["uid"] for m in msgs]
-    attachments_map = get_attachments_by_type("message", msg_uids) if msg_uids else {}
+    attachments_map = get_attachments_batch("message", msg_uids) if msg_uids else {}
     for m in msgs:
         result.append({
             "message": m,
@@ -115,7 +121,7 @@ async def messages_page(request: Request, with_uid: str = None, search: str = ""
             {"name": "Messages", "url": "/messages"},
         ],
     )
-    return templates.TemplateResponse("messages.html", {
+    return respond(request, "messages.html", {
         **seo_ctx,
         "request": request,
         "user": user,
@@ -124,7 +130,7 @@ async def messages_page(request: Request, with_uid: str = None, search: str = ""
         "other_user": other_user,
         "current_conversation": current_conversation,
         "search": search,
-    })
+    }, model=MessagesOut)
 
 
 @router.get("/search")
@@ -144,22 +150,44 @@ async def search_users(request: Request, q: str = ""):
 
 
 @router.post("/send")
-async def send_message(request: Request):
+async def send_message(request: Request, data: Annotated[MessageForm, Form()]):
     user = require_user(request)
-    form = await request.form()
-    content = form.get("content", "").strip()
-    receiver_uid = form.get("receiver_uid", "")
+    content = data.content.strip()
+    receiver_uid = data.receiver_uid
 
-    if content and receiver_uid:
-        messages_table = get_table("messages")
-        msg_uid = generate_uid()
-        messages_table.insert({
-            "uid": msg_uid,
-            "sender_uid": user["uid"],
-            "receiver_uid": receiver_uid,
-            "content": content,
+    if not get_table("users").find_one(uid=receiver_uid):
+        return action_result(request, "/messages")
+
+    messages_table = get_table("messages")
+    msg_uid = generate_uid()
+    messages_table.insert({
+        "uid": msg_uid,
+        "sender_uid": user["uid"],
+        "receiver_uid": receiver_uid,
+        "content": content,
+        "read": False,
+        "created_at": datetime.now(timezone.utc).isoformat(),
+    })
+
+    from devplacepy.attachments import link_attachments
+    link_attachments(data.attachment_uids, "message", msg_uid)
+
+    if user["uid"] != receiver_uid:
+        notifications = get_table("notifications")
+        notifications.insert({
+            "uid": generate_uid(),
+            "user_uid": receiver_uid,
+            "type": "message",
+            "message": f"{user['username']} sent you a message",
+            "related_uid": user["uid"],
+            "target_url": f"/messages?with_uid={user['uid']}",
             "read": False,
-            "created_at": datetime.utcnow().isoformat(),
+            "created_at": datetime.now(timezone.utc).isoformat(),
         })
+        clear_unread_cache(receiver_uid)
+        clear_messages_cache(receiver_uid)
 
-        attachment_uids = form.getlist("attachment_uids") if hasattr(form, "getlist") else []
+    create_mention_notifications(content, user["uid"], f"/messages?with_uid={receiver_uid}")
+
+    logger.info(f"Message {msg_uid} sent from {user['username']} to {receiver_uid}")
+    return action_result(request, f"/messages?with_uid={receiver_uid}", data={"uid": msg_uid})

devplacepy/routers/news.py

@@ -1,11 +1,14 @@
 import logging
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from fastapi import APIRouter, Request
 from fastapi.responses import HTMLResponse
-from devplacepy.database import get_table, db, load_comments, resolve_by_slug
+from devplacepy.database import get_table, db, load_comments, resolve_by_slug, get_news_images_by_uids, get_user_bookmarks, paginate
 from devplacepy.templating import templates
-from devplacepy.utils import get_current_user, time_ago
-from devplacepy.seo import base_seo_context, website_schema, site_url, discussion_forum_posting, combine
+from devplacepy.utils import get_current_user, time_ago, not_found
+from devplacepy.content import canonical_redirect
+from devplacepy.seo import base_seo_context, website_schema, site_url, news_article_schema, list_page_seo, next_page_url
+from devplacepy.responses import respond
+from devplacepy.schemas import NewsListOut, NewsDetailOut
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
@@ -14,26 +17,22 @@ NEWS_MAX_AGE_DAYS = 4
 
 
 @router.get("", response_class=HTMLResponse)
-async def news_page(request: Request):
+async def news_page(request: Request, before: str = None):
     user = get_current_user(request)
-    cutoff = (datetime.utcnow() - timedelta(days=NEWS_MAX_AGE_DAYS)).isoformat()
+    cutoff = (datetime.now(timezone.utc) - timedelta(days=NEWS_MAX_AGE_DAYS)).isoformat()
 
     news_table = get_table("news")
-    articles = list(news_table.find(
+    articles, next_cursor = paginate(
+        news_table,
+        before=before,
+        order=["-grade", "-synced_at"],
+        cursor_field="synced_at",
         status="published",
         synced_at={">=": cutoff},
-        order_by=["-grade", "-synced_at"],
-    ))
+    )
 
     article_uids = [a["uid"] for a in articles]
-
-    images_by_news = {}
-    if article_uids and "news_images" in db.tables:
-        images_table = get_table("news_images")
-        for uid in article_uids:
-            img = images_table.find_one(news_uid=uid, order_by=["uid"])
-            if img:
-                images_by_news[uid] = img["url"]
+    images_by_news = get_news_images_by_uids(article_uids)
 
     enriched = []
     for a in articles:
@@ -44,21 +43,21 @@ async def news_page(request: Request):
             "grade": a.get("grade", 0),
         })
 
-    base = site_url(request)
-    seo_ctx = base_seo_context(
+    seo_ctx = list_page_seo(
         request,
         title="Developer News",
         description="Curated developer news and industry signals. Stay ahead with hand-picked articles.",
         breadcrumbs=[{"name": "Home", "url": "/feed"}, {"name": "News", "url": "/news"}],
-        schemas=[website_schema(base)],
+        next_url=next_page_url(request, next_cursor),
     )
 
-    return templates.TemplateResponse("news.html", {
+    return respond(request, "news.html", {
         **seo_ctx,
         "request": request,
         "user": user,
         "articles": enriched,
-    })
+        "next_cursor": next_cursor,
+    }, model=NewsListOut)
 
 
 @router.get("/{news_slug}", response_class=HTMLResponse)
@@ -67,7 +66,10 @@ async def news_detail_page(request: Request, news_slug: str):
     news_table = get_table("news")
     article = resolve_by_slug(news_table, news_slug)
     if not article:
-        return HTMLResponse("News article not found", status_code=404)
+        raise not_found("News article not found")
+    redirect = canonical_redirect("news", article, news_slug)
+    if redirect:
+        return redirect
 
     image_url = ""
     if "news_images" in db.tables:
@@ -76,7 +78,8 @@ async def news_detail_page(request: Request, news_slug: str):
             image_url = img["url"]
 
     canonical_slug = article.get("slug", "") or article["uid"]
-    comments = load_comments("news", article["uid"])
+    comments = load_comments("news", article["uid"], user)
+    bookmarked = bool(user) and article["uid"] in get_user_bookmarks(user["uid"], "news", [article["uid"]])
 
     base = site_url(request)
     page_url = f"{base}/news/{canonical_slug}"
@@ -84,11 +87,13 @@ async def news_detail_page(request: Request, news_slug: str):
         request,
         title=article.get("title", "News Article"),
         description=(article.get("description", "") or "")[:200],
+        og_type="article",
+        og_image=image_url or None,
         breadcrumbs=[{"name": "Home", "url": "/feed"}, {"name": "News", "url": "/news"}, {"name": article.get("title", "")[:60], "url": page_url}],
-        schemas=[website_schema(base)],
+        schemas=[website_schema(base), news_article_schema(article, base, image_url)],
     )
 
-    return templates.TemplateResponse("news_detail.html", {
+    return respond(request, "news_detail.html", {
         **seo_ctx,
         "request": request,
         "user": user,
@@ -98,4 +103,5 @@ async def news_detail_page(request: Request, news_slug: str):
         "grade": article.get("grade", 0),
         "time_ago": time_ago(article["synced_at"]),
         "comments": comments,
-    })
+        "bookmarked": bookmarked,
+    }, model=NewsDetailOut)

devplacepy/routers/notifications.py

@@ -1,20 +1,29 @@
 import logging
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from fastapi import APIRouter, Request
-from fastapi.responses import HTMLResponse, RedirectResponse
-from devplacepy.database import get_table
-from devplacepy.templating import templates
-from devplacepy.utils import require_user, time_ago
+from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse
+from devplacepy.database import get_table, db
+from devplacepy.templating import (
+    templates,
+    clear_unread_cache,
+    jinja_unread_count,
+    jinja_unread_messages,
+)
+from devplacepy.utils import require_user, get_current_user, time_ago
 from devplacepy.seo import base_seo_context
+from devplacepy.responses import respond, action_result
+from devplacepy.schemas import NotificationsOut
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
+PAGE_SIZE = 25
+
 
 def _group_label(created_at: str) -> str:
     try:
         dt = datetime.fromisoformat(created_at)
-        now = datetime.utcnow()
+        now = datetime.now(timezone.utc)
         today = now.date()
         date = dt.date()
         if date == today:
@@ -29,15 +38,24 @@ def _group_label(created_at: str) -> str:
 
 
 @router.get("", response_class=HTMLResponse)
-async def notifications_page(request: Request):
+async def notifications_page(request: Request, before: str = None):
     user = require_user(request)
 
+    next_cursor = None
     try:
         notifications_table = get_table("notifications")
+        filters = {"user_uid": user["uid"]}
+        if before:
+            filters["created_at"] = {"<": before}
         raw_notifications = list(
-            notifications_table.find(user_uid=user["uid"], order_by=["-created_at"])
+            notifications_table.find(**filters, order_by=["-created_at"], _limit=PAGE_SIZE + 1)
         )
 
+        has_more = len(raw_notifications) > PAGE_SIZE
+        raw_notifications = raw_notifications[:PAGE_SIZE]
+        if has_more and raw_notifications:
+            next_cursor = raw_notifications[-1]["created_at"]
+
         enriched = []
         if raw_notifications:
             from devplacepy.database import get_users_by_uids
@@ -78,14 +96,39 @@ async def notifications_page(request: Request):
             {"name": "Notifications", "url": "/notifications"},
         ],
     )
-    return templates.TemplateResponse("notifications.html", {
+    return respond(request, "notifications.html", {
         **seo_ctx,
         "request": request,
         "user": user,
         "notification_groups": groups,
+        "next_cursor": next_cursor,
+    }, model=NotificationsOut)
+
+
+@router.get("/counts")
+async def unread_counts(request: Request):
+    user = get_current_user(request)
+    if not user:
+        return JSONResponse({"notifications": 0, "messages": 0})
+    return JSONResponse({
+        "notifications": jinja_unread_count(user["uid"]),
+        "messages": jinja_unread_messages(user["uid"]),
     })
 
 
+@router.get("/open/{notification_uid}")
+async def open_notification(request: Request, notification_uid: str):
+    user = require_user(request)
+    notifications_table = get_table("notifications")
+    n = notifications_table.find_one(uid=notification_uid)
+    if not n or n["user_uid"] != user["uid"]:
+        return action_result(request, "/notifications")
+    if not n["read"]:
+        notifications_table.update({"id": n["id"], "read": True}, ["id"])
+        clear_unread_cache(user["uid"])
+    return action_result(request, n.get("target_url") or "/notifications")
+
+
 @router.post("/mark-read/{notification_uid}")
 async def mark_read(request: Request, notification_uid: str):
     user = require_user(request)
@@ -93,13 +136,14 @@ async def mark_read(request: Request, notification_uid: str):
     n = notifications_table.find_one(uid=notification_uid)
     if n and n["user_uid"] == user["uid"]:
         notifications_table.update({"id": n["id"], "read": True}, ["id"])
-    return RedirectResponse(url="/notifications", status_code=302)
+        clear_unread_cache(user["uid"])
+    return action_result(request, "/notifications")
 
 
 @router.post("/mark-all-read")
 async def mark_all_read(request: Request):
     user = require_user(request)
-    notifications_table = get_table("notifications")
-    for n in notifications_table.find(user_uid=user["uid"], read=False):
-        notifications_table.update({"id": n["id"], "read": True}, ["id"])
-    return RedirectResponse(url="/notifications", status_code=302)
+    with db:
+        db.query("UPDATE notifications SET read = 1 WHERE user_uid = :u AND read = 0", u=user["uid"])
+    clear_unread_cache(user["uid"])
+    return action_result(request, "/notifications")

devplacepy/routers/openai_gateway.py

@@ -0,0 +1,23 @@
+import logging
+from fastapi import APIRouter, Request, HTTPException
+from devplacepy.services.manager import service_manager
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+
+def _service():
+    svc = service_manager.get_service("openai")
+    if svc is None:
+        raise HTTPException(status_code=503, detail="Gateway is not available")
+    return svc
+
+
+@router.post("/v1/chat/completions")
+async def chat_completions(request: Request):
+    return await _service().handle(request, "chat/completions")
+
+
+@router.api_route("/v1/{path:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"])
+async def passthrough(request: Request, path: str):
+    return await _service().handle(request, path)

devplacepy/routers/polls.py

@@ -0,0 +1,44 @@
+# retoor <retoor@molodetz.nl>
+import logging
+from typing import Annotated
+from datetime import datetime, timezone
+from fastapi import APIRouter, Request, Form
+from fastapi.responses import RedirectResponse, JSONResponse
+from devplacepy.database import get_table, get_poll_for_post
+from devplacepy.utils import generate_uid, require_user
+from devplacepy.models import PollVoteForm
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+
+@router.post("/{poll_uid}/vote")
+async def vote_poll(request: Request, poll_uid: str, data: Annotated[PollVoteForm, Form()]):
+    user = require_user(request)
+    poll = get_table("polls").find_one(uid=poll_uid)
+    if not poll:
+        return JSONResponse({"error": "Poll not found"}, status_code=404)
+    option = get_table("poll_options").find_one(uid=data.option_uid, poll_uid=poll_uid)
+    if not option:
+        return JSONResponse({"error": "Invalid option"}, status_code=400)
+
+    poll_votes = get_table("poll_votes")
+    existing = poll_votes.find_one(poll_uid=poll_uid, user_uid=user["uid"])
+    if existing:
+        if existing["option_uid"] == data.option_uid:
+            poll_votes.delete(id=existing["id"])
+        else:
+            poll_votes.update({"id": existing["id"], "option_uid": data.option_uid}, ["id"])
+    else:
+        poll_votes.insert({
+            "uid": generate_uid(),
+            "poll_uid": poll_uid,
+            "option_uid": data.option_uid,
+            "user_uid": user["uid"],
+            "created_at": datetime.now(timezone.utc).isoformat(),
+        })
+
+    result = get_poll_for_post(poll["post_uid"], user)
+    if request.headers.get("x-requested-with") == "fetch":
+        return JSONResponse(result)
+    return RedirectResponse(url=request.headers.get("Referer", "/feed"), status_code=302)

devplacepy/routers/posts.py

@@ -1,116 +1,86 @@
 import logging
-import aiofiles
-from datetime import datetime
-from pathlib import Path
-from fastapi import APIRouter, Request, HTTPException
+from typing import Annotated
+from datetime import datetime, timezone
+from fastapi import APIRouter, Request, Form
 from fastapi.responses import RedirectResponse, HTMLResponse
-from devplacepy.database import get_table, get_comment_counts_by_post_uids, load_comments, db
+from devplacepy.constants import TOPICS
+from devplacepy.database import db, get_table, resolve_by_slug
 from devplacepy.templating import templates
-from devplacepy.config import STATIC_DIR
-from devplacepy.utils import generate_uid, get_current_user, require_user, time_ago, make_combined_slug, create_mention_notifications
-from devplacepy.seo import base_seo_context, site_url, website_schema, breadcrumb_schema, discussion_forum_posting, combine, truncate
+from devplacepy.utils import get_current_user, require_user, time_ago, not_found, generate_uid, XP_POST
+from devplacepy.content import load_detail, edit_content_item, delete_content_item, create_content_item, detail_context, canonical_redirect, first_image_url
+from devplacepy.responses import respond, action_result
+from devplacepy.schemas import PostDetailOut
+from devplacepy.seo import base_seo_context, site_url, website_schema, discussion_forum_posting, truncate
+from devplacepy.attachments import save_inline_image
+from devplacepy.models import PostForm, PostEditForm
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
 
 @router.post("/create")
-async def create_post(request: Request):
+async def create_post(request: Request, data: Annotated[PostForm, Form()]):
     user = require_user(request)
-    form = await request.form()
-    content = form.get("content", "").strip()
-    title = form.get("title", "").strip()
-    topic = form.get("topic", "random")
-    project_uid = form.get("project_uid", "")
-
-    errors = []
-    if not content:
-        errors.append("Content is required")
-    if content and len(content) < 10:
-        errors.append("Content must be at least 10 characters")
-    if len(content) > 2000:
-        errors.append("Content too long (max 2000 characters)")
-    if topic not in TOPICS:
-        topic = "random"
-
-    if errors:
-        return RedirectResponse(url="/feed", status_code=302)
+    content = data.content.strip()
+    title = data.title.strip()
+    topic = data.topic
+    project_uid = data.project_uid
 
     image_filename = None
+    form = await request.form()
     image_file = form.get("image")
-    if image_file and hasattr(image_file, "filename") and image_file.filename:
-        try:
-            content_bytes = await image_file.read()
-            if len(content_bytes) > 5 * 1024 * 1024:
-                logger.warning(f"Image too large: {image_file.filename}")
-            else:
-                import imghdr
-                ext = Path(image_file.filename).suffix.lower()
-                allowed = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg"}
-                if ext not in allowed:
-                    logger.warning(f"Unsupported image type: {ext}")
-                else:
-                    upload_dir = STATIC_DIR / "uploads"
-                    upload_dir.mkdir(parents=True, exist_ok=True)
-                    image_filename = f"{generate_uid()}{ext}"
-                    file_path = upload_dir / image_filename
-                    async with aiofiles.open(str(file_path), "wb") as f:
-                        await f.write(content_bytes)
-                    logger.info(f"Image saved: {image_filename}")
-                    content += f"\n\n![](/static/uploads/{image_filename})"
-        except Exception as e:
-            logger.warning(f"Image upload failed: {e}")
+    if image_file is not None and hasattr(image_file, "filename") and image_file.filename:
+        image_filename = save_inline_image(await image_file.read(), image_file.filename)
+        if image_filename:
+            content += f"\n\n![](/static/uploads/{image_filename})"
 
-    posts = get_table("posts")
-    uid = generate_uid()
     slug_text = title if title else content[:50]
-    post_slug = make_combined_slug(slug_text, uid)
-    posts.insert({
-        "uid": uid,
-        "user_uid": user["uid"],
+    uid, post_slug = create_content_item("posts", "post", user, {
         "title": title or None,
-        "slug": post_slug,
         "content": content,
         "topic": topic,
         "project_uid": project_uid or None,
         "image": image_filename,
-        "stars": 0,
-        "created_at": datetime.utcnow().isoformat(),
+    }, slug_text, XP_POST, "First Post", content, data.attachment_uids)
+
+    create_poll(uid, user, data.poll_question, data.poll_options)
+    url = f"/posts/{post_slug}"
+    return action_result(request, url, data={"uid": uid, "slug": post_slug, "url": url})
+
+
+def create_poll(post_uid: str, user: dict, question: str, options: list[str]) -> None:
+    question = question.strip()
+    labels = [option.strip() for option in options if option.strip()][:6]
+    if not question or len(labels) < 2:
+        return
+    poll_uid = generate_uid()
+    get_table("polls").insert({
+        "uid": poll_uid,
+        "post_uid": post_uid,
+        "user_uid": user["uid"],
+        "question": question,
+        "created_at": datetime.now(timezone.utc).isoformat(),
     })
-
-    badges = get_table("badges")
-    existing = badges.find_one(user_uid=user["uid"], badge_name="First Post")
-    if not existing:
-        badges.insert({
-            "uid": generate_uid(),
-            "user_uid": user["uid"],
-            "badge_name": "First Post",
-            "created_at": datetime.utcnow().isoformat(),
-        })
-
-    attachment_uids = form.getlist("attachment_uids")
-    if attachment_uids:
-        link_attachments(attachment_uids, "post", uid)
-    if attachment_uids:
-        link_attachments(attachment_uids, "post", uid)
-
-    create_mention_notifications(content, user["uid"], f"/posts/{post_slug}")
-    logger.info(f"Post {uid} created by {user['username']}")
-    return RedirectResponse(url=f"/posts/{post_slug}", status_code=302)
+    get_table("poll_options").insert_many([{
+        "uid": generate_uid(),
+        "poll_uid": poll_uid,
+        "label": label,
+        "position": index,
+    } for index, label in enumerate(labels)])
 
 
 @router.get("/{post_slug}", response_class=HTMLResponse)
 async def view_post(request: Request, post_slug: str):
     user = get_current_user(request)
-    posts = get_table("posts")
-    post = resolve_by_slug(posts, post_slug)
-    if not post:
-        raise HTTPException(status_code=404, detail="Post not found")
-
-    users_table = get_table("users")
-    author = users_table.find_one(uid=post["user_uid"])
-
-    top_level = load_comments("post", post["uid"])
+    detail = load_detail("posts", "post", post_slug, user)
+    if not detail:
+        raise not_found("Post not found")
+    post = detail["item"]
+    redirect = canonical_redirect("posts", post, post_slug)
+    if redirect:
+        return redirect
+    author = detail["author"]
+    top_level = detail["comments"]
 
     def count_all(items):
         total = len(items)
@@ -118,7 +88,6 @@ async def view_post(request: Request, post_slug: str):
             total += count_all(item.get("children", []))
         return total
     comment_count = count_all(top_level)
-    star_count = post.get("stars", 0)
     base = site_url(request)
     seo_ctx = base_seo_context(
         request,
@@ -130,9 +99,10 @@ async def view_post(request: Request, post_slug: str):
             {"name": post.get("title") or "Post", "url": f"/posts/{post['slug'] or post['uid']}"},
         ],
         og_type="article",
+        og_image=first_image_url(post, detail["attachments"]),
         schemas=[
             website_schema(base),
-            discussion_forum_posting(post, author, comment_count, star_count, base),
+            discussion_forum_posting(post, author, comment_count, detail["star_count"], base),
         ],
     )
 
@@ -149,76 +119,29 @@ async def view_post(request: Request, post_slug: str):
                 "time_ago": time_ago(r["created_at"]),
             })
 
-    post_attachments = get_attachments("post", post["uid"])
-
-    return templates.TemplateResponse("post.html", {
-        **seo_ctx,
-        "request": request,
-        "user": user,
-        "post": post,
-        "author": author,
-        "comments": top_level,
-        "time_ago": time_ago(post["created_at"]),
+    return respond(request, "post.html", detail_context(request, user, detail, "post", seo_ctx, {
         "comment_count": comment_count,
         "related_posts": related_posts,
         "topics": list(TOPICS),
-        "attachments": post_attachments,
-    })
+    }), model=PostDetailOut)
 
 
 @router.post("/edit/{post_slug}")
-async def edit_post(request: Request, post_slug: str):
+async def edit_post(request: Request, post_slug: str, data: Annotated[PostEditForm, Form()]):
     user = require_user(request)
-    form = await request.form()
-    content = form.get("content", "").strip()
-    title = form.get("title", "").strip()
-    topic = form.get("topic", "random")
-
-    errors = []
-    if not content:
-        errors.append("Content is required")
-    if content and len(content) < 10:
-        errors.append("Content must be at least 10 characters")
-    if len(content) > 2000:
-        errors.append("Content too long (max 2000 characters)")
-    if topic not in TOPICS:
-        topic = "random"
-
-    posts = get_table("posts")
-    post = resolve_by_slug(posts, post_slug)
-    if not post or post["user_uid"] != user["uid"]:
-        return RedirectResponse(url="/feed", status_code=302)
-
-    if errors:
-        return RedirectResponse(url=f"/posts/{post['slug'] or post['uid']}", status_code=302)
-
-    posts.update({
-        "uid": post["uid"],
-        "content": content,
-        "title": title or None,
-        "topic": topic,
-    }, ["uid"])
-
-    logger.info(f"Post {post['uid']} edited by {user['username']}")
-    return RedirectResponse(url=f"/posts/{post['slug'] or post['uid']}", status_code=302)
+    result = edit_content_item(request, "posts", user, post_slug, {
+        "content": data.content.strip(),
+        "title": data.title.strip() or None,
+        "topic": data.topic,
+    }, "/feed")
+    if data.poll_question.strip():
+        post = resolve_by_slug(get_table("posts"), post_slug)
+        if post and post["user_uid"] == user["uid"] and not get_table("polls").find_one(post_uid=post["uid"]):
+            create_poll(post["uid"], user, data.poll_question, data.poll_options)
+    return result
 
 
 @router.post("/delete/{post_slug}")
 async def delete_post(request: Request, post_slug: str):
     user = require_user(request)
-    posts = get_table("posts")
-    post = resolve_by_slug(posts, post_slug)
-    if post and post["user_uid"] == user["uid"]:
-        delete_target_attachments("post", post["uid"])
-        get_table("comments").delete(post_uid=post["uid"])
-        get_table("votes").delete(target_uid=post["uid"])
-        image = post.get("image")
-        if image:
-            try:
-                (STATIC_DIR / "uploads" / image).unlink(missing_ok=True)
-            except Exception as e:
-                logger.warning(f"Failed to delete image {image}: {e}")
-        posts.delete(id=post["id"])
-        logger.info(f"Post {post['uid']} deleted by {user['username']}")
-    return RedirectResponse(url="/feed", status_code=302)
-from devplacepy.attachments import get_attachments, link_attachments, delete_target_attachments
+    return delete_content_item(request, "posts", "post", user, post_slug, "/feed", inline_image_field="image")

devplacepy/routers/profile.py

@@ -1,97 +1,44 @@
 import logging
-from fastapi import APIRouter, Request
+from typing import Annotated
+from fastapi import APIRouter, Request, Form
+from devplacepy.models import ProfileForm
 from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse
-from devplacepy.database import get_table, db
+from devplacepy.database import get_table, db, get_user_stars, get_user_rank, get_comment_counts_by_post_uids, get_reactions_by_targets, get_user_bookmarks, get_polls_by_post_uids, get_activity_heatmap, get_activity_months, get_streaks, get_follow_counts, get_follow_list, get_following_among
+from devplacepy.content import enrich_items
 from devplacepy.templating import templates
-from devplacepy.utils import get_current_user, require_user, time_ago
-from devplacepy.seo import base_seo_context, site_url, website_schema, breadcrumb_schema, profile_page_schema, combine
+from devplacepy.utils import get_current_user, require_user, require_user_api, time_ago, clear_user_cache, not_found, generate_uid
+from devplacepy.responses import respond, action_result
+from devplacepy.schemas import ProfileOut
+from devplacepy.avatar import avatar_url
+from devplacepy.seo import base_seo_context, site_url, website_schema, profile_page_schema
+from devplacepy.services.manager import service_manager
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
 
-@router.get("/{username}", response_class=HTMLResponse)
-async def profile_page(request: Request, username: str, tab: str = "posts"):
-    current_user = get_current_user(request)
-    users = get_table("users")
-    profile_user = users.find_one(username=username)
-    if not profile_user:
-        return RedirectResponse(url="/feed", status_code=302)
-
-    posts = []
-    if tab == "posts":
-        posts_table = get_table("posts")
-        raw_posts = list(posts_table.find(user_uid=profile_user["uid"], order_by=["-created_at"]))
-        if raw_posts:
-            from devplacepy.database import get_comment_counts_by_post_uids
-            counts = get_comment_counts_by_post_uids([p["uid"] for p in raw_posts])
-        else:
-            counts = {}
-        for p in raw_posts:
-            posts.append({
-                "post": p,
-                "time_ago": time_ago(p["created_at"]),
-                "comment_count": counts.get(p["uid"], 0),
-            })
-
-    badges = list(get_table("badges").find(user_uid=profile_user["uid"]))
-    projects = list(get_table("projects").find(user_uid=profile_user["uid"]))
-    gists = list(get_table("gists").find(user_uid=profile_user["uid"]))
-    posts_count = len(posts) or len(list(get_table("posts").find(user_uid=profile_user["uid"])))
-
-    activities = []
-    if tab == "activity":
-        posts_table = get_table("posts")
-        for p in posts_table.find(user_uid=profile_user["uid"], order_by=["-created_at"], _limit=10):
-            activities.append({"type": "post", "content": p.get("title") or p["content"][:80], "time_ago": time_ago(p["created_at"]), "created_at": p["created_at"], "uid": p["uid"]})
-        comments_table = get_table("comments")
-        for c in comments_table.find(user_uid=profile_user["uid"], order_by=["-created_at"], _limit=10):
-            activities.append({"type": "comment", "content": c["content"][:80], "time_ago": time_ago(c["created_at"]), "created_at": c["created_at"], "uid": c["post_uid"]})
-        activities.sort(key=lambda a: a["created_at"], reverse=True)
-
-    is_following = False
-    if current_user:
-        follows = get_table("follows")
-        is_following = bool(follows.find_one(follower_uid=current_user["uid"], following_uid=profile_user["uid"]))
-
-    base = site_url(request)
-    robots = "noindex,follow" if posts_count < 2 else "index,follow"
-    bio = profile_user.get("bio", "")
-    desc = bio or f"View {profile_user['username']}'s profile on DevPlace. {posts_count} posts."
-    seo_ctx = base_seo_context(
-        request,
-        title=f"{profile_user['username']} (@{profile_user['username']})",
-        description=desc,
-        robots=robots,
-        breadcrumbs=[
-            {"name": "Home", "url": "/feed"},
-            {"name": profile_user['username'], "url": f"/profile/{profile_user['username']}"},
-        ],
-        schemas=[
-            website_schema(base),
-            profile_page_schema(profile_user, posts_count, base),
-        ],
-    )
-
-    return templates.TemplateResponse("profile.html", {
-        **seo_ctx,
-        "request": request,
-        "user": current_user,
-        "profile_user": profile_user,
-        "posts": posts,
-        "badges": badges,
-        "projects": projects,
-        "gists": gists,
-        "current_tab": tab,
-        "posts_count": posts_count,
-        "is_following": is_following,
-        "activities": activities,
-    })
+def _ai_quota(user_uid: str, include_cost: bool = False, is_admin: bool = False) -> dict | None:
+    devii = service_manager.get_service("devii")
+    if devii is None:
+        return None
+    try:
+        limit = devii.daily_limit_for("user", is_admin)
+        spent = devii.spent_24h("user", user_uid)
+        turns = devii.hub().ledger.turns_24h("user", user_uid)
+    except Exception:
+        logger.exception("Failed to compute AI quota for %s", user_uid)
+        return None
+    used_pct = round(min(100.0, spent / limit * 100), 1) if limit > 0 else 0.0
+    quota = {"used_pct": used_pct, "turns": turns}
+    if include_cost:
+        quota["spent_usd"] = round(spent, 4)
+        quota["limit_usd"] = round(limit, 2)
+    return quota
 
 
 @router.get("/search")
 async def search_users(request: Request, q: str = ""):
-    require_user(request)
+    require_user_api(request)
     if not q or len(q) < 1:
         return JSONResponse({"results": []})
     if "users" in db.tables:
@@ -105,22 +52,181 @@ async def search_users(request: Request, q: str = ""):
     return JSONResponse({"results": results})
 
 
+def _follow_people(profile_uid: str, mode: str, current_user, page: int):
+    people, pagination = get_follow_list(profile_uid, mode, page)
+    mine = get_following_among(current_user["uid"], [p["uid"] for p in people]) if current_user else set()
+    for person in people:
+        person["is_following"] = person["uid"] in mine
+        person["is_self"] = bool(current_user and current_user["uid"] == person["uid"])
+    return people, pagination
+
+
+@router.get("/{username}/followers")
+async def profile_followers(request: Request, username: str, page: int = 1):
+    return _follow_list_json(request, username, "followers", page)
+
+
+@router.get("/{username}/following")
+async def profile_following(request: Request, username: str, page: int = 1):
+    return _follow_list_json(request, username, "following", page)
+
+
+def _follow_list_json(request: Request, username: str, mode: str, page: int):
+    profile_user = get_table("users").find_one(username=username)
+    if not profile_user:
+        raise not_found("Profile not found")
+    current_user = get_current_user(request)
+    people, pagination = _follow_people(profile_user["uid"], mode, current_user, page)
+    return JSONResponse({
+        "username": profile_user["username"],
+        "mode": mode,
+        "count": pagination["total"],
+        "page": pagination["page"],
+        "total_pages": pagination["total_pages"],
+        mode: [{"uid": p["uid"], "username": p["username"], "bio": p["bio"], "is_following": p["is_following"]} for p in people],
+    })
+
+
+@router.get("/{username}", response_class=HTMLResponse)
+async def profile_page(request: Request, username: str, tab: str = "posts", page: int = 1):
+    current_user = get_current_user(request)
+    users = get_table("users")
+    profile_user = users.find_one(username=username)
+    if not profile_user:
+        raise not_found("Profile not found")
+    profile_user["stars"] = get_user_stars(profile_user["uid"])
+    rank = get_user_rank(profile_user["uid"])
+    follow_counts = get_follow_counts(profile_user["uid"])
+
+    people = []
+    follow_pagination = None
+    if tab in ("followers", "following"):
+        people, follow_pagination = _follow_people(profile_user["uid"], tab, current_user, page)
+
+    posts = []
+    if tab == "posts":
+        posts_table = get_table("posts")
+        raw_posts = list(posts_table.find(user_uid=profile_user["uid"], order_by=["-created_at"]))
+        post_uids = [p["uid"] for p in raw_posts]
+        counts = get_comment_counts_by_post_uids(post_uids) if raw_posts else {}
+        authors = {profile_user["uid"]: profile_user}
+        posts = enrich_items(raw_posts, "post", authors, {"comment_count": counts}, user=current_user)
+        reactions_map = get_reactions_by_targets("post", post_uids, current_user)
+        bookmark_set = get_user_bookmarks(current_user["uid"], "post", post_uids) if current_user else set()
+        polls_map = get_polls_by_post_uids(post_uids, current_user)
+        for item in posts:
+            uid = item["post"]["uid"]
+            item["reactions"] = reactions_map.get(uid, {"counts": {}, "mine": []})
+            item["bookmarked"] = uid in bookmark_set
+            item["poll"] = polls_map.get(uid)
+
+    badges = list(get_table("badges").find(user_uid=profile_user["uid"]))
+    projects = list(get_table("projects").find(user_uid=profile_user["uid"]))
+    gists_raw = list(get_table("gists").find(user_uid=profile_user["uid"]))
+    gists = []
+    for g in gists_raw:
+        gists.append({"gist": g, "time_ago": time_ago(g["created_at"])})
+    posts_count = len(posts) or get_table("posts").count(user_uid=profile_user["uid"])
+
+    activities = []
+    if tab == "activity":
+        posts_table = get_table("posts")
+        for p in posts_table.find(user_uid=profile_user["uid"], order_by=["-created_at"], _limit=10):
+            activities.append({"type": "post", "content": p.get("title") or p["content"][:80], "time_ago": time_ago(p["created_at"]), "created_at": p["created_at"], "uid": p["uid"]})
+        comments_table = get_table("comments")
+        for c in comments_table.find(user_uid=profile_user["uid"], order_by=["-created_at"], _limit=10):
+            target_uid = c.get("target_uid", c.get("post_uid", ""))
+            target_type = c.get("target_type", "post")
+            activities.append({"type": "comment", "content": c["content"][:80], "time_ago": time_ago(c["created_at"]), "created_at": c["created_at"], "uid": target_uid, "target_type": target_type})
+        activities.sort(key=lambda a: a["created_at"], reverse=True)
+
+    heatmap = get_activity_heatmap(profile_user["uid"])
+    heatmap_months = get_activity_months(heatmap)
+    streak = get_streaks(profile_user["uid"])
+
+    is_following = False
+    if current_user:
+        follows = get_table("follows")
+        is_following = bool(follows.find_one(follower_uid=current_user["uid"], following_uid=profile_user["uid"]))
+
+    is_owner = bool(current_user and current_user["uid"] == profile_user["uid"])
+    viewer_is_admin = bool(current_user and current_user.get("role") == "Admin")
+    can_view_api_key = bool(current_user and (is_owner or viewer_is_admin))
+    api_key = profile_user.get("api_key", "") if can_view_api_key else ""
+    profile_is_admin = profile_user.get("role") == "Admin"
+    ai_quota = _ai_quota(profile_user["uid"], include_cost=viewer_is_admin, is_admin=profile_is_admin) if (is_owner or viewer_is_admin) else None
+
+    base = site_url(request)
+    robots = "noindex,follow" if posts_count < 2 else "index,follow"
+    bio = profile_user.get("bio", "")
+    desc = bio or f"View {profile_user['username']}'s profile on DevPlace. {posts_count} posts."
+    seo_ctx = base_seo_context(
+        request,
+        title=f"{profile_user['username']} (@{profile_user['username']})",
+        description=desc,
+        robots=robots,
+        og_type="profile",
+        og_image=avatar_url("multiavatar", profile_user["username"], 256),
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {"name": profile_user['username'], "url": f"/profile/{profile_user['username']}"},
+        ],
+        schemas=[
+            website_schema(base),
+            profile_page_schema(profile_user, posts_count, base),
+        ],
+    )
+
+    return respond(request, "profile.html", {
+        **seo_ctx,
+        "request": request,
+        "user": current_user,
+        "profile_user": profile_user,
+        "posts": posts,
+        "badges": badges,
+        "projects": projects,
+        "gists": gists,
+        "current_tab": tab,
+        "posts_count": posts_count,
+        "is_following": is_following,
+        "is_owner": is_owner,
+        "can_view_api_key": can_view_api_key,
+        "api_key": api_key,
+        "ai_quota": ai_quota,
+        "activities": activities,
+        "rank": rank,
+        "heatmap": heatmap,
+        "heatmap_months": heatmap_months,
+        "streak": streak,
+        "people": people,
+        "follow_pagination": follow_pagination,
+        "followers_count": follow_counts["followers"],
+        "following_count": follow_counts["following"],
+    }, model=ProfileOut)
+
+
 @router.post("/update")
-async def update_profile(request: Request):
+async def update_profile(request: Request, data: Annotated[ProfileForm, Form()]):
     user = require_user(request)
-    form = await request.form()
-    bio = form.get("bio", "").strip()
-    location = form.get("location", "").strip()
-    git_link = form.get("git_link", "").strip()
-    website = form.get("website", "").strip()
     users = get_table("users")
     users.update({
         "uid": user["uid"],
-        "bio": bio,
-        "location": location,
-        "git_link": git_link,
-        "website": website,
+        "bio": data.bio.strip(),
+        "location": data.location.strip(),
+        "git_link": data.git_link.strip(),
+        "website": data.website.strip(),
     }, ["uid"])
+    clear_user_cache(user["uid"])
 
     logger.info(f"Profile updated for {user['username']}")
-    return RedirectResponse(url=f"/profile/{user['username']}", status_code=302)
+    return action_result(request, f"/profile/{user['username']}")
+
+
+@router.post("/regenerate-api-key")
+async def regenerate_api_key(request: Request):
+    user = require_user(request)
+    new_key = generate_uid()
+    get_table("users").update({"uid": user["uid"], "api_key": new_key}, ["uid"])
+    clear_user_cache(user["uid"])
+    logger.info(f"API key regenerated for {user['username']}")
+    return JSONResponse({"api_key": new_key})

devplacepy/routers/project_files.py

@@ -0,0 +1,175 @@
+# retoor <retoor@molodetz.nl>
+
+import logging
+from typing import Annotated
+
+from fastapi import APIRouter, Request, Form
+from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
+
+from devplacepy.database import get_table, resolve_by_slug
+from devplacepy.content import is_owner
+from devplacepy.models import (
+    ProjectFileWriteForm,
+    ProjectFileMkdirForm,
+    ProjectFileMoveForm,
+    ProjectFileDeleteForm,
+)
+from devplacepy.responses import respond, action_result, wants_json, json_error
+from devplacepy.schemas import ProjectFilesOut
+from devplacepy.services.jobs import queue
+from devplacepy.seo import base_seo_context
+from devplacepy.utils import get_current_user, require_user, not_found
+from devplacepy import project_files
+from devplacepy.project_files import ProjectFileError
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+
+def _load_project(project_slug: str) -> dict:
+    project = resolve_by_slug(get_table("projects"), project_slug)
+    if not project:
+        raise not_found("Project not found")
+    return project
+
+
+def _files_url(project: dict) -> str:
+    return f"/projects/{project['slug'] or project['uid']}/files"
+
+
+def _deny(request: Request, project: dict):
+    if wants_json(request):
+        return json_error(403, "Not allowed")
+    return RedirectResponse(url=_files_url(project), status_code=302)
+
+
+def _fail(request: Request, project: dict, exc: ProjectFileError):
+    if wants_json(request):
+        return json_error(400, str(exc))
+    return RedirectResponse(url=_files_url(project), status_code=302)
+
+
+@router.get("/{project_slug}/files", response_class=HTMLResponse)
+async def project_files_page(request: Request, project_slug: str):
+    user = get_current_user(request)
+    project = _load_project(project_slug)
+    files = project_files.list_files(project["uid"])
+    seo_ctx = base_seo_context(
+        request,
+        title=f"Files - {project.get('title', 'Project')}",
+        description=f"Files in the {project.get('title', 'project')} project.",
+        robots="noindex,follow",
+    )
+    return respond(request, "project_files.html", {
+        **seo_ctx,
+        "request": request,
+        "user": user,
+        "project": project,
+        "files": files,
+        "is_owner": is_owner(project, user),
+    }, model=ProjectFilesOut)
+
+
+@router.get("/{project_slug}/files/raw")
+async def project_file_raw(request: Request, project_slug: str, path: str):
+    project = _load_project(project_slug)
+    try:
+        return JSONResponse(project_files.read_file(project["uid"], path))
+    except ProjectFileError as exc:
+        return json_error(404, str(exc))
+
+
+@router.post("/{project_slug}/files/zip")
+async def project_files_zip(request: Request, project_slug: str, path: str = ""):
+    project = _load_project(project_slug)
+    user = get_current_user(request)
+    subpath = ""
+    name = project.get("title") or "project"
+    if path:
+        try:
+            subpath = project_files.normalize_path(path)
+        except ProjectFileError as exc:
+            return json_error(400, str(exc))
+        if project_files.get_node(project["uid"], subpath) is None:
+            return json_error(404, "Path not found")
+        name = subpath.rsplit("/", 1)[-1]
+    owner_kind, owner_id = ("user", user["uid"]) if user else ("guest", request.client.host or "anon")
+    uid = queue.enqueue(
+        "zip",
+        {"source": {"type": "project_tree", "project_uid": project["uid"], "path": subpath}},
+        owner_kind, owner_id, name,
+    )
+    return JSONResponse({"uid": uid, "status_url": f"/zips/{uid}"})
+
+
+@router.post("/{project_slug}/files/write")
+async def project_file_write(request: Request, project_slug: str, data: Annotated[ProjectFileWriteForm, Form()]):
+    user = require_user(request)
+    project = _load_project(project_slug)
+    if not is_owner(project, user):
+        return _deny(request, project)
+    try:
+        node = project_files.write_text_file(project["uid"], user, data.path, data.content)
+    except ProjectFileError as exc:
+        return _fail(request, project, exc)
+    logger.info("project %s file written: %s", project["uid"], node["path"])
+    return action_result(request, _files_url(project), data=project_files.node_to_dict(node))
+
+
+@router.post("/{project_slug}/files/upload")
+async def project_file_upload(request: Request, project_slug: str):
+    user = require_user(request)
+    project = _load_project(project_slug)
+    if not is_owner(project, user):
+        return _deny(request, project)
+    form = await request.form()
+    file = form.get("file")
+    if not file or not hasattr(file, "filename") or not file.filename:
+        return _fail(request, project, ProjectFileError("no file provided"))
+    target_dir = str(form.get("path", "") or "")
+    try:
+        content = await file.read()
+        node = project_files.store_upload(project["uid"], user, target_dir, file.filename, content)
+    except ProjectFileError as exc:
+        return _fail(request, project, exc)
+    logger.info("project %s file uploaded: %s", project["uid"], node["path"])
+    return action_result(request, _files_url(project), data=project_files.node_to_dict(node))
+
+
+@router.post("/{project_slug}/files/mkdir")
+async def project_file_mkdir(request: Request, project_slug: str, data: Annotated[ProjectFileMkdirForm, Form()]):
+    user = require_user(request)
+    project = _load_project(project_slug)
+    if not is_owner(project, user):
+        return _deny(request, project)
+    try:
+        node = project_files.make_dir(project["uid"], user, data.path)
+    except ProjectFileError as exc:
+        return _fail(request, project, exc)
+    return action_result(request, _files_url(project), data=project_files.node_to_dict(node))
+
+
+@router.post("/{project_slug}/files/move")
+async def project_file_move(request: Request, project_slug: str, data: Annotated[ProjectFileMoveForm, Form()]):
+    user = require_user(request)
+    project = _load_project(project_slug)
+    if not is_owner(project, user):
+        return _deny(request, project)
+    try:
+        node = project_files.move_node(project["uid"], user, data.from_path, data.to_path)
+    except ProjectFileError as exc:
+        return _fail(request, project, exc)
+    return action_result(request, _files_url(project), data=project_files.node_to_dict(node))
+
+
+@router.post("/{project_slug}/files/delete")
+async def project_file_delete(request: Request, project_slug: str, data: Annotated[ProjectFileDeleteForm, Form()]):
+    user = require_user(request)
+    project = _load_project(project_slug)
+    if not is_owner(project, user):
+        return _deny(request, project)
+    try:
+        project_files.delete_node(project["uid"], data.path)
+    except ProjectFileError as exc:
+        return _fail(request, project, exc)
+    return action_result(request, _files_url(project), data={"path": data.path})

devplacepy/routers/projects.py

@@ -1,52 +1,51 @@
 import logging
-from datetime import datetime
-from fastapi import APIRouter, Request, HTTPException
-from fastapi.responses import HTMLResponse, RedirectResponse
-from devplacepy.database import get_table, get_vote_counts, load_comments, get_attachments, delete_attachments
+from typing import Annotated
+from sqlalchemy import or_
+from fastapi import APIRouter, Request, Form
+from devplacepy.models import ProjectForm
+from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse
+from devplacepy.database import get_table, get_users_by_uids, get_site_stats, get_user_votes, paginate, resolve_by_slug
+from devplacepy.services.jobs import queue
+from devplacepy.content import load_detail, delete_content_item, create_content_item, detail_context, canonical_redirect, first_image_url
 from devplacepy.templating import templates
-from devplacepy.utils import generate_uid, get_current_user, require_user, time_ago, make_combined_slug, create_mention_notifications
-from devplacepy.seo import base_seo_context, site_url, website_schema, breadcrumb_schema, combine, software_application_schema
+from devplacepy.utils import get_current_user, require_user, not_found, XP_PROJECT
+from devplacepy.seo import base_seo_context, site_url, website_schema, software_application_schema, list_page_seo, next_page_url
+from devplacepy.responses import respond, action_result
+from devplacepy.schemas import ProjectsOut, ProjectDetailOut
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
 
-def get_projects_list(tab: str = "recent", search: str = "", user_uid: str = None, project_type: str = None):
+def get_projects_list(tab: str = "recent", search: str = "", user_uid: str = None, project_type: str = None, before: str = None, viewer: dict = None):
     projects = get_table("projects")
-    all_projects = list(projects.all())
 
+    filters = {}
     if user_uid:
-        all_projects = [p for p in all_projects if p["user_uid"] == user_uid]
-
+        filters["user_uid"] = user_uid
     if project_type:
-        all_projects = [p for p in all_projects if p.get("project_type") == project_type]
+        filters["project_type"] = project_type
+    if tab == "released":
+        filters["status"] = "Released"
 
-    if search:
-        search_lower = search.lower()
-        all_projects = [
-            p for p in all_projects
-            if search_lower in p.get("title", "").lower()
-            or search_lower in p.get("description", "").lower()
-        ]
+    clauses = []
+    if search and projects.exists:
+        like = f"%{search}%"
+        clauses.append(or_(projects.table.columns.title.ilike(like), projects.table.columns.description.ilike(like)))
 
-    if all_projects:
-        from devplacepy.database import get_users_by_uids
-        uids = [p["user_uid"] for p in all_projects]
-        users_map = get_users_by_uids(uids)
-        for p in all_projects:
+    order = ["-stars", "-created_at"] if tab == "popular" else ["-created_at"]
+    total = projects.count(*clauses, **filters)
+    page, next_cursor = paginate(projects, *clauses, before=before, order=order, **filters)
+
+    if page:
+        users_map = get_users_by_uids([p["user_uid"] for p in page])
+        my_votes = get_user_votes(viewer["uid"], [p["uid"] for p in page]) if viewer else {}
+        for p in page:
             author = users_map.get(p["user_uid"])
             p["author_name"] = author["username"] if author else "Unknown"
+            p["my_vote"] = my_votes.get(p["uid"], 0)
 
-    if tab == "released":
-        all_projects = [p for p in all_projects if p.get("status") == "Released"]
-    elif tab == "popular":
-        all_projects.sort(key=lambda p: int(p.get("stars", 0)), reverse=True)
-    elif tab == "new":
-        all_projects.sort(key=lambda p: p.get("created_at", ""), reverse=True)
-    else:
-        all_projects.sort(key=lambda p: p.get("created_at", ""), reverse=True)
-
-    return all_projects
+    return page, next_cursor, total
 
 
 @router.get("", response_class=HTMLResponse)
@@ -56,24 +55,23 @@ async def projects_page(
     search: str = "",
     user_uid: str = None,
     project_type: str = None,
+    before: str = None,
 ):
     user = get_current_user(request)
-    projects = get_projects_list(tab, search, user_uid, project_type)
-    users = get_table("users")
-    total_members = len(list(users.all()))
+    projects, next_cursor, total_count = get_projects_list(tab, search, user_uid, project_type, before, viewer=user)
+    total_members = get_site_stats()["total_members"]
 
-    base = site_url(request)
-    seo_ctx = base_seo_context(
+    seo_ctx = list_page_seo(
         request,
         title="Projects",
-        description=f"Explore {len(projects)} developer projects on DevPlace. Games, software, mobile apps and more.",
+        description=f"Explore {total_count} developer projects on DevPlace. Games, software, mobile apps and more.",
         breadcrumbs=[
             {"name": "Home", "url": "/feed"},
             {"name": "Projects", "url": "/projects"},
         ],
-        schemas=[website_schema(base)],
+        next_url=next_page_url(request, next_cursor),
     )
-    return templates.TemplateResponse("projects.html", {
+    return respond(request, "projects.html", {
         **seo_ctx,
         "request": request,
         "user": user,
@@ -81,37 +79,29 @@ async def projects_page(
         "current_tab": tab,
         "search": search,
         "project_type": project_type,
-        "total_count": len(projects),
+        "total_count": total_count,
+        "next_cursor": next_cursor,
         "total_members": total_members,
-    })
+    }, model=ProjectsOut)
 
 
 @router.get("/{project_slug}", response_class=HTMLResponse)
 async def project_detail(request: Request, project_slug: str):
     user = get_current_user(request)
-    projects = get_table("projects")
-    project = resolve_by_slug(projects, project_slug)
-    if not project:
-        raise HTTPException(status_code=404, detail="Project not found")
-
-    from devplacepy.database import get_users_by_uids
-    users_map = get_users_by_uids([project["user_uid"]])
-    author = users_map.get(project["user_uid"])
-
-    is_owner = user and user["uid"] == project["user_uid"]
-
-    ups, downs = get_vote_counts([project["uid"]])
-    star_count = ups.get(project["uid"], 0) - downs.get(project["uid"], 0)
-
-    comments = load_comments("project", project["uid"])
-
-    project_attachments = get_attachments("project", project["uid"])
+    detail = load_detail("projects", "project", project_slug, user)
+    if not detail:
+        raise not_found("Project not found")
+    project = detail["item"]
+    redirect = canonical_redirect("projects", project, project_slug)
+    if redirect:
+        return redirect
 
     base = site_url(request)
     seo_ctx = base_seo_context(
         request,
         title=project.get("title", "Project"),
         description=project.get("description", "")[:160],
+        og_image=first_image_url(project, detail["attachments"]),
         breadcrumbs=[
             {"name": "Home", "url": "/feed"},
             {"name": "Projects", "url": "/projects"},
@@ -119,63 +109,47 @@ async def project_detail(request: Request, project_slug: str):
         ],
         schemas=[website_schema(base), software_application_schema(project, base)],
     )
-    return templates.TemplateResponse("project_detail.html", {
-        **seo_ctx,
-        "request": request,
-        "user": user,
-        "project": project,
-        "author": author,
-        "is_owner": is_owner,
-        "star_count": star_count,
+    return respond(request, "project_detail.html", detail_context(request, user, detail, "project", seo_ctx, {
         "platforms": project.get("platforms", "").split(",") if project.get("platforms") else [],
-        "comments": comments,
-        "attachments": project_attachments,
-    })
+    }), model=ProjectDetailOut)
+
+
+@router.post("/{project_slug}/zip")
+async def zip_project(request: Request, project_slug: str):
+    project = resolve_by_slug(get_table("projects"), project_slug)
+    if not project:
+        raise not_found("Project not found")
+    user = get_current_user(request)
+    owner_kind, owner_id = ("user", user["uid"]) if user else ("guest", request.client.host or "anon")
+    uid = queue.enqueue(
+        "zip",
+        {"source": {"type": "project_tree", "project_uid": project["uid"], "path": ""}},
+        owner_kind, owner_id,
+        project.get("title") or "project",
+    )
+    return JSONResponse({"uid": uid, "status_url": f"/zips/{uid}"})
 
 
 @router.post("/delete/{project_slug}")
 async def delete_project(request: Request, project_slug: str):
     user = require_user(request)
-    projects = get_table("projects")
-    project = resolve_by_slug(projects, project_slug)
-    if project and project["user_uid"] == user["uid"]:
-        delete_attachments("project", project["uid"])
-        projects.delete(id=project["id"])
-        logger.info(f"Project {project['uid']} deleted by {user['username']}")
-    return RedirectResponse(url="/projects", status_code=302)
+    return delete_content_item(request, "projects", "project", user, project_slug, "/projects")
 
 
 @router.post("/create")
-async def create_project(request: Request):
+async def create_project(request: Request, data: Annotated[ProjectForm, Form()]):
     user = require_user(request)
-    form = await request.form()
-    title = form.get("title", "").strip()
-    description = form.get("description", "").strip()
-    release_date = form.get("release_date", "")
-    demo_date = form.get("demo_date", "")
-    project_type = form.get("project_type", "software")
-    platforms = form.get("platforms", "").strip()
-    status = form.get("status", "In Development")
+    title = data.title.strip()
+    description = data.description.strip()
 
-    if not title or not description:
-        return RedirectResponse(url="/projects", status_code=302)
-
-    projects = get_table("projects")
-    uid = generate_uid()
-    project_slug = make_combined_slug(title, uid)
-    projects.insert({
-        "uid": uid,
-        "user_uid": user["uid"],
+    uid, project_slug = create_content_item("projects", "project", user, {
         "title": title,
-        "slug": project_slug,
         "description": description,
-        "release_date": release_date or None,
-        "demo_date": demo_date or None,
-        "project_type": project_type,
-        "platforms": platforms,
-        "status": status,
-        "stars": 0,
-        "created_at": datetime.utcnow().isoformat(),
-    })
-
-    attachment_uids = form.getlist("attachment_uids") if hasattr(form, "getlist") else []
+        "release_date": data.release_date or None,
+        "demo_date": data.demo_date or None,
+        "project_type": data.project_type,
+        "platforms": data.platforms.strip(),
+        "status": data.status,
+    }, title, XP_PROJECT, "First Project", description, data.attachment_uids)
+    url = f"/projects/{project_slug}"
+    return action_result(request, url, data={"uid": uid, "slug": project_slug, "url": url})

devplacepy/routers/push.py

@@ -0,0 +1,65 @@
+# retoor <retoor@molodetz.nl>
+
+import logging
+from fastapi import APIRouter, Request
+from fastapi.responses import FileResponse, JSONResponse
+from devplacepy import push
+from devplacepy.config import STATIC_DIR
+from devplacepy.utils import require_user_api
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+WELCOME_PAYLOAD = {
+    "title": "DevPlace",
+    "message": "Push notifications enabled.",
+    "icon": "/static/apple-touch-icon.png",
+    "url": "/notifications",
+}
+
+
+@router.get("/push.json")
+async def push_public_key() -> JSONResponse:
+    return JSONResponse({"publicKey": push.public_key_standard_b64()})
+
+
+@router.post("/push.json")
+async def push_register(request: Request) -> JSONResponse:
+    user = require_user_api(request)
+    try:
+        body = await request.json()
+    except ValueError:
+        return JSONResponse({"error": "Invalid JSON"}, status_code=400)
+
+    keys = body.get("keys") if isinstance(body, dict) else None
+    if not (isinstance(keys, dict) and body.get("endpoint") and keys.get("p256dh") and keys.get("auth")):
+        return JSONResponse({"error": "Invalid request"}, status_code=400)
+
+    _, created = await push.register(
+        user_uid=user["uid"],
+        endpoint=body["endpoint"],
+        key_auth=keys["auth"],
+        key_p256dh=keys["p256dh"],
+    )
+
+    if created:
+        try:
+            await push.notify_user(user["uid"], WELCOME_PAYLOAD)
+        except Exception as exc:
+            logger.warning("Welcome push failed for %s: %s", user["uid"], exc)
+
+    return JSONResponse({"registered": True})
+
+
+@router.get("/service-worker.js")
+async def service_worker() -> FileResponse:
+    return FileResponse(
+        STATIC_DIR / "service-worker.js",
+        media_type="application/javascript",
+        headers={"Service-Worker-Allowed": "/", "Cache-Control": "no-cache"},
+    )
+
+
+@router.get("/manifest.json")
+async def manifest() -> FileResponse:
+    return FileResponse(STATIC_DIR / "manifest.json", media_type="application/manifest+json")

devplacepy/routers/reactions.py

@@ -0,0 +1,46 @@
+# retoor <retoor@molodetz.nl>
+import logging
+from typing import Annotated
+from datetime import datetime, timezone
+from fastapi import APIRouter, Request, Form
+from fastapi.responses import RedirectResponse, JSONResponse
+from devplacepy.database import get_table, db
+from devplacepy.utils import generate_uid, require_user
+from devplacepy.models import ReactionForm
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+REACTABLE: set[str] = {"post", "comment", "gist", "project"}
+
+
+@router.post("/{target_type}/{target_uid}")
+async def react(request: Request, target_type: str, target_uid: str, data: Annotated[ReactionForm, Form()]):
+    user = require_user(request)
+    if target_type not in REACTABLE:
+        return JSONResponse({"error": "Invalid target"}, status_code=400)
+
+    emoji = data.emoji
+    reactions = get_table("reactions")
+    existing = reactions.find_one(user_uid=user["uid"], target_uid=target_uid, target_type=target_type, emoji=emoji)
+    if existing:
+        reactions.delete(id=existing["id"])
+    else:
+        reactions.insert({
+            "uid": generate_uid(),
+            "user_uid": user["uid"],
+            "target_uid": target_uid,
+            "target_type": target_type,
+            "emoji": emoji,
+            "created_at": datetime.now(timezone.utc).isoformat(),
+        })
+
+    counts = {row["emoji"]: row["c"] for row in db.query(
+        "SELECT emoji, COUNT(*) as c FROM reactions WHERE target_type=:tt AND target_uid=:tu GROUP BY emoji",
+        tt=target_type, tu=target_uid,
+    )}
+    mine = [row["emoji"] for row in reactions.find(user_uid=user["uid"], target_uid=target_uid, target_type=target_type)]
+
+    if request.headers.get("x-requested-with") == "fetch":
+        return JSONResponse({"counts": counts, "mine": mine})
+    return RedirectResponse(url=request.headers.get("Referer", "/feed"), status_code=302)

devplacepy/routers/seo.py

@@ -1,6 +1,6 @@
 import logging
 from fastapi import APIRouter, Request
-from fastapi.responses import HTMLResponse, PlainTextResponse
+from fastapi.responses import PlainTextResponse, Response
 from devplacepy.seo import make_sitemap, site_url
 
 logger = logging.getLogger(__name__)
@@ -17,16 +17,18 @@ Disallow: /notifications/
 Disallow: /votes/
 Disallow: /avatar/
 Disallow: /follow/
+Disallow: /admin/
+Disallow: /uploads/
 Disallow: /*?tab=
 Disallow: /*?sort=
 Allow: /static/
 
 Sitemap: {base}/sitemap.xml
-""")
+""", headers={"Cache-Control": "public, max-age=3600"})
 
 
-@router.get("/sitemap.xml", response_class=HTMLResponse)
+@router.get("/sitemap.xml")
 async def sitemap_xml(request: Request):
     base = site_url(request)
     xml = make_sitemap(base)
-    return HTMLResponse(content=xml, media_type="application/xml")
+    return Response(content=xml, media_type="application/xml", headers={"Cache-Control": "public, max-age=3600"})

devplacepy/routers/services.py

@@ -2,7 +2,7 @@ import logging
 from fastapi import APIRouter, Request
 from fastapi.responses import HTMLResponse, JSONResponse
 from devplacepy.templating import templates
-from devplacepy.utils import require_admin
+from devplacepy.utils import require_admin, not_found
 from devplacepy.services.manager import service_manager
 from devplacepy.seo import base_seo_context, site_url, website_schema
 
@@ -13,12 +13,12 @@ router = APIRouter()
 @router.get("", response_class=HTMLResponse)
 async def services_page(request: Request):
     admin = require_admin(request)
-    services = service_manager.list_services()
+    services = service_manager.describe_all()
     base = site_url(request)
     seo_ctx = base_seo_context(
         request,
-        title="Services — Admin",
-        description="Monitor background services on DevPlace.",
+        title="Services - Admin",
+        description="Monitor and configure background services on DevPlace.",
         breadcrumbs=[
             {"name": "Home", "url": "/feed"},
             {"name": "Admin", "url": "/admin"},
@@ -26,7 +26,7 @@ async def services_page(request: Request):
         ],
         schemas=[website_schema(base)],
     )
-    return templates.TemplateResponse("services.html", {
+    return templates.TemplateResponse(request, "services.html", {
         **seo_ctx,
         "request": request,
         "user": admin,
@@ -38,5 +38,87 @@ async def services_page(request: Request):
 @router.get("/data")
 async def services_data(request: Request):
     require_admin(request)
-    services = service_manager.list_services()
-    return JSONResponse({"services": services})
+    return JSONResponse({"services": service_manager.describe_all()})
+
+
+@router.get("/{name}", response_class=HTMLResponse)
+async def service_detail(request: Request, name: str):
+    admin = require_admin(request)
+    svc = service_manager.get_service(name)
+    if svc is None:
+        raise not_found("Service not found")
+    info = svc.describe()
+    base = site_url(request)
+    seo_ctx = base_seo_context(
+        request,
+        title=f"{info['title']} - Services",
+        description=info["description"] or f"Configure the {info['title']} service.",
+        robots="noindex",
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {"name": "Admin", "url": "/admin"},
+            {"name": "Services", "url": "/admin/services"},
+            {"name": info["title"], "url": f"/admin/services/{name}"},
+        ],
+        schemas=[website_schema(base)],
+    )
+    return templates.TemplateResponse(request, "service_detail.html", {
+        **seo_ctx,
+        "request": request,
+        "user": admin,
+        "service": info,
+        "admin_section": "services",
+    })
+
+
+@router.get("/{name}/data")
+async def service_detail_data(request: Request, name: str):
+    require_admin(request)
+    svc = service_manager.get_service(name)
+    if svc is None:
+        return JSONResponse({"error": "unknown service"}, status_code=404)
+    return JSONResponse({"service": svc.describe()})
+
+
+@router.post("/{name}/start")
+async def service_start(request: Request, name: str):
+    require_admin(request)
+    if not service_manager.set_enabled(name, True):
+        return JSONResponse({"ok": False, "error": "unknown service"}, status_code=404)
+    return JSONResponse({"ok": True})
+
+
+@router.post("/{name}/stop")
+async def service_stop(request: Request, name: str):
+    require_admin(request)
+    if not service_manager.set_enabled(name, False):
+        return JSONResponse({"ok": False, "error": "unknown service"}, status_code=404)
+    return JSONResponse({"ok": True})
+
+
+@router.post("/{name}/run")
+async def service_run(request: Request, name: str):
+    require_admin(request)
+    if not service_manager.send_command(name, "run"):
+        return JSONResponse({"ok": False, "error": "unknown service"}, status_code=404)
+    return JSONResponse({"ok": True})
+
+
+@router.post("/{name}/clear-logs")
+async def service_clear_logs(request: Request, name: str):
+    require_admin(request)
+    if not service_manager.send_command(name, "clear"):
+        return JSONResponse({"ok": False, "error": "unknown service"}, status_code=404)
+    return JSONResponse({"ok": True})
+
+
+@router.post("/{name}/config")
+async def service_config(request: Request, name: str):
+    require_admin(request)
+    form = await request.form()
+    result = service_manager.save_config(name, dict(form))
+    if result is None:
+        return JSONResponse({"ok": False, "error": "unknown service"}, status_code=404)
+    if not result["ok"]:
+        return JSONResponse(result, status_code=400)
+    return JSONResponse(result)

devplacepy/routers/uploads.py

@@ -1,54 +1,27 @@
-import hashlib
 import logging
-import aiofiles
 from pathlib import Path
 from fastapi import APIRouter, Request
 from fastapi.responses import JSONResponse
-from devplacepy.database import get_table, get_setting
-from devplacepy.config import STATIC_DIR
-from devplacepy.utils import generate_uid, require_user
+from devplacepy.database import get_table, get_int_setting
+from devplacepy.utils import require_user_api
+from devplacepy.attachments import store_attachment, delete_attachment as _delete_attachment, is_extension_allowed
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
-IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".bmp"}
-
-
-def _store_file(content: bytes, original_filename: str) -> tuple[str, str]:
-    uid = generate_uid()
-    ext = Path(original_filename).suffix.lower() or ""
-    stored_name = f"{uid}{ext}"
-    hash_str = hashlib.sha256(stored_name.encode()).hexdigest()
-    subdir = f"{hash_str[:2]}/{hash_str[2:4]}"
-    storage_path = f"{subdir}/{stored_name}"
-    full_dir = STATIC_DIR / "uploads" / subdir
-    full_dir.mkdir(parents=True, exist_ok=True)
-    full_path = full_dir / stored_name
-    with open(str(full_path), "wb") as f:
-        f.write(content)
-    return uid, storage_path
-
 
 @router.post("/upload")
 async def upload_file(request: Request):
-    user = require_user(request)
+    user = require_user_api(request)
     form = await request.form()
     file = form.get("file")
 
     if not file or not hasattr(file, "filename") or not file.filename:
         return JSONResponse({"error": "No file provided"}, status_code=400)
 
-    max_size_mb = int(get_setting("max_upload_size_mb", "10"))
-    max_size_bytes = max_size_mb * 1024 * 1024
-    allowed_types_raw = get_setting("allowed_file_types", "").strip()
-    allowed_extensions = set()
-    if allowed_types_raw:
-        for ext in allowed_types_raw.split(","):
-            ext = ext.strip().lower()
-            if ext.startswith("."):
-                allowed_extensions.add(ext)
-            else:
-                allowed_extensions.add(f".{ext}")
+    ext = Path(file.filename).suffix.lower()
+    if not is_extension_allowed(ext):
+        return JSONResponse({"error": f"File type '{ext}' not allowed"}, status_code=415)
 
     try:
         content = await file.read()
@@ -56,61 +29,23 @@ async def upload_file(request: Request):
         logger.warning(f"Failed to read uploaded file: {e}")
         return JSONResponse({"error": "Failed to read file"}, status_code=400)
 
-    if len(content) > max_size_bytes:
+    result = store_attachment(content, file.filename, user["uid"])
+    if result is None:
+        max_size_mb = get_int_setting("max_upload_size_mb", 10)
         return JSONResponse({"error": f"File exceeds {max_size_mb}MB limit"}, status_code=413)
 
-    ext = Path(file.filename).suffix.lower()
-    if allowed_extensions and ext not in allowed_extensions:
-        return JSONResponse({"error": f"File type '{ext}' not allowed"}, status_code=415)
-
-    uid, storage_path = _store_file(content, file.filename)
-
-    attachments = get_table("attachments")
-    attachments.insert({
-        "uid": uid,
-        "resource_uid": "",
-        "resource_type": "",
-        "original_filename": file.filename,
-        "stored_filename": f"{uid}{ext}",
-        "mime_type": file.content_type or "application/octet-stream",
-        "file_size": len(content),
-        "storage_path": storage_path,
-        "created_at": __import__("datetime").datetime.utcnow().isoformat(),
-    })
-
-    is_image = ext in IMAGE_EXTENSIONS
-    file_url = f"/static/uploads/{storage_path}"
-
-    logger.info(f"File uploaded: {file.filename} ({len(content)} bytes) -> {storage_path}")
-    return JSONResponse({
-        "uid": uid,
-        "original_filename": file.filename,
-        "url": file_url,
-        "mime_type": file.content_type or "application/octet-stream",
-        "file_size": len(content),
-        "is_image": is_image,
-    }, status_code=201)
+    logger.info(f"File uploaded: {file.filename} ({len(content)} bytes) -> {result['url']}")
+    return JSONResponse(result, status_code=201)
 
 
 @router.delete("/delete/{attachment_uid}")
-async def delete_attachment(request: Request, attachment_uid: str):
-    user = require_user(request)
-    attachments = get_table("attachments")
-    att = attachments.find_one(uid=attachment_uid)
+async def delete_attachment_route(request: Request, attachment_uid: str):
+    user = require_user_api(request)
+    att = get_table("attachments").find_one(uid=attachment_uid)
     if not att:
         return JSONResponse({"error": "Attachment not found"}, status_code=404)
-    if att.get("resource_uid"):
-        resource_type = att["resource_type"]
-        if resource_type == "post":
-            post = get_table("posts").find_one(uid=att["resource_uid"])
-            if not post or post["user_uid"] != user["uid"]:
-                return JSONResponse({"error": "Not authorized"}, status_code=403)
-        elif resource_type == "comment":
-            comment = get_table("comments").find_one(uid=att["resource_uid"])
-            if not comment or comment["user_uid"] != user["uid"]:
-                return JSONResponse({"error": "Not authorized"}, status_code=403)
-    from devplacepy.database import _delete_attachment_file
-    _delete_attachment_file(att.get("storage_path", ""))
-    attachments.delete(id=att["id"])
+    if att.get("user_uid") and att["user_uid"] != user["uid"]:
+        return JSONResponse({"error": "Not authorized"}, status_code=403)
+    _delete_attachment(attachment_uid)
     logger.info(f"Attachment {attachment_uid} deleted by {user['username']}")
     return JSONResponse({"status": "deleted"})

devplacepy/routers/votes.py

@@ -1,31 +1,33 @@
 import logging
-from datetime import datetime
-from fastapi import APIRouter, Request
-from fastapi.responses import RedirectResponse
-from devplacepy.database import get_table
-from devplacepy.utils import generate_uid, require_user
+from typing import Annotated
+from datetime import datetime, timezone
+from fastapi import APIRouter, Request, Form
+from fastapi.responses import RedirectResponse, JSONResponse
+from devplacepy.database import get_table, update_target_stars, get_target_owner_uid, resolve_object_url
+from devplacepy.utils import generate_uid, require_user, create_notification, award_rewards, XP_UPVOTE
+from devplacepy.models import VoteForm
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
+NOTIFY_ON_VOTE: set[str] = {"post", "comment", "gist", "project"}
+
 
 @router.post("/{target_type}/{target_uid}")
-async def vote(request: Request, target_type: str, target_uid: str):
+async def vote(request: Request, target_type: str, target_uid: str, data: Annotated[VoteForm, Form()]):
     user = require_user(request)
-    form = await request.form()
-    value = int(form.get("value", "1"))
-
-    if value not in (1, -1):
-        return RedirectResponse(url="/feed", status_code=302)
+    value = data.value
 
     votes = get_table("votes")
     existing = votes.find_one(user_uid=user["uid"], target_uid=target_uid, target_type=target_type)
 
+    did_upvote = False
     if existing:
         if int(existing["value"]) == value:
             votes.delete(id=existing["id"])
         else:
             votes.update({"id": existing["id"], "value": value}, ["id"])
+            did_upvote = value == 1
     else:
         votes.insert({
             "uid": generate_uid(),
@@ -33,52 +35,28 @@ async def vote(request: Request, target_type: str, target_uid: str):
             "target_uid": target_uid,
             "target_type": target_type,
             "value": value,
-            "created_at": datetime.utcnow().isoformat(),
+            "created_at": datetime.now(timezone.utc).isoformat(),
         })
+        did_upvote = value == 1
 
-    up_count = len(list(votes.find(target_uid=target_uid, value=1)))
-    down_count = len(list(votes.find(target_uid=target_uid, value=-1)))
+    up_count = votes.count(target_uid=target_uid, target_type=target_type, value=1)
+    down_count = votes.count(target_uid=target_uid, target_type=target_type, value=-1)
     net = up_count - down_count
 
-    if target_type == "post":
-        posts = get_table("posts")
-        posts.update({"uid": target_uid, "stars": net}, ["uid"])
-    elif target_type == "project":
-        projects = get_table("projects")
-        projects.update({"uid": target_uid, "stars": net}, ["uid"])
-    elif target_type == "gist":
-        gists = get_table("gists")
-        gists.update({"uid": target_uid, "stars": net}, ["uid"])
+    update_target_stars(target_type, target_uid, net)
 
-    if value == 1:
-        target_owner_uid = None
-        if target_type == "post":
-            target_owner = posts.find_one(uid=target_uid)
-            if target_owner:
-                target_owner_uid = target_owner["user_uid"]
-        elif target_type == "comment":
-            comments = get_table("comments")
-            target_comment = comments.find_one(uid=target_uid)
-            if target_comment:
-                target_owner_uid = target_comment["user_uid"]
-        elif target_type == "gist":
-            gists = get_table("gists")
-            target_gist = gists.find_one(uid=target_uid)
-            if target_gist:
-                target_owner_uid = target_gist["user_uid"]
+    if did_upvote and target_type in NOTIFY_ON_VOTE:
+        owner_uid = get_target_owner_uid(target_type, target_uid)
+        if owner_uid and owner_uid != user["uid"]:
+            target_url = resolve_object_url(target_type, target_uid)
+            create_notification(owner_uid, "vote", f"{user['username']} ++'d your {target_type}", user["uid"], target_url)
+            award_rewards(owner_uid, XP_UPVOTE)
 
-        if target_owner_uid and target_owner_uid != user["uid"]:
-            label = "post" if target_type == "post" else "comment"
-            notifications = get_table("notifications")
-            notifications.insert({
-                "uid": generate_uid(),
-                "user_uid": target_owner_uid,
-                "type": "vote",
-                "message": f"{user['username']} ++'d your {label}",
-                "related_uid": user["uid"],
-                "read": False,
-                "created_at": datetime.utcnow().isoformat(),
-            })
+    if request.headers.get("x-requested-with") == "fetch":
+        current = votes.find_one(user_uid=user["uid"], target_uid=target_uid, target_type=target_type)
+        current_value = int(current["value"]) if current else 0
+        logger.debug("ajax vote response target=%s/%s net=%s value=%s", target_type, target_uid, net, current_value)
+        return JSONResponse({"net": net, "up": up_count, "down": down_count, "value": current_value})
 
     referer = request.headers.get("Referer", "/feed")
     return RedirectResponse(url=referer, status_code=302)

devplacepy/routers/zips.py

@@ -0,0 +1,59 @@
+# retoor <retoor@molodetz.nl>
+
+import logging
+from pathlib import Path
+
+from fastapi import APIRouter, Request
+from fastapi.responses import FileResponse, JSONResponse
+
+from devplacepy.services.jobs import queue
+from devplacepy.services.jobs.zip_service import ZIPS_DIR
+from devplacepy.schemas import ZipJobOut
+from devplacepy.utils import not_found
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+RETENTION_EXTEND_SECONDS = 7 * 24 * 60 * 60
+
+
+def _status_payload(job: dict) -> dict:
+    result = job.get("result", {})
+    done = job.get("status") == queue.DONE
+    return {
+        "uid": job.get("uid", ""),
+        "kind": job.get("kind", ""),
+        "status": job.get("status", ""),
+        "preferred_name": job.get("preferred_name"),
+        "download_url": result.get("download_url") if done else None,
+        "error": job.get("error") or None,
+        "bytes_in": int(job.get("bytes_in") or 0),
+        "bytes_out": int(job.get("bytes_out") or 0),
+        "item_count": int(job.get("item_count") or 0),
+        "file_count": int(result.get("file_count") or 0),
+        "dir_count": int(result.get("dir_count") or 0),
+        "created_at": job.get("created_at"),
+        "completed_at": job.get("completed_at") or None,
+    }
+
+
+@router.get("/{uid}")
+async def zip_status(request: Request, uid: str):
+    job = queue.get_job(uid)
+    if not job or job.get("kind") != "zip":
+        raise not_found("Job not found")
+    return JSONResponse(ZipJobOut.model_validate(_status_payload(job)).model_dump(mode="json"))
+
+
+@router.get("/{uid}/download")
+async def zip_download(request: Request, uid: str):
+    job = queue.get_job(uid)
+    if not job or job.get("kind") != "zip" or job.get("status") != queue.DONE:
+        raise not_found("Archive not available")
+    result = job.get("result", {})
+    local_path = result.get("local_path", "")
+    resolved = Path(local_path).resolve()
+    if not local_path or not resolved.is_relative_to(ZIPS_DIR.resolve()) or not resolved.is_file():
+        raise not_found("Archive not available")
+    queue.touch_job(uid, RETENTION_EXTEND_SECONDS)
+    return FileResponse(resolved, filename=result.get("final_name", "download.zip"), media_type="application/zip")

devplacepy/schemas.py

@@ -0,0 +1,554 @@
+"""Pydantic response models for JSON content negotiation.
+
+Every model uses ``extra="ignore"`` so it can be validated directly against the
+dict context a template already receives, dropping internal keys (request, SEO)
+and - importantly - sensitive user fields (email, api_key, password_hash) that
+are never part of ``UserOut``.
+"""
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from pydantic import BaseModel, ConfigDict
+
+
+class _Out(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+
+# ---------- envelopes ----------
+
+class ActionResult(_Out):
+    ok: bool = True
+    redirect: Optional[str] = None
+    data: Optional[Any] = None
+
+
+class ErrorOut(_Out):
+    error: dict
+
+
+class ValidationErrorOut(_Out):
+    error: str = "validation"
+    fields: dict = {}
+    messages: list = []
+
+
+class ZipJobOut(_Out):
+    uid: str = ""
+    kind: str = ""
+    status: str = ""
+    preferred_name: Optional[str] = None
+    download_url: Optional[str] = None
+    error: Optional[str] = None
+    bytes_in: int = 0
+    bytes_out: int = 0
+    item_count: int = 0
+    file_count: int = 0
+    dir_count: int = 0
+    created_at: Optional[str] = None
+    completed_at: Optional[str] = None
+
+
+# ---------- shared resource projections ----------
+
+class UserOut(_Out):
+    uid: str = ""
+    username: str = ""
+    avatar_seed: Optional[str] = None
+    role: Optional[str] = None
+    bio: Optional[str] = None
+    location: Optional[str] = None
+    git_link: Optional[str] = None
+    website: Optional[str] = None
+    level: Optional[int] = None
+    xp: Optional[int] = None
+    stars: Optional[int] = None
+    created_at: Optional[str] = None
+
+
+class AttachmentOut(_Out):
+    uid: str = ""
+    filename: Optional[str] = None
+    url: Optional[str] = None
+    size: Optional[int] = None
+    is_image: Optional[bool] = None
+    is_video: Optional[bool] = None
+    mime_type: Optional[str] = None
+    created_at: Optional[str] = None
+
+
+class VotesOut(_Out):
+    up: int = 0
+    down: int = 0
+
+
+class ReactionsOut(_Out):
+    counts: dict = {}
+    mine: list = []
+
+
+class PollOptionOut(_Out):
+    uid: str = ""
+    label: Optional[str] = None
+    count: Optional[int] = None
+    votes: Optional[int] = None
+    pct: Optional[int] = None
+
+
+class PollOut(_Out):
+    uid: str = ""
+    question: Optional[str] = None
+    options: list[PollOptionOut] = []
+    total: int = 0
+    my_choice: Optional[str] = None
+    voted: Optional[str] = None
+
+
+class BadgeOut(_Out):
+    name: Optional[str] = None
+    created_at: Optional[str] = None
+
+
+class PostOut(_Out):
+    uid: str = ""
+    slug: Optional[str] = None
+    user_uid: Optional[str] = None
+    title: Optional[str] = None
+    content: Optional[str] = None
+    topic: Optional[str] = None
+    stars: Optional[int] = None
+    image: Optional[str] = None
+    created_at: Optional[str] = None
+    updated_at: Optional[str] = None
+
+
+class GistOut(_Out):
+    uid: str = ""
+    slug: Optional[str] = None
+    user_uid: Optional[str] = None
+    title: Optional[str] = None
+    description: Optional[str] = None
+    source_code: Optional[str] = None
+    language: Optional[str] = None
+    stars: Optional[int] = None
+    created_at: Optional[str] = None
+    updated_at: Optional[str] = None
+
+
+class ProjectOut(_Out):
+    uid: str = ""
+    slug: Optional[str] = None
+    user_uid: Optional[str] = None
+    title: Optional[str] = None
+    description: Optional[str] = None
+    project_type: Optional[str] = None
+    status: Optional[str] = None
+    platforms: Optional[Any] = None
+    stars: Optional[int] = None
+    release_date: Optional[str] = None
+    demo_date: Optional[str] = None
+    created_at: Optional[str] = None
+    updated_at: Optional[str] = None
+
+
+class ProjectFileOut(_Out):
+    uid: str = ""
+    path: str = ""
+    name: Optional[str] = None
+    type: Optional[str] = None
+    is_binary: Optional[bool] = None
+    mime_type: Optional[str] = None
+    size: Optional[int] = None
+    url: Optional[str] = None
+    updated_at: Optional[str] = None
+
+
+class ProjectFileRawOut(ProjectFileOut):
+    content: Optional[str] = None
+
+
+class ProjectFilesOut(_Out):
+    project: ProjectOut
+    files: list[ProjectFileOut] = []
+    is_owner: bool = False
+
+
+class NewsOut(_Out):
+    uid: str = ""
+    slug: Optional[str] = None
+    title: Optional[str] = None
+    description: Optional[str] = None
+    content: Optional[str] = None
+    url: Optional[str] = None
+    source_name: Optional[str] = None
+    author: Optional[str] = None
+    grade: Optional[int] = None
+    status: Optional[str] = None
+    image_url: Optional[str] = None
+    article_published: Optional[str] = None
+    created_at: Optional[str] = None
+    synced_at: Optional[str] = None
+
+
+class CommentOut(_Out):
+    uid: str = ""
+    user_uid: Optional[str] = None
+    content: Optional[str] = None
+    parent_uid: Optional[str] = None
+    target_type: Optional[str] = None
+    target_uid: Optional[str] = None
+    created_at: Optional[str] = None
+
+
+class CommentItemOut(_Out):
+    comment: CommentOut
+    author: Optional[UserOut] = None
+    time_ago: Optional[str] = None
+    votes: VotesOut = VotesOut()
+    my_vote: int = 0
+    children: list["CommentItemOut"] = []
+    attachments: list[AttachmentOut] = []
+    reactions: ReactionsOut = ReactionsOut()
+
+
+class NotificationOut(_Out):
+    uid: str = ""
+    type: Optional[str] = None
+    message: Optional[str] = None
+    read: Optional[bool] = None
+    related_uid: Optional[str] = None
+    target_url: Optional[str] = None
+    created_at: Optional[str] = None
+
+
+class MessageOut(_Out):
+    uid: str = ""
+    sender_uid: Optional[str] = None
+    receiver_uid: Optional[str] = None
+    content: Optional[str] = None
+    read: Optional[bool] = None
+    created_at: Optional[str] = None
+
+
+class BugOut(_Out):
+    uid: str = ""
+    user_uid: Optional[str] = None
+    title: Optional[str] = None
+    description: Optional[str] = None
+    status: Optional[str] = None
+    created_at: Optional[str] = None
+
+
+# ---------- list-item wrappers ----------
+
+class FeedItemOut(_Out):
+    post: PostOut
+    author: Optional[UserOut] = None
+    time_ago: Optional[str] = None
+    my_vote: int = 0
+    comment_count: int = 0
+    attachments: list[AttachmentOut] = []
+    recent_comments: list[CommentItemOut] = []
+    reactions: ReactionsOut = ReactionsOut()
+    bookmarked: bool = False
+    poll: Optional[PollOut] = None
+
+
+class GistItemOut(_Out):
+    gist: GistOut
+    author: Optional[UserOut] = None
+    time_ago: Optional[str] = None
+    my_vote: int = 0
+    comment_count: int = 0
+
+
+class ProjectListItemOut(ProjectOut):
+    author_name: Optional[str] = None
+    my_vote: int = 0
+
+
+class NewsListItemOut(_Out):
+    article: NewsOut
+    time_ago: Optional[str] = None
+    image_url: Optional[str] = None
+    grade: Optional[int] = None
+
+
+class ConversationOut(_Out):
+    other_user: Optional[UserOut] = None
+    last_message: Optional[str] = None
+    last_message_at: Optional[str] = None
+    unread: bool = False
+
+
+class MessageItemOut(_Out):
+    message: MessageOut
+    sender: Optional[UserOut] = None
+    is_mine: bool = False
+    time_ago: Optional[str] = None
+    attachments: list[AttachmentOut] = []
+
+
+class NotificationItemOut(_Out):
+    notification: NotificationOut
+    actor: Optional[UserOut] = None
+    time_ago: Optional[str] = None
+
+
+class NotificationGroupOut(_Out):
+    label: Optional[str] = None
+    entries: list[NotificationItemOut] = []
+
+
+class BugItemOut(_Out):
+    bug: BugOut
+    author: Optional[UserOut] = None
+    time_ago: Optional[str] = None
+    comments: list[CommentItemOut] = []
+    attachments: list[AttachmentOut] = []
+
+
+class LeaderboardEntryOut(_Out):
+    uid: Optional[str] = None
+    username: Optional[str] = None
+    avatar_seed: Optional[str] = None
+    stars: Optional[int] = None
+    level: Optional[int] = None
+    rank: Optional[int] = None
+
+
+class SavedItemOut(_Out):
+    target_type: Optional[str] = None
+    target_uid: Optional[str] = None
+    type_label: Optional[str] = None
+    title: Optional[str] = None
+    url: Optional[str] = None
+    time_ago: Optional[str] = None
+
+
+class AdminUserOut(UserOut):
+    email: Optional[str] = None
+    is_active: Optional[bool] = None
+    posts_count: Optional[int] = None
+
+
+class AdminNewsItemOut(_Out):
+    article: NewsOut
+    time_ago: Optional[str] = None
+    synced_at: Optional[str] = None
+    grade: Optional[int] = None
+    has_image: Optional[bool] = None
+
+
+# ---------- page envelopes ----------
+
+class FeedOut(_Out):
+    posts: list[FeedItemOut] = []
+    current_tab: Optional[str] = None
+    current_topic: Optional[str] = None
+    next_cursor: Optional[str] = None
+    total_members: Optional[int] = None
+    posts_today: Optional[int] = None
+    total_projects: Optional[int] = None
+    total_gists: Optional[int] = None
+    top_authors: list[UserOut] = []
+    daily_topic: Optional[Any] = None
+
+
+class PostDetailOut(_Out):
+    post: PostOut
+    author: Optional[UserOut] = None
+    is_owner: bool = False
+    star_count: int = 0
+    my_vote: int = 0
+    time_ago: Optional[str] = None
+    comments: list[CommentItemOut] = []
+    attachments: list[AttachmentOut] = []
+    reactions: ReactionsOut = ReactionsOut()
+    bookmarked: bool = False
+    poll: Optional[PollOut] = None
+    comment_count: Optional[int] = None
+    related_posts: list[FeedItemOut] = []
+
+
+class ProjectsOut(_Out):
+    projects: list[ProjectListItemOut] = []
+    current_tab: Optional[str] = None
+    search: Optional[str] = None
+    project_type: Optional[str] = None
+    total_count: Optional[int] = None
+    next_cursor: Optional[str] = None
+    total_members: Optional[int] = None
+
+
+class ProjectDetailOut(_Out):
+    project: ProjectOut
+    author: Optional[UserOut] = None
+    is_owner: bool = False
+    star_count: int = 0
+    my_vote: int = 0
+    time_ago: Optional[str] = None
+    comments: list[CommentItemOut] = []
+    attachments: list[AttachmentOut] = []
+    reactions: ReactionsOut = ReactionsOut()
+    bookmarked: bool = False
+    platforms: Optional[Any] = None
+
+
+class GistsOut(_Out):
+    gists: list[GistItemOut] = []
+    total_count: Optional[int] = None
+    next_cursor: Optional[str] = None
+    current_language: Optional[str] = None
+
+
+class GistDetailOut(_Out):
+    gist: GistOut
+    author: Optional[UserOut] = None
+    is_owner: bool = False
+    star_count: int = 0
+    my_vote: int = 0
+    time_ago: Optional[str] = None
+    comments: list[CommentItemOut] = []
+    attachments: list[AttachmentOut] = []
+    reactions: ReactionsOut = ReactionsOut()
+    bookmarked: bool = False
+
+
+class NewsListOut(_Out):
+    articles: list[NewsListItemOut] = []
+    next_cursor: Optional[str] = None
+
+
+class NewsDetailOut(_Out):
+    article: NewsOut
+    canonical_slug: Optional[str] = None
+    image_url: Optional[str] = None
+    grade: Optional[int] = None
+    time_ago: Optional[str] = None
+    comments: list[CommentItemOut] = []
+    bookmarked: bool = False
+
+
+class ProfileOut(_Out):
+    profile_user: UserOut
+    posts: list[FeedItemOut] = []
+    badges: list[BadgeOut] = []
+    projects: list[ProjectOut] = []
+    gists: list[GistItemOut] = []
+    current_tab: Optional[str] = None
+    posts_count: Optional[int] = None
+    is_following: bool = False
+    is_owner: bool = False
+    can_view_api_key: bool = False
+    api_key: Optional[str] = None
+    ai_quota: Optional[dict] = None
+    activities: list[Any] = []
+    rank: Optional[int] = None
+    heatmap: Optional[Any] = None
+    heatmap_months: Optional[Any] = None
+    streak: Optional[Any] = None
+    people: list[Any] = []
+    follow_pagination: Optional[Any] = None
+    followers_count: Optional[int] = None
+    following_count: Optional[int] = None
+
+
+class MessagesOut(_Out):
+    conversations: list[ConversationOut] = []
+    messages: list[MessageItemOut] = []
+    other_user: Optional[UserOut] = None
+    current_conversation: Optional[str] = None
+    search: Optional[str] = None
+
+
+class NotificationsOut(_Out):
+    notification_groups: list[NotificationGroupOut] = []
+    next_cursor: Optional[str] = None
+
+
+class LeaderboardOut(_Out):
+    entries: list[LeaderboardEntryOut] = []
+    user_rank: Optional[int] = None
+    total_members: Optional[int] = None
+    posts_today: Optional[int] = None
+    total_projects: Optional[int] = None
+    total_gists: Optional[int] = None
+    top_authors: list[UserOut] = []
+    featured_news: list[NewsOut] = []
+
+
+class BugsOut(_Out):
+    bugs: list[BugItemOut] = []
+
+
+class SavedOut(_Out):
+    items: list[SavedItemOut] = []
+    next_cursor: Optional[str] = None
+
+
+class AdminUsersOut(_Out):
+    users: list[AdminUserOut] = []
+    pagination: Optional[Any] = None
+    admin_section: Optional[str] = None
+
+
+class AdminNewsOut(_Out):
+    articles: list[AdminNewsItemOut] = []
+    pagination: Optional[Any] = None
+    admin_section: Optional[str] = None
+
+
+class AdminSettingsOut(_Out):
+    settings: dict = {}
+    admin_section: Optional[str] = None
+
+
+class GatewayUsageOut(_Out):
+    window_hours: int = 48
+    generated_at: Optional[str] = None
+    requests: int = 0
+    volume: dict = {}
+    tokens: dict = {}
+    latency: dict = {}
+    errors: dict = {}
+    cost: dict = {}
+    behavior: dict = {}
+    hourly: list = []
+    notes: dict = {}
+    admin_section: Optional[str] = None
+
+
+class UserAiUsageOut(_Out):
+    owner_id: str = ""
+    window_hours: int = 24
+    generated_at: Optional[str] = None
+    requests: int = 0
+    success: int = 0
+    failed: int = 0
+    success_pct: float = 0.0
+    error_pct: float = 0.0
+    tokens: dict = {}
+    cost: dict = {}
+    latency: dict = {}
+    first_used: Optional[str] = None
+    last_used: Optional[str] = None
+    by_model: list = []
+    by_backend: list = []
+    hourly: list = []
+    notes: dict = {}
+
+
+# ---------- auth pages ----------
+
+class AuthPageOut(_Out):
+    page: str = ""
+    next_url: Optional[str] = None
+    registration_closed: Optional[bool] = None
+    sent: Optional[bool] = None
+    token: Optional[str] = None
+    errors: list = []
+
+
+CommentItemOut.model_rebuild()

devplacepy/seo.py

@@ -1,26 +1,34 @@
 import json
 import logging
-from datetime import datetime
+import os
+import time
+from urllib.parse import urlencode
 from xml.etree.ElementTree import Element, tostring
 from xml.dom import minidom
+from devplacepy.config import SITE_URL
+from devplacepy.utils import strip_html
 
 logger = logging.getLogger(__name__)
 
 
 SITE_NAME = "DevPlace"
+SITEMAP_URL_LIMIT = 5000
+SITEMAP_TTL = int(os.environ.get("DEVPLACE_SITEMAP_TTL", "3600"))
+_sitemap_cache = {}
 
 
 def truncate(text, max_len=160):
     if not text:
         return ""
-    text = " ".join(text.split())[:max_len]
-    if len(text) >= max_len:
-        text = text.rsplit(" ", 1)[0] + "..."
-    return text
+    text = " ".join(text.split())
+    if len(text) <= max_len:
+        return text
+    text = text[:max_len - 3].rsplit(" ", 1)[0]
+    return text + "..."
 
 
 def site_url(request):
-    return str(request.base_url).rstrip("/")
+    return SITE_URL or str(request.base_url).rstrip("/")
 
 
 def website_schema(base_url):
@@ -68,13 +76,12 @@ def discussion_forum_posting(post, author, comment_count, star_count, base_url):
             "url": f"{base_url}/profile/{author['username']}" if author else ""
         },
         "datePublished": post.get("created_at", ""),
+        "dateModified": post.get("updated_at") or post.get("created_at", ""),
         "interactionStatistic": [
             {"@type": "InteractionCounter", "interactionType": "https://schema.org/LikeAction", "userInteractionCount": star_count},
             {"@type": "InteractionCounter", "interactionType": "https://schema.org/CommentAction", "userInteractionCount": comment_count}
         ]
     }
-    if post.get("title"):
-        schema["headline"] = post["title"]
     return schema
 
 
@@ -98,7 +105,7 @@ def software_application_schema(project, base_url):
         "@type": "SoftwareApplication",
         "name": project.get("title", "Untitled"),
         "description": truncate(project.get("description", ""), 300),
-        "url": f"{base_url}/projects",
+        "url": f"{base_url}/projects/{project.get('slug') or project['uid']}",
         "applicationCategory": "DeveloperApplication",
         "operatingSystem": project.get("platforms", "Cross-platform"),
         "author": {
@@ -106,6 +113,7 @@ def software_application_schema(project, base_url):
             "name": project.get("author_name", "Unknown")
         },
         "datePublished": project.get("created_at", ""),
+        "dateModified": project.get("updated_at") or project.get("created_at", ""),
         "offers": {
             "@type": "Offer",
             "price": "0",
@@ -114,6 +122,56 @@ def software_application_schema(project, base_url):
     }
 
 
+def organization_schema(base_url):
+    return {
+        "@type": "Organization",
+        "name": SITE_NAME,
+        "url": base_url,
+        "logo": f"{base_url}{DEFAULT_OG_IMAGE}",
+    }
+
+
+def news_article_schema(article, base_url, image_url=""):
+    url = f"{base_url}/news/{article.get('slug') or article['uid']}"
+    schema = {
+        "@type": "NewsArticle",
+        "headline": (article.get("title") or "Untitled")[:110],
+        "description": truncate(strip_html(article.get("description", "") or ""), 200),
+        "url": url,
+        "datePublished": article.get("synced_at", "") or article.get("created_at", ""),
+        "dateModified": article.get("synced_at", "") or article.get("created_at", ""),
+        "mainEntityOfPage": {"@type": "WebPage", "@id": url},
+        "author": {"@type": "Organization", "name": article.get("source_name") or SITE_NAME},
+        "publisher": organization_schema(base_url),
+    }
+    if image_url:
+        schema["image"] = image_url
+    return schema
+
+
+def software_source_code_schema(gist, base_url):
+    return {
+        "@type": "SoftwareSourceCode",
+        "name": gist.get("title") or "Gist",
+        "description": truncate(strip_html(gist.get("description", "") or ""), 200),
+        "url": f"{base_url}/gists/{gist.get('slug') or gist['uid']}",
+        "programmingLanguage": gist.get("language", "") or "text",
+        "dateCreated": gist.get("created_at", ""),
+        "dateModified": gist.get("updated_at") or gist.get("created_at", ""),
+    }
+
+
+def _json_ld_dumps(payload):
+    raw = json.dumps(payload, ensure_ascii=False)
+    return (
+        raw.replace("<", "\\u003c")
+           .replace(">", "\\u003e")
+           .replace("&", "\\u0026")
+           .replace("
", "\\u2028")
+           .replace("
", "\\u2029")
+    )
+
+
 def combine(schemas):
     if not schemas:
         return None
@@ -124,34 +182,89 @@ def combine(schemas):
             cleaned.append(s)
     if not cleaned:
         return None
-    if len(cleaned) == 1:
-        return json.dumps({"@context": "https://schema.org", **cleaned[0]}, ensure_ascii=False)
-    return json.dumps({"@context": "https://schema.org", "@graph": cleaned}, ensure_ascii=False)
+    payload = (
+        {"@context": "https://schema.org", **cleaned[0]}
+        if len(cleaned) == 1
+        else {"@context": "https://schema.org", "@graph": cleaned}
+    )
+    return _json_ld_dumps(payload)
 
 
-DEFAULT_OG_IMAGE = "/static/og-default.svg"
+DEFAULT_OG_IMAGE = "/static/og-default.png"
 
 
-def base_seo_context(request, title="", description="", robots="index,follow", og_type="website", og_image=None, breadcrumbs=None, schemas=None):
+def absolute_url(base, path):
+    if not path:
+        return ""
+    return path if path.startswith("http") else f"{base}{path}"
+
+
+def base_seo_context(request, title="", description="", robots="index,follow", og_type="website", og_image=None, breadcrumbs=None, schemas=None, prev_url=None, next_url=None):
     base = site_url(request)
-    page_title = f"{title} — {SITE_NAME}" if title else SITE_NAME
-    canonical = str(request.url).split("?")[0]
-    og_img = og_image or f"{base}{DEFAULT_OG_IMAGE}"
+    page_title = f"{title} - {SITE_NAME}" if title else SITE_NAME
+    canonical = f"{base}{request.url.path}"
+    page = request.query_params.get("page")
+    if page and page not in ("", "1"):
+        canonical = f"{canonical}?page={page}"
+    clean_description = truncate(strip_html(description), 160)
+    og_img = absolute_url(base, og_image) or f"{base}{DEFAULT_OG_IMAGE}"
+    page_schemas = list(schemas or [])
+    if breadcrumbs:
+        page_schemas.append(breadcrumb_schema(breadcrumbs, base))
     return {
         "page_title": page_title,
-        "meta_description": description,
+        "meta_description": clean_description,
         "meta_robots": robots,
         "canonical_url": canonical,
         "og_title": title or SITE_NAME,
-        "og_description": description,
+        "og_description": clean_description,
         "og_image": og_img,
         "og_type": og_type,
         "breadcrumbs": breadcrumbs or [],
-        "page_schema": combine(schemas or []),
+        "page_schema": combine(page_schemas),
+        "prev_url": absolute_url(base, prev_url),
+        "next_url": absolute_url(base, next_url),
     }
 
 
+def next_page_url(request, next_cursor):
+    if not next_cursor:
+        return None
+    params = dict(request.query_params)
+    params["before"] = next_cursor
+    return f"{request.url.path}?{urlencode(params)}"
+
+
+def list_page_seo(request, title="", description="", breadcrumbs=None, prev_url=None, next_url=None):
+    base = site_url(request)
+    return base_seo_context(
+        request,
+        title=title,
+        description=description,
+        breadcrumbs=breadcrumbs,
+        schemas=[website_schema(base)],
+        prev_url=prev_url,
+        next_url=next_url,
+    )
+
+
 def make_sitemap(base_url):
+    cached = _sitemap_cache.get(base_url)
+    if cached and time.time() - cached[0] < SITEMAP_TTL:
+        return cached[1]
+    xml = _build_sitemap(base_url)
+    _sitemap_cache[base_url] = (time.time(), xml)
+    return xml
+
+
+def _collect(table, limit, label, **query):
+    rows = list(table.find(_limit=limit, **query))
+    if len(rows) >= limit:
+        logger.warning("sitemap: %s truncated at %d entries", label, limit)
+    return rows
+
+
+def _build_sitemap(base_url):
     from devplacepy.database import get_table, db
 
     def url_element(loc, lastmod=None, changefreq=None, priority=None):
@@ -180,9 +293,11 @@ def make_sitemap(base_url):
     urlset.append(url_element(f"{base_url}/feed", changefreq="hourly", priority="0.9"))
     urlset.append(url_element(f"{base_url}/news", changefreq="hourly", priority="0.9"))
     urlset.append(url_element(f"{base_url}/projects", changefreq="daily", priority="0.8"))
+    urlset.append(url_element(f"{base_url}/gists", changefreq="daily", priority="0.8"))
+    urlset.append(url_element(f"{base_url}/leaderboard", changefreq="daily", priority="0.7"))
 
     if "posts" in db.tables:
-        posts = list(get_table("posts").find(order_by=["-created_at"], _limit=500))
+        posts = _collect(get_table("posts"), SITEMAP_URL_LIMIT, "posts", order_by=["-created_at"])
         for p in posts:
             urlset.append(url_element(
                 f"{base_url}/posts/{p.get('slug') or p['uid']}",
@@ -192,7 +307,7 @@ def make_sitemap(base_url):
             ))
 
     if "projects" in db.tables:
-        projects = list(get_table("projects").find(order_by=["-created_at"], _limit=1000))
+        projects = _collect(get_table("projects"), SITEMAP_URL_LIMIT, "projects", order_by=["-created_at"])
         for p in projects:
             urlset.append(url_element(
                 f"{base_url}/projects/{p.get('slug') or p['uid']}",
@@ -202,7 +317,7 @@ def make_sitemap(base_url):
             ))
 
     if "gists" in db.tables:
-        gists = list(get_table("gists").find(order_by=["-created_at"], _limit=500))
+        gists = _collect(get_table("gists"), SITEMAP_URL_LIMIT, "gists", order_by=["-created_at"])
         for g in gists:
             urlset.append(url_element(
                 f"{base_url}/gists/{g.get('slug') or g['uid']}",
@@ -211,11 +326,28 @@ def make_sitemap(base_url):
                 priority="0.6"
             ))
 
+    if "news" in db.tables:
+        articles = _collect(get_table("news"), SITEMAP_URL_LIMIT, "news", status="published", order_by=["-synced_at"])
+        for a in articles:
+            urlset.append(url_element(
+                f"{base_url}/news/{a.get('slug') or a['uid']}",
+                lastmod=a.get("synced_at", "") or a.get("created_at", ""),
+                changefreq="weekly",
+                priority="0.7"
+            ))
+
     if "users" in db.tables:
-        users = list(get_table("users").find(order_by=["-created_at"], _limit=200))
+        post_counts = {}
+        if "posts" in db.tables:
+            for row in db.query("SELECT user_uid, COUNT(*) AS c FROM posts GROUP BY user_uid"):
+                post_counts[row["user_uid"]] = row["c"]
+        users = _collect(get_table("users"), SITEMAP_URL_LIMIT, "users", order_by=["-created_at"])
         for u in users:
+            if post_counts.get(u["uid"], 0) < 2:
+                continue
             urlset.append(url_element(
                 f"{base_url}/profile/{u['username']}",
+                lastmod=u.get("created_at", ""),
                 changefreq="weekly",
                 priority="0.4"
             ))

devplacepy/services/base.py

@@ -1,93 +1,391 @@
 import asyncio
+import json
 import logging
 from abc import ABC, abstractmethod
 from collections import deque
-from datetime import datetime
+from datetime import datetime, timedelta, timezone
+
+from devplacepy.database import db, get_table, get_setting, get_int_setting
 
 logger = logging.getLogger(__name__)
 
 
+class ConfigField:
+    def __init__(self, key, label, type="str", default="", help="",
+                 options=None, secret=False, minimum=None, maximum=None, group="General"):
+        self.key = key
+        self.label = label
+        self.type = type
+        self.default = default
+        self.help = help
+        self.options = options or []
+        self.secret = secret
+        self.minimum = minimum
+        self.maximum = maximum
+        self.group = group
+
+    def coerce(self, raw):
+        if self.type == "int":
+            try:
+                value = int(str(raw).strip())
+            except (TypeError, ValueError):
+                raise ValueError(f"{self.label} must be a whole number")
+            if self.minimum is not None and value < self.minimum:
+                raise ValueError(f"{self.label} must be at least {self.minimum}")
+            if self.maximum is not None and value > self.maximum:
+                raise ValueError(f"{self.label} must be at most {self.maximum}")
+            return value
+        if self.type == "float":
+            try:
+                value = float(str(raw).strip())
+            except (TypeError, ValueError):
+                raise ValueError(f"{self.label} must be a number")
+            if self.minimum is not None and value < self.minimum:
+                raise ValueError(f"{self.label} must be at least {self.minimum}")
+            if self.maximum is not None and value > self.maximum:
+                raise ValueError(f"{self.label} must be at most {self.maximum}")
+            return value
+        if self.type == "bool":
+            text = str(raw).strip()
+            if text in ("0", "1"):
+                return text == "1"
+            raise ValueError(f"{self.label} must be enabled or disabled")
+        if self.type == "select":
+            allowed = [option["value"] for option in self.options]
+            if raw not in allowed:
+                raise ValueError(f"{self.label} has an invalid selection")
+            return raw
+        if self.type == "url":
+            text = str(raw).strip()
+            if text and not (text.startswith("http://") or text.startswith("https://")):
+                raise ValueError(f"{self.label} must be a http(s) URL")
+            return text
+        return str(raw)
+
+    def to_storage(self, value):
+        if isinstance(value, bool):
+            return "1" if value else "0"
+        return str(value)
+
+    def read(self):
+        raw = get_setting(self.key, "")
+        if raw == "":
+            return self.default
+        try:
+            return self.coerce(raw)
+        except ValueError:
+            return self.default
+
+    def display_value(self):
+        if self.secret:
+            return ""
+        raw = get_setting(self.key, "")
+        if raw == "":
+            return self.to_storage(self.default)
+        return raw
+
+    def spec(self):
+        return {
+            "key": self.key,
+            "label": self.label,
+            "type": self.type,
+            "help": self.help,
+            "options": self.options,
+            "secret": self.secret,
+            "minimum": self.minimum,
+            "maximum": self.maximum,
+            "value": self.display_value(),
+            "is_set": bool(get_setting(self.key, "")) if self.secret else None,
+            "group": self.group,
+        }
+
+
 class BaseService(ABC):
+    config_fields = []
+    interval_key = None
+    min_interval = 1
+    default_enabled = True
+    title = ""
+    description = ""
+    DEFAULT_LOG_SIZE = 20
+    TICK_SECONDS = 1
+    PERSIST_SECONDS = 3
+    STALE_SECONDS = 15
+
     def __init__(self, name: str, interval_seconds: int = 3600):
         self.name = name
         self.interval_seconds = interval_seconds
-        self.log_buffer = deque(maxlen=20)
+        self.interval_key = self.interval_key or f"service_{name}_interval"
+        self.enabled_key = f"service_{name}_enabled"
+        self.command_key = f"service_{name}_command"
+        self.log_size_key = f"service_{name}_log_size"
+        self.log_buffer = deque(maxlen=self.DEFAULT_LOG_SIZE)
         self._task = None
         self._running = False
+        self._shutdown = False
+        self._started_at = None
         self._last_run = None
         self._next_run = None
-        self._started_at = None
+        self._next_due = None
+        self._run_pending = False
+        self._last_command = None
+        self._last_persist = None
+        self.enabled_field = ConfigField(
+            self.enabled_key, "Enabled", type="bool", default=self.default_enabled,
+            help="When enabled the service runs on its interval and starts on boot.",
+            group="General")
+        self.interval_field = ConfigField(
+            self.interval_key, "Run interval (seconds)", type="int",
+            default=interval_seconds, minimum=self.min_interval,
+            help=f"Delay between runs. Minimum {self.min_interval} seconds.",
+            group="General")
+        self.log_size_field = ConfigField(
+            self.log_size_key, "Log buffer size", type="int",
+            default=self.DEFAULT_LOG_SIZE, minimum=1, maximum=200,
+            help="Number of recent log lines to keep.",
+            group="Advanced")
+
+    def all_fields(self) -> list:
+        return [self.enabled_field, self.interval_field, *self.config_fields, self.log_size_field]
+
+    def _grouped_fields(self) -> list:
+        groups: dict = {}
+        order: list = []
+        for field in self.all_fields():
+            spec = field.spec()
+            group = spec["group"]
+            if group not in groups:
+                groups[group] = []
+                order.append(group)
+            groups[group].append(spec)
+        return [{"name": name, "fields": groups[name]} for name in order]
 
     @property
     def status(self) -> str:
         return "running" if self._running else "stopped"
 
-    @property
-    def last_run(self) -> str | None:
-        return self._last_run
+    def is_enabled(self) -> bool:
+        return get_setting(self.enabled_key, "1" if self.default_enabled else "0") == "1"
 
-    @property
-    def next_run(self) -> str | None:
-        return self._next_run
+    def current_interval(self) -> int:
+        return max(self.min_interval, get_int_setting(self.interval_key, self.interval_seconds))
 
-    @property
-    def uptime(self) -> str | None:
-        if self._started_at is None:
-            return None
-        delta = datetime.utcnow() - self._started_at
-        return str(delta).split(".")[0]
+    def get_config(self) -> dict:
+        return {field.key: field.read() for field in self.all_fields()}
 
     def log(self, message: str) -> None:
-        stamp = datetime.utcnow().strftime("%H:%M:%S")
-        entry = f"[{stamp}] {message}"
-        self.log_buffer.append(entry)
+        stamp = datetime.now(timezone.utc).strftime("%H:%M:%S")
+        self.log_buffer.append(f"[{stamp}] {message}")
         logger.info(f"[{self.name}] {message}")
 
     @abstractmethod
     async def run_once(self) -> None:
         pass
 
-    async def _run_loop(self) -> None:
-        self._running = True
-        self._started_at = datetime.utcnow()
-        self.log(f"Service started (interval={self.interval_seconds}s)")
-        while self._running:
-            try:
-                self._last_run = datetime.utcnow().isoformat()
-                self._next_run = None
-                await self.run_once()
-            except asyncio.CancelledError:
-                self.log("Service cancelled")
-                break
-            except Exception as e:
-                self.log(f"Error in run_once: {e}")
-            if not self._running:
-                break
-            self._next_run = datetime.utcnow().isoformat()
-            self.log(f"Sleeping for {self.interval_seconds}s")
-            try:
-                await asyncio.sleep(self.interval_seconds)
-            except asyncio.CancelledError:
-                self.log("Service cancelled during sleep")
-                break
-        self._running = False
-        self.log("Service stopped")
+    async def on_enable(self) -> None:
+        pass
 
-    async def start(self) -> None:
-        if self._running:
-            self.log("Already running")
+    async def on_disable(self) -> None:
+        pass
+
+    def collect_metrics(self) -> dict:
+        return {}
+
+    def _safe_metrics(self) -> dict:
+        try:
+            return self.collect_metrics()
+        except Exception as e:
+            logger.warning(f"Could not collect metrics for {self.name}: {e}")
+            return {}
+
+    def start_supervisor(self) -> None:
+        if self._task is not None:
             return
-        self._running = True
+        self._shutdown = False
         self._task = asyncio.create_task(self._run_loop())
 
-    async def stop(self) -> None:
-        if not self._running:
-            return
-        self._running = False
-        if self._task is not None:
-            self._task.cancel()
+    def request_shutdown(self) -> None:
+        self._shutdown = True
+
+    async def _run_loop(self) -> None:
+        self._last_command = get_setting(self.command_key, "")
+        self.log("Supervisor started")
+        self._persist_state(force=True)
+        while not self._shutdown:
             try:
-                await asyncio.wait_for(self._task, timeout=10.0)
-            except (asyncio.CancelledError, asyncio.TimeoutError):
-                pass
-            self._task = None
+                await self._tick()
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                self.log(f"Loop error: {e}")
+            try:
+                await asyncio.sleep(self.TICK_SECONDS)
+            except asyncio.CancelledError:
+                break
+        if self._started_at is not None:
+            await self._safe_disable()
+        self._running = False
+        self._started_at = None
+        self._persist_state(force=True)
+        self.log("Supervisor stopped")
+
+    async def _tick(self) -> None:
+        self._sync_log_size()
+        self._handle_commands()
+        if self.is_enabled():
+            if self._started_at is None:
+                self._started_at = datetime.now(timezone.utc)
+                self._running = True
+                self.log("Service enabled")
+                await self.on_enable()
+            due = self._next_due is None or datetime.now(timezone.utc) >= self._next_due
+            if self._run_pending or due:
+                self._run_pending = False
+                await self._execute_run()
+        elif self._started_at is not None:
+            await self._safe_disable()
+            self._started_at = None
+            self._running = False
+            self._next_due = None
+            self._next_run = None
+            self.log("Service disabled")
+        self._persist_state()
+
+    async def _safe_disable(self) -> None:
+        try:
+            await self.on_disable()
+        except Exception as e:
+            self.log(f"on_disable error: {e}")
+
+    async def _execute_run(self) -> None:
+        self.interval_seconds = self.current_interval()
+        self._last_run = datetime.now(timezone.utc).isoformat()
+        self._next_run = None
+        self._persist_state(force=True)
+        try:
+            await self.run_once()
+        except asyncio.CancelledError:
+            raise
+        except Exception as e:
+            self.log(f"Error in run_once: {e}")
+        self.interval_seconds = self.current_interval()
+        self._next_due = datetime.now(timezone.utc) + timedelta(seconds=self.interval_seconds)
+        self._next_run = self._next_due.isoformat()
+        self.log(f"Next run in {self.interval_seconds}s")
+        self._persist_state(force=True)
+
+    def _handle_commands(self) -> None:
+        raw = get_setting(self.command_key, "")
+        if not raw or raw == self._last_command:
+            return
+        self._last_command = raw
+        verb = raw.split(":", 1)[0]
+        if verb == "run":
+            self._run_pending = True
+            self.log("Run requested")
+        elif verb == "clear":
+            self.log_buffer.clear()
+            self.log("Logs cleared")
+
+    def _sync_log_size(self) -> None:
+        size = max(1, get_int_setting(self.log_size_key, self.DEFAULT_LOG_SIZE))
+        if size != self.log_buffer.maxlen:
+            self.log_buffer = deque(self.log_buffer, maxlen=size)
+
+    def _persist_state(self, force: bool = False) -> None:
+        now = datetime.now(timezone.utc)
+        if not force and self._last_persist is not None:
+            if (now - self._last_persist).total_seconds() < self.PERSIST_SECONDS:
+                return
+        self._last_persist = now
+        record = {
+            "name": self.name,
+            "status": self.status,
+            "last_run": self._last_run or "",
+            "next_run": self._next_run or "",
+            "started_at": self._started_at.isoformat() if self._started_at else "",
+            "heartbeat": now.isoformat(),
+            "logs": json.dumps(list(self.log_buffer)),
+            "metrics": json.dumps(self._safe_metrics()),
+            "updated_at": now.isoformat(),
+        }
+        try:
+            table = get_table("service_state")
+            existing = table.find_one(name=self.name)
+            if existing:
+                table.update({**record, "id": existing["id"]}, ["id"])
+            else:
+                table.insert(record)
+        except Exception as e:
+            logger.warning(f"Could not persist state for {self.name}: {e}")
+
+    def _read_state(self) -> dict:
+        if "service_state" not in db.tables:
+            return {}
+        row = db["service_state"].find_one(name=self.name)
+        return row or {}
+
+    def _display_status(self, enabled: bool, state: dict) -> str:
+        if not enabled:
+            return "stopped"
+        heartbeat = state.get("heartbeat")
+        if not heartbeat:
+            return "stalled"
+        try:
+            beat = datetime.fromisoformat(heartbeat)
+        except ValueError:
+            return "stalled"
+        if (datetime.now(timezone.utc) - beat).total_seconds() > self.STALE_SECONDS:
+            return "stalled"
+        return "running"
+
+    def _uptime(self, state: dict, status: str) -> str | None:
+        if status != "running":
+            return None
+        started = state.get("started_at")
+        if not started:
+            return None
+        try:
+            since = datetime.fromisoformat(started)
+        except ValueError:
+            return None
+        return str(datetime.now(timezone.utc) - since).split(".")[0]
+
+    def _logs(self, state: dict) -> list:
+        raw = state.get("logs")
+        if not raw:
+            return []
+        try:
+            return json.loads(raw)
+        except (ValueError, TypeError):
+            return []
+
+    def _metrics(self, state: dict) -> dict:
+        raw = state.get("metrics")
+        if not raw:
+            return {}
+        try:
+            return json.loads(raw)
+        except (ValueError, TypeError):
+            return {}
+
+    def describe(self) -> dict:
+        state = self._read_state()
+        enabled = self.is_enabled()
+        status = self._display_status(enabled, state)
+        return {
+            "name": self.name,
+            "title": self.title or self.name.capitalize(),
+            "description": self.description,
+            "enabled": enabled,
+            "status": status,
+            "interval_seconds": self.current_interval(),
+            "last_run": state.get("last_run") or None,
+            "next_run": state.get("next_run") or None,
+            "heartbeat": state.get("heartbeat") or None,
+            "uptime": self._uptime(state, status),
+            "log_buffer": self._logs(state),
+            "metrics": self._metrics(state),
+            "fields": [field.spec() for field in self.all_fields()],
+            "field_groups": self._grouped_fields(),
+        }

devplacepy/services/bot/__init__.py

@@ -0,0 +1,3 @@
+from devplacepy.services.bot.service import BotsService
+
+__all__ = ["BotsService"]

devplacepy/services/bot/browser.py

@@ -0,0 +1,162 @@
+import asyncio
+import logging
+import random
+from typing import Optional
+
+from playwright.async_api import Page, Playwright, async_playwright
+
+logger = logging.getLogger(__name__)
+
+
+class BotBrowser:
+    def __init__(self, headless: bool = True):
+        self._headless = headless
+        self._playwright: Optional[Playwright] = None
+        self._browser = None
+        self._context = None
+        self._page: Optional[Page] = None
+
+    async def launch(self) -> None:
+        self._playwright = await async_playwright().start()
+        self._browser = await self._playwright.chromium.launch(
+            headless=self._headless,
+            args=[
+                "--disable-blink-features=AutomationControlled", "--no-first-run",
+                "--disable-dev-shm-usage", "--window-size=1280,900",
+            ],
+        )
+        self._context = await self._browser.new_context(
+            user_agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36",
+            viewport={"width": 1280, "height": 900},
+            locale="en-US",
+        )
+        self._page = await self._context.new_page()
+
+    async def close(self) -> None:
+        for label, resource, shutdown in (
+            ("context", self._context, lambda r: r.close()),
+            ("browser", self._browser, lambda r: r.close()),
+            ("playwright", self._playwright, lambda r: r.stop()),
+        ):
+            if resource is None:
+                continue
+            try:
+                await shutdown(resource)
+            except Exception as e:
+                logger.debug("Browser %s close error: %s", label, e)
+        self._page = None
+        self._context = None
+        self._browser = None
+        self._playwright = None
+
+    @property
+    def page(self) -> Page:
+        return self._page
+
+    async def goto(self, url: str) -> None:
+        try:
+            await self._page.goto(url, wait_until="domcontentloaded", timeout=15000)
+        except Exception as e:
+            logger.debug("goto (domcontentloaded) failed for %s: %s", url[:80], e)
+            try:
+                await self._page.goto(url, timeout=20000)
+            except Exception as e2:
+                logger.warning("goto failed for %s: %s", url[:80], e2)
+        await self._idle(0.5, 1.5)
+
+    async def _idle(self, lo: float = 0.5, hi: float = 2.0) -> None:
+        await asyncio.sleep(random.uniform(lo, hi))
+
+    async def scroll(self, amount: Optional[int] = None) -> None:
+        if amount is None:
+            amount = random.randint(300, 800)
+        try:
+            await self._page.evaluate(f"window.scrollBy(0, {amount})")
+        except Exception as e:
+            logger.debug("scroll failed: %s", e)
+        await self._idle(0.2, 0.6)
+
+    async def scroll_to_bottom(self) -> None:
+        try:
+            await self._page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+        except Exception as e:
+            logger.debug("scroll_to_bottom failed: %s", e)
+
+    async def fill(self, sel: str, val: str) -> bool:
+        try:
+            el = self._page.locator(sel).first
+            if await el.count() == 0:
+                logger.debug("fill: selector '%s' not found", sel)
+                return False
+            await el.click(timeout=3000)
+            await self._idle(0.05, 0.15)
+            await el.fill("")
+            for ch in val:
+                await self._page.keyboard.type(ch, delay=random.randint(20, 60))
+                if ch == " ":
+                    await asyncio.sleep(random.uniform(0.02, 0.08))
+            return True
+        except Exception as e:
+            logger.debug("fill failed for '%s': %s", sel, e)
+            return False
+
+    async def click(self, sel: str) -> bool:
+        try:
+            el = self._page.locator(sel).first
+            if await el.count() == 0:
+                logger.debug("click: selector '%s' not found", sel)
+                return False
+            box = await el.bounding_box()
+            if box:
+                tx = box["x"] + box["width"] / 2 + random.uniform(-3, 3)
+                ty = box["y"] + box["height"] / 2 + random.uniform(-3, 3)
+                await self._page.mouse.move(tx, ty, steps=random.randint(8, 18))
+                await self._idle(0.02, 0.06)
+            await el.click(timeout=5000)
+            return True
+        except Exception as e:
+            logger.debug("click failed for '%s': %s", sel, e)
+            return False
+
+    async def text(self, sel: str) -> str:
+        try:
+            return (await self._page.locator(sel).first.inner_text()) or ""
+        except Exception as e:
+            logger.debug("text failed for '%s': %s", sel, e)
+            return ""
+
+    async def attr(self, sel: str, attr: str) -> str:
+        try:
+            val = await self._page.locator(sel).first.get_attribute(attr)
+            return val or ""
+        except Exception as e:
+            logger.debug("attr '%s' on '%s' failed: %s", attr, sel, e)
+            return ""
+
+    async def html(self) -> str:
+        try:
+            return await self._page.content()
+        except Exception as e:
+            logger.debug("html() failed: %s", e)
+            return ""
+
+    async def url(self) -> str:
+        try:
+            return self._page.url
+        except Exception as e:
+            logger.debug("url() failed: %s", e)
+            return ""
+
+    async def title(self) -> str:
+        try:
+            return await self._page.title()
+        except Exception as e:
+            logger.debug("title() failed: %s", e)
+            return ""
+
+    async def reload(self) -> None:
+        try:
+            await self._page.reload(timeout=15000)
+            await self._idle()
+        except Exception as e:
+            logger.debug("reload failed: %s", e)

devplacepy/services/bot/config.py

@@ -0,0 +1,73 @@
+from pathlib import Path
+
+from devplacepy.config import INTERNAL_GATEWAY_URL, INTERNAL_MODEL
+
+BASE_URL_DEFAULT = "https://pravda.education"
+API_URL_DEFAULT = INTERNAL_GATEWAY_URL
+NEWS_API_DEFAULT = "https://news.app.molodetz.nl/api"
+MODEL_DEFAULT = INTERNAL_MODEL
+INPUT_COST_PER_1M_DEFAULT = 0.27
+OUTPUT_COST_PER_1M_DEFAULT = 1.10
+
+COST_WINDOW_SECONDS = 600
+COST_WARMUP_SECONDS = 120
+
+STATE_DIR = Path.home() / ".devplace_bots"
+ARTICLE_REGISTRY_PATH = Path.home() / ".dpbot_article_registry.json"
+
+PROJECT_STATUSES = ["In Development", "Released"]
+
+PERSONAS = [
+    "enthusiastic_junior",
+    "grumpy_senior",
+    "hobbyist_maker",
+    "academic_type",
+    "minimalist",
+    "storyteller",
+    "rebel",
+    "mentor",
+]
+
+REACT_RATES = {
+    "enthusiastic_junior": 0.50,
+    "hobbyist_maker": 0.40,
+    "storyteller": 0.30,
+    "mentor": 0.30,
+    "rebel": 0.25,
+    "academic_type": 0.20,
+    "minimalist": 0.12,
+    "grumpy_senior": 0.12,
+}
+REACT_RATE_DEFAULT = 0.20
+
+CATEGORIES = ["devlog", "showcase", "question", "rant", "fun", "random"]
+
+GIST_LANGUAGES = [
+    "python", "javascript", "typescript", "bash", "go", "rust",
+    "sql", "html", "css", "java", "c", "cpp", "ruby", "php", "lua",
+]
+
+FEED_TOPICS = ["devlog", "showcase", "question", "rant", "fun"]
+PROJECT_TYPES = ["game", "game_asset", "software", "mobile_app", "website"]
+
+SEARCH_TERMS = {
+    "enthusiastic_junior": ["vibe coding", "first app", "react", "python", "ai tools"],
+    "grumpy_senior": ["rust", "perl", "legacy", "tech debt", "kubernetes"],
+    "hobbyist_maker": ["arduino", "raspberry pi", "side project", "3d printing", "game"],
+    "academic_type": ["algorithms", "type theory", "distributed systems", "compilers"],
+    "minimalist": ["cli", "vim", "golang", "suckless"],
+    "storyteller": ["postmortem", "war story", "migration", "outage"],
+    "rebel": ["hot take", "overrated", "monorepo", "microservices"],
+    "mentor": ["best practices", "code review", "testing", "career"],
+}
+
+MIN_COMMENT_LEN = 40
+MIN_POST_LEN = 120
+RESTATEMENT_OVERLAP_THRESHOLD = 0.6
+
+GENERIC_COMMENT_PHRASES = [
+    "great point", "good point", "i agree", "totally agree", "well said",
+    "nice post", "great post", "thanks for sharing", "interesting read",
+    "this is interesting", "so true", "couldn't agree more", "spot on",
+    "great article", "love this", "this is great", "good read",
+]

devplacepy/services/bot/llm.py

@@ -0,0 +1,303 @@
+import json
+import logging
+import random
+import re
+import time
+import urllib.error
+import urllib.request
+
+from devplacepy.services.bot.config import CATEGORIES, GIST_LANGUAGES
+
+logger = logging.getLogger(__name__)
+
+
+class LLMClient:
+    def __init__(self, api_key: str, api_url: str, model: str,
+                 input_cost_per_1m: float, output_cost_per_1m: float):
+        if not api_key:
+            raise RuntimeError("LLM API key not set")
+        self.api_key = api_key
+        self.api_url = api_url
+        self.model = model
+        self.input_cost_per_1m = input_cost_per_1m
+        self.output_cost_per_1m = output_cost_per_1m
+        self.total_cost = 0.0
+        self.total_calls = 0
+        self.total_in_tokens = 0
+        self.total_out_tokens = 0
+
+    def _call(self, system: str, prompt: str, temperature: float = 0.7) -> str:
+        for attempt in range(3):
+            try:
+                payload = {
+                    "model": self.model,
+                    "messages": [{"role": "system", "content": system}, {"role": "user", "content": prompt}],
+                    "temperature": temperature,
+                }
+                logger.info("LLM >>> model=%s system=%s prompt=%s",
+                            self.model, json.dumps(system[:500]), json.dumps(prompt[:500]))
+                data = json.dumps(payload).encode()
+                req = urllib.request.Request(
+                    self.api_url, data=data,
+                    headers={"Content-Type": "application/json", "Authorization": f"Bearer {self.api_key}"},
+                )
+                with urllib.request.urlopen(req, timeout=30) as resp:
+                    raw = resp.read()
+                    logger.info("LLM <<< %s", raw[:2000].decode(errors="replace"))
+                    result = json.loads(raw)
+                    usage = result.get("usage", {})
+                    in_tokens = usage.get("prompt_tokens", 0)
+                    out_tokens = usage.get("completion_tokens", 0)
+                    cost = (in_tokens * self.input_cost_per_1m / 1_000_000) + \
+                           (out_tokens * self.output_cost_per_1m / 1_000_000)
+                    self.total_cost += cost
+                    self.total_calls += 1
+                    self.total_in_tokens += in_tokens
+                    self.total_out_tokens += out_tokens
+                    return self.clean(result["choices"][0]["message"]["content"])
+            except (urllib.error.HTTPError, urllib.error.URLError, json.JSONDecodeError, KeyError) as e:
+                logger.warning("LLM call attempt %d/3 failed: %s", attempt + 1, e)
+                if attempt == 2:
+                    raise
+                time.sleep(2 ** attempt)
+        return ""
+
+    @staticmethod
+    def clean(text: str, preserve_md: bool = False) -> str:
+        if not preserve_md:
+            text = re.sub(r"\*\*|__|\*|_", "", text)
+        text = text.replace("-", "-").replace("–", "-")
+        text = text.replace("‘", "'").replace("’", "'")
+        text = text.replace("“", '"').replace("”", '"')
+        text = re.sub(r"\s{2,}", " ", text)
+        return text.strip()
+
+    @staticmethod
+    def strip_md(text: str) -> str:
+        return LLMClient.clean(text)[:200]
+
+    @staticmethod
+    def strip_code_fences(text: str) -> str:
+        text = text.strip()
+        if text.startswith("```"):
+            lines = text.split("\n")
+            if lines and lines[0].lstrip().startswith("```"):
+                lines = lines[1:]
+            if lines and lines[-1].strip().startswith("```"):
+                lines = lines[:-1]
+            text = "\n".join(lines)
+        return text.strip()
+
+    def generate_post(self, title: str, desc: str, persona: str = "", category: str = "") -> str:
+        persona_extras = {
+            "enthusiastic_junior": "Be excited. Use **bold** for emphasis. Short excited sentences. End with a question sometimes.",
+            "grumpy_senior": "Be slightly cynical but helpful. Short blunt sentences. No fluff. Call out bad practices.",
+            "hobbyist_maker": "Be casual and friendly. Mention side projects and tinkering. Use *italics* for tools/libraries.",
+            "academic_type": "Be precise and structured. Use *italics* for terminology. Well thought out arguments.",
+            "storyteller": "Use **bold** for key points. Write anecdotal, narrative style. Longer flowing sentences.",
+            "rebel": "Be informal. Skip punctuation sometimes. Use slang. Type like you're in a hurry.",
+            "mentor": "Be helpful and explanatory. Use **bold** for key takeaways. Include practical advice.",
+            "minimalist": "One paragraph. Short blunt declarative sentences. No markdown. No fluff.",
+        }
+        category_extras = {
+            "devlog": "Write it as a personal devlog entry - share your learning process and insights about this topic.",
+            "showcase": "Write it as a showcase - express genuine excitement about what makes this impressive.",
+            "question": "Write it as a discussion starter - ask thoughtful questions, invite others to share perspectives.",
+            "rant": "Write it as an opinionated rant - strong viewpoint, passionate criticism, but keep it substantive.",
+            "fun": "Write it lighthearted and playful, but stay tied to the actual tech topic. Humor about the technology itself, not off-topic jokes or lyrics.",
+            "random": "Be natural and conversational - share your thoughts like any casual discussion.",
+        }
+        persona_extra = f" {persona_extras.get(persona, 'Be casual. No markdown.')}" if persona else " Be casual. No markdown."
+        category_extra = f" {category_extras.get(category, '')}" if category else ""
+        preserve = persona in ("enthusiastic_junior", "hobbyist_maker", "academic_type", "storyteller", "mentor")
+        text = self._call(
+            f"You are a dev writing a social media post reacting to tech news. Write 2-4 short paragraphs."
+            f"{persona_extra}{category_extra} Do not summarize the article; assume the reader already saw it. "
+            f"Add your own value: a concrete opinion, an implication, a personal experience, or a pointed question. "
+            f"Reference a specific detail rather than restating the headline. No em dashes. Under 300 words.",
+            f"News: {title}\n\n{desc[:800]}",
+        )
+        return self.clean(text, preserve_md=preserve)
+
+    def select_category(self, title: str, desc: str) -> str:
+        text = self._call(
+            "You are a classifier. Given tech news, choose the single best post category. Reply with ONLY the category name.",
+            f"Categories:\n- devlog: learning journey, building something, personal dev experience\n- showcase: impressive release, new tool, achievement worth highlighting\n- question: uncertainty, asking for advice, curiosity about implications\n- rant: frustration, bad practices, criticism of industry trends\n- fun: amusing, surprising, entertaining but not deeply serious\n- random: anything else that doesn't clearly fit above\n\nNews: {title}\n\n{desc[:600]}",
+            temperature=0.2,
+        )
+        for c in CATEGORIES:
+            if c in text.lower():
+                return c
+        return "random"
+
+    def select_reaction(self, content_snippet: str, persona: str = "") -> str:
+        from devplacepy.constants import REACTION_EMOJI
+
+        flavor = {
+            "enthusiastic_junior": "You react readily and warmly.",
+            "grumpy_senior": "You react rarely, only to genuinely notable content.",
+            "hobbyist_maker": "You react to anything hands-on, clever, or fun.",
+            "academic_type": "You react only to substantive, rigorous content.",
+            "minimalist": "You react very sparingly.",
+            "storyteller": "You react to anything with a human angle.",
+            "rebel": "You react to bold or contrarian takes.",
+            "mentor": "You react supportively to effort and learning.",
+        }.get(persona, "")
+        options = " ".join(REACTION_EMOJI)
+        verdict = self._call(
+            "You are a developer browsing a community feed. Decide whether a post deserves an "
+            "emoji reaction and which single emoji best fits its content and sentiment. "
+            f"{flavor} Reply with EXACTLY one of these emojis: {options} "
+            "or the word NONE if the content is mundane or would not move you to react. "
+            "Output only the emoji or NONE, nothing else.",
+            f"Post:\n{content_snippet[:1200]}",
+            temperature=0.4,
+        )
+        text = verdict or ""
+        for emoji in REACTION_EMOJI:
+            if emoji in text or emoji.replace("\ufe0f", "") in text:
+                return emoji
+        return ""
+
+    def generate_comment(self, post_snippet: str, persona: str = "",
+                         mention_target: str = "", parent_context: str = "") -> str:
+        extras = {
+            "enthusiastic_junior": "Be excited. Use **bold** for agreement. Short replies.",
+            "grumpy_senior": "Be blunt and short. No markdown. One sarcastic remark or actual advice.",
+            "rebel": "Super casual. Skip caps sometimes. Short.",
+            "mentor": "Be helpful. Use **bold** for key point.",
+            "storyteller": "Share a quick related story. Use *italics* for emphasis.",
+            "minimalist": "Shortest possible reply. One sentence max.",
+        }
+        extra = f" {extras.get(persona, 'Be casual. No markdown.')}" if persona else " Be casual. No markdown."
+        preserve = persona in ("enthusiastic_junior", "mentor", "storyteller")
+        mention_rule = ""
+        if mention_target:
+            mention_rule = (
+                f" Address @{mention_target} naturally in the first sentence "
+                f"(weave the @{mention_target} mention inline, do not append it at the end)."
+            )
+        ctx = ""
+        if parent_context:
+            ctx = f"\n\nReplying to this comment:\n{parent_context[:800]}"
+        system = (
+            f"You are a dev replying to a post. Write 1-3 short sentences.{extra}{mention_rule} "
+            "No em dashes. Reference a specific detail from the post. "
+            "Do not open with or rely on generic filler like 'great point', 'I agree', 'nice', "
+            "'thanks for sharing', or 'interesting'. Add something real: a concrete experience, "
+            "a caveat, a counterexample, respectful disagreement, or a pointed question. "
+            "Never just paraphrase or agree blandly."
+        )
+        reply = self.clean(
+            self._call(system, f"Post:\n{post_snippet[:1500]}{ctx}").strip(),
+            preserve_md=preserve,
+        )
+        return reply[:2000]
+
+    @staticmethod
+    def _overlap_ratio(text: str, context: str) -> float:
+        def tokens(s: str) -> list[str]:
+            return [w for w in re.findall(r"[a-z0-9]+", s.lower()) if len(w) > 3]
+
+        candidate = tokens(text)
+        source = set(tokens(context))
+        if not candidate or not source:
+            return 0.0
+        hits = sum(1 for w in candidate if w in source)
+        return hits / len(candidate)
+
+    def quality_check(self, kind: str, text: str, context: str = "") -> tuple[bool, str]:
+        from devplacepy.services.bot.config import (
+            MIN_COMMENT_LEN, MIN_POST_LEN, RESTATEMENT_OVERLAP_THRESHOLD, GENERIC_COMMENT_PHRASES,
+        )
+        stripped = self.clean(text or "").strip()
+        min_len = MIN_COMMENT_LEN if kind == "comment" else MIN_POST_LEN
+        if len(stripped) < min_len:
+            return False, f"too short ({len(stripped)} < {min_len})"
+        lowered = stripped.lower()
+        if kind == "comment":
+            for phrase in GENERIC_COMMENT_PHRASES:
+                if phrase in lowered:
+                    return False, f"generic phrase '{phrase}'"
+        if context and self._overlap_ratio(stripped, context) > RESTATEMENT_OVERLAP_THRESHOLD:
+            return False, "restates the source"
+        verdict = self._call(
+            "You are a strict content quality reviewer for a developer community. "
+            "Reject text that is generic, low-effort, pure filler, or merely summarizes its "
+            "source without adding an opinion, experience, or insight. "
+            "Reply with exactly PASS or 'FAIL: <short reason>'.",
+            f"Type: {kind}\nText:\n{stripped[:1200]}",
+            temperature=0.0,
+        )
+        if verdict.strip().upper().startswith("PASS"):
+            return True, "ok"
+        return False, verdict.strip()[:120] or "judge rejected"
+
+    def generate_bug(self, topic: str) -> tuple[str, str]:
+        text = self._call(
+            "Write a bug report. One line title. Then 2-3 sentences describing what happened and what should have happened. Like a real dev reporting a bug. No em dashes.",
+            f"Bug: {topic}",
+        )
+        lines = text.strip().split("\n")
+        title = lines[0].strip().lstrip("#").strip()[:120] or f"Bug report: {topic}"
+        desc = " ".join(line.lstrip("-* ").strip() for line in lines[1:] if line.strip())
+        return title, desc[:2000] or text[:500]
+
+    def generate_bio(self) -> str:
+        return self._call(
+            "Write a short bio for a developer profile. 1-2 sentences. What they work on, what they like. Sound human. No em dashes.",
+            "Bio:",
+        )
+
+    def generate_profile_fields(self, handle: str) -> tuple[str, str, str]:
+        location = self.clean(self._call(
+            "Name one plausible city and country for a developer. Reply with ONLY 'City, Country'. No extra words.",
+            "Location:", temperature=0.9,
+        ))[:80]
+        slug = re.sub(r"[^a-z0-9_-]", "", handle.lower()) or "dev"
+        git_link = f"https://github.com/{slug}"
+        website = f"https://{slug}.dev"
+        return location, git_link, website
+
+    def generate_dm(self, persona: str = "", context: str = "") -> str:
+        extra = {
+            "enthusiastic_junior": "Be friendly and excited.",
+            "grumpy_senior": "Be blunt but not rude.",
+            "rebel": "Be super casual.",
+            "mentor": "Be warm and encouraging.",
+            "storyteller": "Open with a small hook.",
+            "minimalist": "One short sentence.",
+        }.get(persona, "Be casual and friendly.")
+        ctx = f"\n\nEarlier message:\n{context[:400]}" if context else ""
+        return self.clean(self._call(
+            f"Write one short, friendly direct message to another developer. 1-2 sentences. {extra} No em dashes.",
+            f"Write a DM to start or continue a chat.{ctx}",
+        ))[:500]
+
+    def generate_project_title(self) -> str:
+        return self._call("Come up with a project name. 2-4 words. A tool, game, or app idea. No em dashes.", "Name:", temperature=0.9)
+
+    def generate_project_desc(self, title: str) -> str:
+        return self._call(
+            "You are a developer. Write a compelling 2-3 sentence project description including tech stack and purpose.",
+            f"Project: {title}",
+        )
+
+    def generate_gist(self, persona: str = "") -> tuple[str, str, str, str]:
+        language = random.choice(GIST_LANGUAGES)
+        title = self.clean(self._call(
+            "Name a short, useful code snippet. 2-5 words. No quotes. No markdown.",
+            f"Language: {language}. Snippet name:",
+            temperature=0.9,
+        ))[:120]
+        description = self.clean(self._call(
+            "Write a one-sentence description of what a code snippet does, like a dev sharing something handy. No em dashes.",
+            f"Snippet: {title}\nLanguage: {language}",
+        ))[:400]
+        code = self.strip_code_fences(self._call(
+            f"Write a short, correct, self-contained {language} snippet of 5 to 20 lines for the description. "
+            "Output ONLY raw code. No markdown fences. No commentary.",
+            f"Title: {title}\nDescription: {description}\nLanguage: {language}",
+            temperature=0.4,
+        ))[:4000]
+        return title, description, language, code

devplacepy/services/bot/news_fetcher.py

@@ -0,0 +1,31 @@
+import json
+import logging
+import time
+import urllib.request
+
+logger = logging.getLogger(__name__)
+
+
+class NewsFetcher:
+    TTL_SECONDS = 1800
+
+    def __init__(self, news_api: str):
+        self.news_api = news_api
+        self.articles: list[dict] = []
+        self.fetched_at: float = 0.0
+
+    def fetch(self) -> list[dict]:
+        fresh = self.articles and (time.monotonic() - self.fetched_at) < self.TTL_SECONDS
+        if fresh:
+            return self.articles
+        try:
+            req = urllib.request.Request(self.news_api, headers={"User-Agent": "dpbot/1.0"})
+            with urllib.request.urlopen(req, timeout=10) as resp:
+                articles = json.loads(resp.read()).get("articles", [])
+            if articles:
+                self.articles = articles
+                self.fetched_at = time.monotonic()
+                logger.info("Fetched %d news articles", len(self.articles))
+        except Exception as e:
+            logger.warning("News fetch failed: %s", e)
+        return self.articles

devplacepy/services/bot/registry.py

@@ -0,0 +1,94 @@
+import fcntl
+import json
+import logging
+import os
+import random
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+
+class ArticleRegistry:
+    def __init__(self, path: Path):
+        self.path = path
+
+    def _lock(self) -> tuple:
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        try:
+            fd = os.open(str(self.path), os.O_RDWR | os.O_CREAT)
+            fcntl.flock(fd, fcntl.LOCK_EX)
+            data = {}
+            if os.path.getsize(str(self.path)) > 0:
+                chunks = []
+                while True:
+                    chunk = os.read(fd, 65536)
+                    if not chunk:
+                        break
+                    chunks.append(chunk)
+                raw = b"".join(chunks).decode(errors="replace")
+                if raw.strip():
+                    try:
+                        data = json.loads(raw)
+                    except json.JSONDecodeError:
+                        data = {}
+            if not isinstance(data, dict):
+                data = {}
+            now = datetime.now()
+            kept: dict = {}
+            for title, meta in data.items():
+                if not isinstance(meta, dict):
+                    continue
+                ts = meta.get("time", "")
+                try:
+                    age_days = (now - datetime.fromisoformat(ts)).total_seconds() / 86400
+                except (ValueError, TypeError):
+                    age_days = 0
+                if age_days < 7:
+                    kept[title] = meta
+            return fd, kept
+        except Exception as e:
+            logger.warning("Failed to lock article registry: %s", e)
+            return None, {}
+
+    @staticmethod
+    def _unlock(fd) -> None:
+        if fd is not None:
+            try:
+                fcntl.flock(fd, fcntl.LOCK_UN)
+                os.close(fd)
+            except Exception as e:
+                logger.debug("Failed to unlock article registry: %s", e)
+
+    def reserve(self, title: str, owner: str) -> bool:
+        fd, registry = self._lock()
+        if fd is None:
+            return True
+        try:
+            if title in registry:
+                return False
+            registry[title] = {"bot": owner, "time": datetime.now().isoformat()}
+            self.path.write_text(json.dumps(registry, indent=2, default=str))
+            return True
+        finally:
+            self._unlock(fd)
+
+    def reserve_unused(self, articles: list[dict], owner: str, clean) -> Optional[dict]:
+        if not articles:
+            return None
+        fd, registry = self._lock()
+        if fd is None:
+            return random.choice(articles) if articles else None
+        try:
+            used = set(registry.keys())
+            random.shuffle(articles)
+            for a in articles:
+                title = clean(a.get("title", "")[:200])
+                if title and title not in used:
+                    registry[title] = {"bot": owner, "time": datetime.now().isoformat()}
+                    self.path.write_text(json.dumps(registry, indent=2, default=str))
+                    return a
+            return None
+        finally:
+            self._unlock(fd)

devplacepy/services/bot/runtime.py

@@ -0,0 +1,18 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+from devplacepy.services.bot import config
+
+
+@dataclass
+class BotRuntimeConfig:
+    state_path: Path
+    base_url: str = config.BASE_URL_DEFAULT
+    api_url: str = config.API_URL_DEFAULT
+    news_api: str = config.NEWS_API_DEFAULT
+    model: str = config.MODEL_DEFAULT
+    api_key: str = ""
+    input_cost_per_1m: float = config.INPUT_COST_PER_1M_DEFAULT
+    output_cost_per_1m: float = config.OUTPUT_COST_PER_1M_DEFAULT
+    headless: bool = True
+    registry_path: Path = config.ARTICLE_REGISTRY_PATH

devplacepy/services/bot/service.py

@@ -0,0 +1,228 @@
+import asyncio
+import importlib
+import logging
+from collections import deque
+from datetime import datetime, timedelta, timezone
+
+from devplacepy.services.base import BaseService, ConfigField
+from devplacepy.services.bot import config
+
+logger = logging.getLogger(__name__)
+
+
+class BotsService(BaseService):
+    default_enabled = False
+    min_interval = 5
+    title = "Bots"
+    description = (
+        "Runs a fleet of Playwright-driven AI personas that browse a DevPlace "
+        "instance and post, comment, vote, and react like real users. Fleet size, "
+        "target site, and LLM settings are configurable, with live cost and usage "
+        "metrics per bot."
+    )
+    config_fields = [
+        ConfigField("bot_fleet_size", "Fleet size (bots)", type="int", default=1, minimum=0, maximum=20,
+                    help="Number of concurrent bot accounts to run.", group="Fleet"),
+        ConfigField("bot_headless", "Headless browser", type="bool", default=True,
+                    help="Run Chromium headless. Disable only for local debugging on a desktop.",
+                    group="Fleet"),
+        ConfigField("bot_max_actions", "Max actions per bot", type="int", default=0, minimum=0,
+                    help="Stop a bot after this many actions (0 = run indefinitely).", group="Fleet"),
+        ConfigField("bot_base_url", "Target site URL", type="url", default=config.BASE_URL_DEFAULT,
+                    help="DevPlace instance the bots browse and post to.", group="Target"),
+        ConfigField("bot_news_api", "News API URL", type="url", default=config.NEWS_API_DEFAULT,
+                    help="Source of articles the bots react to.", group="Target"),
+        ConfigField("bot_api_url", "LLM API URL", type="url", default=config.API_URL_DEFAULT,
+                    help="OpenAI-compatible chat-completions endpoint.", group="LLM"),
+        ConfigField("bot_model", "LLM model", type="str", default=config.MODEL_DEFAULT,
+                    help="Model name sent to the LLM API.", group="LLM"),
+        ConfigField("bot_api_key", "LLM API key", type="password", default="", secret=True,
+                    help="Defaults to the gateway's internal key (the bots use the local AI gateway).",
+                    group="LLM"),
+        ConfigField("bot_input_cost_per_1m", "Input cost per 1M tokens ($)", type="float",
+                    default=config.INPUT_COST_PER_1M_DEFAULT, minimum=0,
+                    help="Used to compute live spend per bot and across the fleet.", group="LLM"),
+        ConfigField("bot_output_cost_per_1m", "Output cost per 1M tokens ($)", type="float",
+                    default=config.OUTPUT_COST_PER_1M_DEFAULT, minimum=0,
+                    help="Used to compute live spend per bot and across the fleet.", group="LLM"),
+    ]
+
+    def __init__(self):
+        super().__init__(name="bots", interval_seconds=15)
+        self._fleet = {}
+        self._state_dir = config.STATE_DIR
+        self._cost_samples = deque()
+        self._cost_anchor = None
+
+    def _resolve_api_key(self, cfg: dict) -> str:
+        from devplacepy.database import internal_gateway_key
+        return cfg["bot_api_key"] or internal_gateway_key()
+
+    @staticmethod
+    def _ensure_importable() -> None:
+        importlib.import_module("devplacepy.services.bot.bot")
+
+    async def run_once(self) -> None:
+        cfg = self.get_config()
+        desired = cfg["bot_fleet_size"]
+        api_key = self._resolve_api_key(cfg)
+        if not api_key:
+            if self._fleet:
+                await self._stop_fleet()
+            self.log("No LLM API key available (bot_api_key / gateway internal key); fleet idle")
+            return
+        try:
+            self._ensure_importable()
+        except ImportError as e:
+            if self._fleet:
+                await self._stop_fleet()
+            self.log(f"Bot dependencies unavailable ({e}); install the 'bots' extra")
+            return
+
+        for slot in list(self._fleet):
+            task = self._fleet[slot]["task"]
+            if task.done():
+                self._report_exit(slot, task)
+                del self._fleet[slot]
+
+        while len(self._fleet) > desired:
+            await self._stop_slot(max(self._fleet))
+
+        for slot in range(desired):
+            if slot not in self._fleet:
+                self._launch_slot(slot, cfg, api_key)
+
+        self.log(f"Fleet: {len(self._fleet)}/{desired} bots active")
+
+    def _report_exit(self, slot: int, task) -> None:
+        if task.cancelled():
+            return
+        exc = task.exception()
+        if exc:
+            self.log(f"bot{slot} exited with error: {exc}; relaunching")
+        else:
+            self.log(f"bot{slot} finished; relaunching")
+
+    def _launch_slot(self, slot: int, cfg: dict, api_key: str) -> None:
+        from devplacepy.services.bot.bot import DevPlaceBot
+        from devplacepy.services.bot.runtime import BotRuntimeConfig
+        rc = BotRuntimeConfig(
+            state_path=self._state_dir / f"state_slot{slot}.json",
+            base_url=cfg["bot_base_url"],
+            api_url=cfg["bot_api_url"],
+            news_api=cfg["bot_news_api"],
+            model=cfg["bot_model"],
+            api_key=api_key,
+            input_cost_per_1m=cfg["bot_input_cost_per_1m"],
+            output_cost_per_1m=cfg["bot_output_cost_per_1m"],
+            headless=cfg["bot_headless"],
+            registry_path=config.ARTICLE_REGISTRY_PATH,
+        )
+        bot = DevPlaceBot(rc, on_event=self.log)
+        task = asyncio.create_task(self._run_slot(slot, bot, cfg["bot_max_actions"]))
+        self._fleet[slot] = {"bot": bot, "task": task}
+        self.log(f"Launched bot{slot}")
+
+    async def _run_slot(self, slot: int, bot, max_actions: int) -> None:
+        try:
+            await bot.run_forever(action_limit=max_actions)
+        except asyncio.CancelledError:
+            raise
+        except Exception as e:
+            self.log(f"bot{slot} crashed: {e}")
+
+    async def on_disable(self) -> None:
+        await self._stop_fleet()
+
+    async def _stop_fleet(self) -> None:
+        for slot in list(self._fleet):
+            await self._stop_slot(slot)
+
+    async def _stop_slot(self, slot: int) -> None:
+        entry = self._fleet.pop(slot, None)
+        if not entry:
+            return
+        task = entry["task"]
+        task.cancel()
+        try:
+            await asyncio.wait_for(task, timeout=30)
+        except (asyncio.CancelledError, asyncio.TimeoutError):
+            pass
+        except Exception as e:
+            self.log(f"bot{slot} stop error: {e}")
+        self.log(f"Stopped bot{slot}")
+
+    def collect_metrics(self) -> dict:
+        rows = []
+        total_cost = 0.0
+        total_calls = total_in = total_out = 0
+        posts = comments = votes = running = 0
+        for slot in sorted(self._fleet):
+            entry = self._fleet[slot]
+            bot = entry["bot"]
+            alive = not entry["task"].done()
+            running += 1 if alive else 0
+            llm = getattr(bot, "llm", None)
+            st = getattr(bot, "state", None)
+            cost = llm.total_cost if llm else 0.0
+            calls = llm.total_calls if llm else 0
+            total_cost += cost
+            total_calls += calls
+            total_in += llm.total_in_tokens if llm else 0
+            total_out += llm.total_out_tokens if llm else 0
+            if st:
+                posts += st.created_posts
+                comments += st.comments_posted
+                votes += st.votes_cast
+            rows.append([
+                f"bot{slot}",
+                (st.username if st else "") or "-",
+                (st.persona if st else "") or "-",
+                "running" if alive else "stopped",
+                st.created_posts if st else 0,
+                st.comments_posted if st else 0,
+                st.votes_cast if st else 0,
+                calls,
+                f"${cost:.4f}",
+            ])
+        window, observed_cost, ready = self._sample_cost(datetime.now(timezone.utc), total_cost)
+        cost_per_hour = observed_cost / window * 3600 if ready else 0.0
+        projected_24h = observed_cost / window * 86400 if ready else 0.0
+        rate_value = f"${cost_per_hour:.4f}/h" if ready else "warming up"
+        projected_value = f"${projected_24h:.2f}" if ready else "warming up"
+        stats = [
+            {"label": "Bots running", "value": running},
+            {"label": "Fleet cost", "value": f"${total_cost:.4f}"},
+            {"label": "Observed window", "value": f"{window:.0f}s"},
+            {"label": "Cost rate", "value": rate_value},
+            {"label": "Projected 24h cost", "value": projected_value},
+            {"label": "LLM calls", "value": total_calls},
+            {"label": "Tokens in", "value": total_in},
+            {"label": "Tokens out", "value": total_out},
+            {"label": "Posts", "value": posts},
+            {"label": "Comments", "value": comments},
+            {"label": "Votes", "value": votes},
+        ]
+        table = {
+            "columns": ["Bot", "User", "Persona", "Status", "Posts", "Comments", "Votes", "Calls", "Cost"],
+            "rows": rows,
+        }
+        return {"stats": stats, "table": table}
+
+    def _sample_cost(self, now: datetime, total_cost: float) -> tuple[float, float, bool]:
+        if self._started_at is None:
+            self._cost_samples.clear()
+            self._cost_anchor = None
+            return 0.0, 0.0, False
+        if self._cost_anchor != self._started_at:
+            self._cost_samples.clear()
+            self._cost_anchor = self._started_at
+        self._cost_samples.append((now, total_cost))
+        cutoff = now - timedelta(seconds=config.COST_WINDOW_SECONDS)
+        while len(self._cost_samples) > 2 and self._cost_samples[0][0] < cutoff:
+            self._cost_samples.popleft()
+        start_at, start_cost = self._cost_samples[0]
+        window = (now - start_at).total_seconds()
+        observed_cost = max(0.0, total_cost - start_cost)
+        ready = window >= config.COST_WARMUP_SECONDS
+        return window, observed_cost, ready

devplacepy/services/bot/state.py

@@ -0,0 +1,65 @@
+import json
+import logging
+import os
+from dataclasses import dataclass, field, asdict
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class BotState:
+    username: str = ""
+    email: str = ""
+    password: str = ""
+    uid: str = ""
+    persona: str = ""
+    log: list[str] = field(default_factory=list)
+    known_users: list[str] = field(default_factory=list)
+    known_posts: list[str] = field(default_factory=list)
+    created_posts: int = 0
+    posted_content_hashes: list[str] = field(default_factory=list)
+    comments_posted: int = 0
+    quality_rejections: int = 0
+    votes_cast: int = 0
+    bugs_filed: int = 0
+    projects_created: int = 0
+    gists_created: int = 0
+    profiles_viewed: int = 0
+    started_at: str = ""
+    own_post_urls: list[str] = field(default_factory=list)
+    voted_post_ids: list[str] = field(default_factory=list)
+    commented_post_ids: list[str] = field(default_factory=list)
+    notifications_seen: list[str] = field(default_factory=list)
+    mentions_received: int = 0
+    mentions_replied: int = 0
+    replies_to_own_posts: int = 0
+    news_comments: int = 0
+    projects_starred: int = 0
+    messages_sent: int = 0
+    searches_run: int = 0
+    poll_votes_cast: int = 0
+    reactions_cast: int = 0
+    polls_voted: list[str] = field(default_factory=list)
+    reacted_targets: list[str] = field(default_factory=list)
+    reading_speed_wpm: int = 0
+    total_cost: float = 0.0
+    total_calls: int = 0
+    total_in_tokens: int = 0
+    total_out_tokens: int = 0
+
+    def save(self, path: Path) -> None:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        tmp = path.with_suffix(path.suffix + ".tmp")
+        tmp.write_text(json.dumps(asdict(self), indent=2, default=str))
+        os.replace(str(tmp), str(path))
+
+    @classmethod
+    def load(cls, path: Path) -> "BotState":
+        if path.exists():
+            try:
+                data = json.loads(path.read_text())
+                return cls(**data)
+            except Exception as e:
+                logger.warning("Failed to load state: %s", e)
+        return cls()

devplacepy/services/devii/__init__.py

@@ -0,0 +1,5 @@
+# retoor <retoor@molodetz.nl>
+
+from .service import DeviiService
+
+__all__ = ["DeviiService"]

devplacepy/services/devii/__main__.py

@@ -0,0 +1,6 @@
+# retoor <retoor@molodetz.nl>
+
+from .cli import main
+
+if __name__ == "__main__":
+    main()

devplacepy/services/devii/actions/__init__.py

@@ -0,0 +1,7 @@
+# retoor <retoor@molodetz.nl>
+
+from .catalog import ACTIONS, PLATFORM_CATALOG
+from .dispatcher import Dispatcher
+from .spec import Action, Catalog, Param
+
+__all__ = ["ACTIONS", "PLATFORM_CATALOG", "Dispatcher", "Action", "Catalog", "Param"]

devplacepy/services/devii/actions/avatar_actions.py

@@ -0,0 +1,149 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from .spec import Action, Param
+
+
+def arg(name: str, description: str, required: bool = False, kind: str = "string") -> Param:
+    return Param(name=name, location="body", description=description, required=required, type=kind)
+
+
+DEVII = (
+    "Controls devii, the on-screen avatar that is you rendered as a classic animated "
+    "desktop character. Only effective in the web interface; in other interfaces it reports "
+    "that no avatar is attached."
+)
+
+AVATAR_ACTIONS: tuple[Action, ...] = (
+    Action(
+        name="avatar_show",
+        method="LOCAL",
+        path="",
+        summary="Show devii (yourself) on screen",
+        description=DEVII,
+        handler="avatar",
+        requires_auth=False,
+    ),
+    Action(
+        name="avatar_hide",
+        method="LOCAL",
+        path="",
+        summary="Hide devii from the screen (plays a goodbye/hide animation)",
+        description=(
+            DEVII + " This already plays a goodbye animation, so do not queue a separate "
+            "wave/goodbye animation immediately before calling it - hiding interrupts a still-"
+            "queued animation."
+        ),
+        handler="avatar",
+        requires_auth=False,
+    ),
+    Action(
+        name="avatar_speak",
+        method="LOCAL",
+        path="",
+        summary="Show a line of text in devii's speech balloon",
+        description=DEVII,
+        handler="avatar",
+        requires_auth=False,
+        params=(
+            arg("text", "The text devii should say.", required=True),
+            arg("hold", "Keep the balloon open until the next action.", kind="boolean"),
+        ),
+    ),
+    Action(
+        name="avatar_list_animations",
+        method="LOCAL",
+        path="",
+        summary="List every animation the current character supports",
+        description=DEVII,
+        handler="avatar",
+        requires_auth=False,
+    ),
+    Action(
+        name="avatar_play_animation",
+        method="LOCAL",
+        path="",
+        summary="Play a specific named animation (use avatar_list_animations first)",
+        description=(
+            DEVII + " A following avatar_hide or avatar_stop interrupts an animation that is "
+            "still playing, so do not hide immediately after if you want it to finish."
+        ),
+        handler="avatar",
+        requires_auth=False,
+        params=(arg("name", "Exact animation name.", required=True),),
+    ),
+    Action(
+        name="avatar_random_animation",
+        method="LOCAL",
+        path="",
+        summary="Play a random non-idle animation to react expressively",
+        description=DEVII,
+        handler="avatar",
+        requires_auth=False,
+    ),
+    Action(
+        name="avatar_move_to",
+        method="LOCAL",
+        path="",
+        summary="Walk devii to a pixel position on screen",
+        description=DEVII,
+        handler="avatar",
+        requires_auth=False,
+        params=(
+            arg("x", "Target x in pixels.", required=True, kind="integer"),
+            arg("y", "Target y in pixels.", required=True, kind="integer"),
+            arg("duration", "Movement duration in milliseconds.", kind="integer"),
+        ),
+    ),
+    Action(
+        name="avatar_gesture_at",
+        method="LOCAL",
+        path="",
+        summary="Make devii gesture toward a pixel position",
+        description=DEVII,
+        handler="avatar",
+        requires_auth=False,
+        params=(
+            arg("x", "Target x in pixels.", required=True, kind="integer"),
+            arg("y", "Target y in pixels.", required=True, kind="integer"),
+        ),
+    ),
+    Action(
+        name="avatar_get_viewport",
+        method="LOCAL",
+        path="",
+        summary="Get the screen size and devii's current position",
+        description=DEVII,
+        handler="avatar",
+        requires_auth=False,
+    ),
+    Action(
+        name="avatar_stop",
+        method="LOCAL",
+        path="",
+        summary="Stop all of devii's queued animations and speech immediately",
+        description=DEVII,
+        handler="avatar",
+        requires_auth=False,
+    ),
+    Action(
+        name="avatar_list_characters",
+        method="LOCAL",
+        path="",
+        summary="List every character devii can appear as",
+        description=DEVII,
+        handler="avatar",
+        requires_auth=False,
+    ),
+    Action(
+        name="avatar_switch_character",
+        method="LOCAL",
+        path="",
+        summary="Change which character devii is rendered as",
+        description=DEVII,
+        handler="avatar",
+        requires_auth=False,
+        params=(arg("name", "Character name (e.g. Clippy, Merlin, Bonzi, Genie, Rover).", required=True),),
+    ),
+)

devplacepy/services/devii/actions/catalog.py

@@ -0,0 +1,793 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from .spec import Action, Catalog, Param
+
+
+def path(name: str, description: str, required: bool = True) -> Param:
+    return Param(name=name, location="path", description=description, required=required)
+
+
+def query(name: str, description: str, required: bool = False) -> Param:
+    return Param(name=name, location="query", description=description, required=required)
+
+
+def body(name: str, description: str, required: bool = False) -> Param:
+    return Param(name=name, location="body", description=description, required=required)
+
+
+def upload(name: str, description: str) -> Param:
+    return Param(name=name, location="file", description=description, required=True)
+
+
+ATTACHMENTS = "Comma separated attachment uids returned by upload_file."
+TARGET_TYPE = "Target type, e.g. post, comment, project, gist."
+
+ACTIONS: tuple[Action, ...] = (
+    Action(
+        name="auth_status",
+        method="GET",
+        path="",
+        summary="Report whether the current session is authenticated",
+        handler="status",
+        requires_auth=False,
+    ),
+    Action(
+        name="login",
+        method="POST",
+        path="/auth/login",
+        summary="Authenticate with email and password and start a session",
+        handler="login",
+        requires_auth=False,
+        params=(
+            body("email", "Account email address.", required=True),
+            body("password", "Account password.", required=True),
+            body("remember_me", "Keep the session alive longer ('on' or '')."),
+        ),
+    ),
+    Action(
+        name="logout",
+        method="GET",
+        path="/auth/logout",
+        summary="End the current session",
+        handler="logout",
+        requires_auth=True,
+    ),
+    Action(
+        name="signup",
+        method="POST",
+        path="/auth/signup",
+        summary="Create a new account",
+        requires_auth=False,
+        params=(
+            body("username", "Desired username.", required=True),
+            body("email", "Email address.", required=True),
+            body("password", "Password (minimum six characters).", required=True),
+            body("confirm_password", "Password confirmation.", required=True),
+        ),
+    ),
+    Action(
+        name="forgot_password",
+        method="POST",
+        path="/auth/forgot-password",
+        summary="Request a password reset email",
+        requires_auth=False,
+        params=(body("email", "Account email address.", required=True),),
+    ),
+    Action(
+        name="reset_password",
+        method="POST",
+        path="/auth/reset-password/{token}",
+        summary="Reset a password using a reset token",
+        requires_auth=False,
+        params=(
+            path("token", "Reset token from the email link."),
+            body("password", "New password.", required=True),
+            body("confirm_password", "New password confirmation.", required=True),
+        ),
+    ),
+    Action(
+        name="view_feed",
+        method="GET",
+        path="/feed",
+        summary="View the activity feed",
+        params=(
+            query("tab", "Feed tab to view."),
+            query("topic", "Filter by topic."),
+            query("before", "Pagination cursor."),
+        ),
+    ),
+    Action(
+        name="create_post",
+        method="POST",
+        path="/posts/create",
+        summary="Create a new post",
+        params=(
+            body("content", "Post body text.", required=True),
+            body("title", "Optional post title."),
+            body("topic", "Optional topic."),
+            body("project_uid", "Attach the post to a project uid."),
+            body("attachment_uids", ATTACHMENTS),
+            body("poll_question", "Optional poll question."),
+            body("poll_options", "Poll options as a JSON array of strings, or one option per line, or comma separated. At least two are required for the poll to be created."),
+        ),
+    ),
+    Action(
+        name="view_post",
+        method="GET",
+        path="/posts/{post_slug}",
+        summary="View a single post by slug",
+        params=(path("post_slug", "Exact post slug copied from a /posts/... link in a feed or listing response; do not build it from the title."),),
+    ),
+    Action(
+        name="edit_post",
+        method="POST",
+        path="/posts/edit/{post_slug}",
+        summary="Edit an existing post",
+        params=(
+            path("post_slug", "Exact post slug copied from a /posts/... link in a feed or listing response; do not build it from the title."),
+            body("content", "Updated post body.", required=True),
+            body("title", "Updated title."),
+            body("topic", "Updated topic."),
+            body("poll_question", "Optional poll question. Adds a poll to a post that does not already have one."),
+            body("poll_options", "Poll options as a JSON array of strings, or one option per line, or comma separated. At least two are required for the poll to be created."),
+        ),
+    ),
+    Action(
+        name="delete_post",
+        method="POST",
+        path="/posts/delete/{post_slug}",
+        summary="Delete a post",
+        params=(path("post_slug", "Exact post slug copied from a /posts/... link in a feed or listing response; do not build it from the title."),),
+    ),
+    Action(
+        name="create_comment",
+        method="POST",
+        path="/comments/create",
+        summary="Create a comment on a post or other target",
+        params=(
+            body("content", "Comment body.", required=True),
+            body("post_uid", "Uid of the post being commented on."),
+            body("target_uid", "Uid of the target when not a post."),
+            body("target_type", TARGET_TYPE),
+            body("parent_uid", "Parent comment uid for replies."),
+            body("attachment_uids", ATTACHMENTS),
+        ),
+    ),
+    Action(
+        name="delete_comment",
+        method="POST",
+        path="/comments/delete/{comment_uid}",
+        summary="Delete a comment",
+        params=(path("comment_uid", "Uid of the comment."),),
+    ),
+    Action(
+        name="list_projects",
+        method="GET",
+        path="/projects",
+        summary="List projects",
+        params=(
+            query("tab", "Projects tab."),
+            query("search", "Search query."),
+            query("user_uid", "Filter by owner uid."),
+            query("project_type", "Filter by project type."),
+            query("before", "Pagination cursor."),
+        ),
+    ),
+    Action(
+        name="view_project",
+        method="GET",
+        path="/projects/{project_slug}",
+        summary="View a project by slug",
+        params=(path("project_slug", "Exact project slug copied from a /projects/... link in a listing response; do not build it from the title."),),
+    ),
+    Action(
+        name="create_project",
+        method="POST",
+        path="/projects/create",
+        summary="Create a new project",
+        params=(
+            body("title", "Project title.", required=True),
+            body("description", "Project description.", required=True),
+            body("release_date", "Release date."),
+            body("demo_date", "Demo date."),
+            body("project_type", "Project type."),
+            body("platforms", "Supported platforms."),
+            body("status", "Project status."),
+            body("attachment_uids", ATTACHMENTS),
+        ),
+    ),
+    Action(
+        name="delete_project",
+        method="POST",
+        path="/projects/delete/{project_slug}",
+        summary="Delete a project",
+        params=(path("project_slug", "Exact project slug copied from a /projects/... link in a listing response; do not build it from the title."),),
+    ),
+    Action(
+        name="project_list_files",
+        method="GET",
+        path="/projects/{project_slug}/files",
+        summary="List every file and directory in a project filesystem",
+        description="Returns the flat list of nodes (path, type, size, mime). Use it to inspect the project tree before reading or writing files.",
+        params=(path("project_slug", "Project slug or uid."),),
+        requires_auth=False,
+    ),
+    Action(
+        name="project_read_file",
+        method="GET",
+        path="/projects/{project_slug}/files/raw",
+        summary="Read one file from a project filesystem",
+        description="Returns the file metadata plus the text content. Binary files return a url instead of content.",
+        params=(
+            path("project_slug", "Project slug or uid."),
+            query("path", "Relative file path inside the project, e.g. src/main.py.", required=True),
+        ),
+        requires_auth=False,
+    ),
+    Action(
+        name="project_write_file",
+        method="POST",
+        path="/projects/{project_slug}/files/write",
+        summary="Create or overwrite a text file in a project (parent directories are created automatically)",
+        description="The primary tool for building a project: write any text file by path. Missing parent directories are created recursively.",
+        params=(
+            path("project_slug", "Project slug or uid."),
+            body("path", "Relative file path, e.g. src/app/main.py.", required=True),
+            body("content", "Full file content.", required=True),
+        ),
+    ),
+    Action(
+        name="project_upload_file",
+        method="POST",
+        path="/projects/{project_slug}/files/upload",
+        summary="Upload a local file into a project directory (parents created automatically)",
+        params=(
+            path("project_slug", "Project slug or uid."),
+            upload("file", "Local filesystem path of the file to upload."),
+            body("path", "Target directory inside the project, empty for the root."),
+        ),
+    ),
+    Action(
+        name="project_make_dir",
+        method="POST",
+        path="/projects/{project_slug}/files/mkdir",
+        summary="Create a directory (and parents) in a project filesystem",
+        params=(
+            path("project_slug", "Project slug or uid."),
+            body("path", "Relative directory path, e.g. src/components.", required=True),
+        ),
+    ),
+    Action(
+        name="project_move_file",
+        method="POST",
+        path="/projects/{project_slug}/files/move",
+        summary="Move or rename a file or directory within a project",
+        params=(
+            path("project_slug", "Project slug or uid."),
+            body("from_path", "Existing path.", required=True),
+            body("to_path", "New path.", required=True),
+        ),
+    ),
+    Action(
+        name="project_delete_file",
+        method="POST",
+        path="/projects/{project_slug}/files/delete",
+        summary="Delete a file or directory (recursive) from a project filesystem",
+        params=(
+            path("project_slug", "Project slug or uid."),
+            body("path", "Relative path to delete.", required=True),
+        ),
+    ),
+    Action(
+        name="zip_project",
+        method="POST",
+        path="/projects/{project_slug}/zip",
+        summary="Build a downloadable zip archive of a whole project",
+        description=(
+            "Queues a background zip job and returns {uid, status_url}. Poll the status_url with "
+            "zip_status until status is 'done', then give the user the download_url."
+        ),
+        params=(path("project_slug", "Project slug or uid."),),
+        requires_auth=False,
+    ),
+    Action(
+        name="zip_status",
+        method="GET",
+        path="/zips/{uid}",
+        summary="Check a zip job and obtain its download link once finished",
+        description=(
+            "Returns the job status and stats. When status is 'done', download_url points at the "
+            "ready archive; while 'pending' or 'running', poll again shortly."
+        ),
+        params=(path("uid", "Zip job uid returned by zip_project."),),
+        requires_auth=False,
+    ),
+    Action(
+        name="search_users",
+        method="GET",
+        path="/profile/search",
+        summary="Search for users by name",
+        params=(query("q", "Search query."),),
+    ),
+    Action(
+        name="view_profile",
+        method="GET",
+        path="/profile/{username}",
+        summary="View a user profile",
+        params=(
+            path("username", "Username to view."),
+            query("tab", "Profile tab."),
+        ),
+    ),
+    Action(
+        name="update_profile",
+        method="POST",
+        path="/profile/update",
+        summary="Update the current user's profile",
+        params=(
+            body("bio", "Profile biography."),
+            body("location", "Location."),
+            body("git_link", "Git profile link."),
+            body("website", "Personal website."),
+        ),
+    ),
+    Action(
+        name="regenerate_api_key",
+        method="POST",
+        path="/profile/regenerate-api-key",
+        summary="Issue a new API key and invalidate the current one",
+        description=(
+            "Returns the new api_key. Warning: this immediately invalidates any key currently "
+            "used for authentication, so confirm with the user before calling it."
+        ),
+    ),
+    Action(
+        name="list_messages",
+        method="GET",
+        path="/messages",
+        summary="View direct message conversations",
+        params=(
+            query("with_uid", "Open a conversation with a user uid."),
+            query("search", "Search conversations."),
+        ),
+    ),
+    Action(
+        name="search_message_users",
+        method="GET",
+        path="/messages/search",
+        summary="Search users to message",
+        params=(query("q", "Search query."),),
+    ),
+    Action(
+        name="send_message",
+        method="POST",
+        path="/messages/send",
+        summary="Send a direct message",
+        params=(
+            body("content", "Message body.", required=True),
+            body("receiver_uid", "Recipient user uid.", required=True),
+            body("attachment_uids", ATTACHMENTS),
+        ),
+    ),
+    Action(
+        name="list_notifications",
+        method="GET",
+        path="/notifications",
+        summary="List notifications",
+        params=(query("before", "Pagination cursor."),),
+    ),
+    Action(
+        name="notification_counts",
+        method="GET",
+        path="/notifications/counts",
+        summary="Get unread notification and message counts",
+    ),
+    Action(
+        name="open_notification",
+        method="GET",
+        path="/notifications/open/{notification_uid}",
+        summary="Open a notification and follow it",
+        params=(path("notification_uid", "Notification uid."),),
+    ),
+    Action(
+        name="mark_notification_read",
+        method="POST",
+        path="/notifications/mark-read/{notification_uid}",
+        summary="Mark a single notification read",
+        params=(path("notification_uid", "Notification uid."),),
+    ),
+    Action(
+        name="mark_all_notifications_read",
+        method="POST",
+        path="/notifications/mark-all-read",
+        summary="Mark all notifications read",
+    ),
+    Action(
+        name="vote",
+        method="POST",
+        path="/votes/{target_type}/{target_uid}",
+        summary="Cast or toggle a vote on a target",
+        description="Re-sending the same value removes the vote. Returns the net/up/down tally.",
+        ajax=True,
+        params=(
+            path("target_type", TARGET_TYPE),
+            path("target_uid", "Uid of the target, copied from a listing response; do not invent it."),
+            body("value", "Vote value: 1 to upvote, -1 to downvote (re-send to remove).", required=True),
+        ),
+    ),
+    Action(
+        name="react",
+        method="POST",
+        path="/reactions/{target_type}/{target_uid}",
+        summary="Toggle an emoji reaction on a target",
+        description="Re-sending the same emoji removes it. Returns reaction counts.",
+        ajax=True,
+        params=(
+            path("target_type", TARGET_TYPE),
+            path("target_uid", "Uid of the target, copied from a listing response; do not invent it."),
+            body("emoji", "One of the allowed reaction emoji.", required=True),
+        ),
+    ),
+    Action(
+        name="list_bookmarks",
+        method="GET",
+        path="/bookmarks/saved",
+        summary="List saved bookmarks",
+        params=(query("before", "Pagination cursor."),),
+    ),
+    Action(
+        name="toggle_bookmark",
+        method="POST",
+        path="/bookmarks/{target_type}/{target_uid}",
+        summary="Toggle a bookmark on a target",
+        description="Re-sending removes the bookmark. Returns {saved: true|false}.",
+        ajax=True,
+        params=(
+            path("target_type", TARGET_TYPE),
+            path("target_uid", "Uid of the target, copied from a listing response; do not invent it."),
+        ),
+    ),
+    Action(
+        name="vote_poll",
+        method="POST",
+        path="/polls/{poll_uid}/vote",
+        summary="Vote in a poll",
+        description="Casts or changes your vote on a poll option. Returns the option tallies.",
+        ajax=True,
+        params=(
+            path("poll_uid", "Poll uid."),
+            body("option_uid", "Chosen option uid.", required=True),
+        ),
+    ),
+    Action(
+        name="follow_user",
+        method="POST",
+        path="/follow/{username}",
+        summary="Follow a user",
+        params=(path("username", "Username to follow."),),
+    ),
+    Action(
+        name="unfollow_user",
+        method="POST",
+        path="/follow/unfollow/{username}",
+        summary="Unfollow a user",
+        params=(path("username", "Username to unfollow."),),
+    ),
+    Action(
+        name="list_followers",
+        method="GET",
+        path="/profile/{username}/followers",
+        summary="List the users who follow a profile",
+        requires_auth=False,
+        params=(
+            path("username", "Username whose followers to list."),
+            query("page", "Page number, 25 per page."),
+        ),
+    ),
+    Action(
+        name="list_following",
+        method="GET",
+        path="/profile/{username}/following",
+        summary="List the users a profile follows",
+        requires_auth=False,
+        params=(
+            path("username", "Username whose following to list."),
+            query("page", "Page number, 25 per page."),
+        ),
+    ),
+    Action(
+        name="view_leaderboard",
+        method="GET",
+        path="/leaderboard",
+        summary="View the leaderboard",
+    ),
+    Action(
+        name="list_bugs",
+        method="GET",
+        path="/bugs",
+        summary="List reported bugs",
+    ),
+    Action(
+        name="create_bug",
+        method="POST",
+        path="/bugs/create",
+        summary="Report a bug",
+        params=(
+            body("title", "Bug title.", required=True),
+            body("description", "Bug description.", required=True),
+            body("attachment_uids", ATTACHMENTS),
+        ),
+    ),
+    Action(
+        name="list_gists",
+        method="GET",
+        path="/gists",
+        summary="List gists",
+        params=(
+            query("language", "Filter by language."),
+            query("user_uid", "Filter by owner uid."),
+            query("before", "Pagination cursor."),
+        ),
+    ),
+    Action(
+        name="view_gist",
+        method="GET",
+        path="/gists/{gist_slug}",
+        summary="View a gist by slug",
+        params=(path("gist_slug", "Exact gist slug copied from a /gists/... link in a listing response; do not build it from the title."),),
+    ),
+    Action(
+        name="create_gist",
+        method="POST",
+        path="/gists/create",
+        summary="Create a gist",
+        params=(
+            body("title", "Gist title.", required=True),
+            body("source_code", "Gist source code.", required=True),
+            body("description", "Gist description."),
+            body("language", "Programming language."),
+            body("attachment_uids", ATTACHMENTS),
+        ),
+    ),
+    Action(
+        name="edit_gist",
+        method="POST",
+        path="/gists/edit/{gist_slug}",
+        summary="Edit a gist",
+        params=(
+            path("gist_slug", "Exact gist slug copied from a /gists/... link in a listing response; do not build it from the title."),
+            body("title", "Gist title.", required=True),
+            body("source_code", "Gist source code.", required=True),
+            body("description", "Gist description."),
+            body("language", "Programming language."),
+        ),
+    ),
+    Action(
+        name="delete_gist",
+        method="POST",
+        path="/gists/delete/{gist_slug}",
+        summary="Delete a gist",
+        params=(path("gist_slug", "Exact gist slug copied from a /gists/... link in a listing response; do not build it from the title."),),
+    ),
+    Action(
+        name="list_news",
+        method="GET",
+        path="/news",
+        summary="List news articles",
+        params=(query("before", "Pagination cursor."),),
+    ),
+    Action(
+        name="view_news",
+        method="GET",
+        path="/news/{news_slug}",
+        summary="View a news article by slug",
+        params=(path("news_slug", "Exact news slug copied from a /news/... link in a listing response; do not build it from the title."),),
+    ),
+    Action(
+        name="upload_file",
+        method="POST",
+        path="/uploads/upload",
+        summary="Upload a local file and obtain its attachment uid",
+        params=(upload("file", "Local filesystem path of the file to upload."),),
+    ),
+    Action(
+        name="delete_attachment",
+        method="DELETE",
+        path="/uploads/delete/{attachment_uid}",
+        summary="Delete an uploaded attachment",
+        params=(path("attachment_uid", "Uid of the attachment."),),
+    ),
+    Action(
+        name="admin_overview",
+        method="GET",
+        path="/admin",
+        summary="View the admin overview",
+    ),
+    Action(
+        name="site_analytics",
+        method="GET",
+        path="/admin/analytics",
+        summary="Site-wide aggregate analytics in one call (admin only)",
+        description=(
+            "Returns JSON: total members, active users in the last 24h/7d/30d, users signed in "
+            "now, new signups (24h/7d/30d), content totals (posts, comments, gists, projects, "
+            "news), and top authors. Use this for any 'how many'/'how active'/count question "
+            "instead of paging through admin_list_users."
+        ),
+        params=(query("top_n", "How many top authors to include (1-50)."),),
+        requires_admin=True,
+    ),
+    Action(
+        name="ai_usage",
+        method="GET",
+        path="/admin/ai-usage/data",
+        summary="AI gateway usage, cost, latency, and reliability metrics (admin only)",
+        description=(
+            "Returns JSON for a bounded window: request volume and throughput, token usage with "
+            "averages and percentiles, latency (upstream, gateway overhead, queue wait, connect), "
+            "error rates by category, cost in USD (per model, per caller, input vs output, projected "
+            "monthly burn, caching savings), caller behavior, and an hourly breakdown. Use this for "
+            "any question about AI spend, token consumption, gateway performance, or errors."
+        ),
+        params=(
+            query("hours", "Lookback window in hours (1-168, default 48)."),
+            query("top_n", "How many rows in each top-N breakdown (default 10)."),
+        ),
+        requires_admin=True,
+    ),
+    Action(
+        name="admin_list_users",
+        method="GET",
+        path="/admin/users",
+        summary="List users for administration",
+        params=(query("page", "Page number."),),
+    ),
+    Action(
+        name="admin_set_user_role",
+        method="POST",
+        path="/admin/users/{uid}/role",
+        summary="Set a user's role",
+        params=(
+            path("uid", "User uid."),
+            body("role", "New role.", required=True),
+        ),
+    ),
+    Action(
+        name="admin_set_user_password",
+        method="POST",
+        path="/admin/users/{uid}/password",
+        summary="Set a user's password",
+        params=(
+            path("uid", "User uid."),
+            body("password", "New password.", required=True),
+        ),
+    ),
+    Action(
+        name="admin_toggle_user",
+        method="POST",
+        path="/admin/users/{uid}/toggle",
+        summary="Toggle a user's active state",
+        params=(path("uid", "User uid."),),
+    ),
+    Action(
+        name="admin_get_settings",
+        method="GET",
+        path="/admin/settings",
+        summary="View site settings",
+    ),
+    Action(
+        name="admin_save_settings",
+        method="POST",
+        path="/admin/settings",
+        summary="Save site settings",
+        params=(
+            body("site_name", "Site name."),
+            body("site_description", "Site description."),
+            body("site_tagline", "Site tagline."),
+            body("max_upload_size_mb", "Maximum upload size in megabytes."),
+            body("allowed_file_types", "Allowed file types."),
+            body("max_attachments_per_resource", "Maximum attachments per resource."),
+            body("rate_limit_per_minute", "Rate limit per minute."),
+            body("rate_limit_window_seconds", "Rate limit window in seconds."),
+            body("session_max_age_days", "Session maximum age in days."),
+            body("session_remember_days", "Remember-me duration in days."),
+            body("registration_open", "Whether registration is open."),
+            body("maintenance_mode", "Whether maintenance mode is on."),
+            body("maintenance_message", "Maintenance message."),
+        ),
+    ),
+    Action(
+        name="admin_list_news",
+        method="GET",
+        path="/admin/news",
+        summary="List news for administration",
+        params=(query("page", "Page number."),),
+    ),
+    Action(
+        name="admin_toggle_news",
+        method="POST",
+        path="/admin/news/{uid}/toggle",
+        summary="Toggle a news article",
+        params=(path("uid", "News uid."),),
+    ),
+    Action(
+        name="admin_publish_news",
+        method="POST",
+        path="/admin/news/{uid}/publish",
+        summary="Publish a news article",
+        params=(path("uid", "News uid."),),
+    ),
+    Action(
+        name="admin_landing_news",
+        method="POST",
+        path="/admin/news/{uid}/landing",
+        summary="Set a news article as landing content",
+        params=(path("uid", "News uid."),),
+    ),
+    Action(
+        name="admin_delete_news",
+        method="POST",
+        path="/admin/news/{uid}/delete",
+        summary="Delete a news article",
+        params=(path("uid", "News uid."),),
+    ),
+    Action(
+        name="admin_list_services",
+        method="GET",
+        path="/admin/services",
+        summary="View managed services",
+    ),
+    Action(
+        name="admin_services_data",
+        method="GET",
+        path="/admin/services/data",
+        summary="Get live status, metrics, and log tail for every background service",
+    ),
+    Action(
+        name="admin_service_status",
+        method="GET",
+        path="/admin/services/{name}/data",
+        summary="Get live status, metrics, and log tail for one background service",
+        params=(path("name", "Service name (e.g. news, bots, openai)."),),
+    ),
+    Action(
+        name="admin_start_service",
+        method="POST",
+        path="/admin/services/{name}/start",
+        summary="Start a managed service",
+        params=(path("name", "Service name."),),
+    ),
+    Action(
+        name="admin_stop_service",
+        method="POST",
+        path="/admin/services/{name}/stop",
+        summary="Stop a managed service",
+        params=(path("name", "Service name."),),
+    ),
+    Action(
+        name="admin_run_service",
+        method="POST",
+        path="/admin/services/{name}/run",
+        summary="Run a managed service once",
+        params=(path("name", "Service name."),),
+    ),
+    Action(
+        name="admin_clear_service_logs",
+        method="POST",
+        path="/admin/services/{name}/clear-logs",
+        summary="Clear a managed service's logs",
+        params=(path("name", "Service name."),),
+    ),
+    Action(
+        name="admin_config_service",
+        method="POST",
+        path="/admin/services/{name}/config",
+        summary="Update a managed service's configuration",
+        params=(path("name", "Service name."),),
+        freeform_body=True,
+    ),
+)
+
+PLATFORM_CATALOG = Catalog(actions=ACTIONS)

devplacepy/services/devii/actions/chunk_actions.py

@@ -0,0 +1,41 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from .spec import Action, Param
+
+CHUNK_ACTIONS: tuple[Action, ...] = (
+    Action(
+        name="read_more",
+        method="LOCAL",
+        path="",
+        summary="Read the next part of a previously truncated tool result",
+        description=(
+            "When any tool result is truncated it includes a chunk_id, total_chars, "
+            "remaining_chars, and next_offset. Call read_more with that chunk_id and offset to "
+            "page through the rest until remaining_chars is 0, so you can read the whole content."
+        ),
+        handler="chunks",
+        requires_auth=False,
+        params=(
+            Param(
+                name="chunk_id",
+                location="body",
+                description="The chunk_id from a truncated result.",
+                required=True,
+            ),
+            Param(
+                name="offset",
+                location="body",
+                description="Character offset to read from (use next_offset from the prior part).",
+                type="integer",
+            ),
+            Param(
+                name="length",
+                location="body",
+                description="Optional number of characters to return (capped to the response limit).",
+                type="integer",
+            ),
+        ),
+    ),
+)

devplacepy/services/devii/actions/client_actions.py

@@ -0,0 +1,112 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from .spec import Action, Param
+
+
+def arg(name: str, description: str, required: bool = False, kind: str = "string") -> Param:
+    return Param(name=name, location="body", description=description, required=required, type=kind)
+
+
+CLIENT = (
+    "Runs in the user's own browser through the web terminal. Only effective in the web "
+    "interface with a live connection; elsewhere it reports that no browser is attached."
+)
+
+CLIENT_ACTIONS: tuple[Action, ...] = (
+    Action(
+        name="get_page_context",
+        method="LOCAL",
+        path="",
+        summary="Read the user's current page: URL, title, viewport, scroll, selected text, visible headings, and whether they are signed in",
+        description=CLIENT + " Use this to understand where the user is and what they are looking at before acting or guiding them.",
+        handler="client",
+        requires_auth=False,
+    ),
+    Action(
+        name="run_js",
+        method="LOCAL",
+        path="",
+        summary="Execute JavaScript in the user's browser and return its result",
+        description=(
+            CLIENT + " The code is the body of an async function; use 'return value' to return a "
+            "JSON-serializable result. You have full access to window and document. Use this for "
+            "anything not covered by the dedicated tools: read or change the DOM, drive a live "
+            "demo, inspect state, or update the screen. Prefer the dedicated tools "
+            "(highlight_element, show_toast, scroll_to_element, navigate_to, reload_page) when they fit."
+        ),
+        handler="client",
+        requires_auth=False,
+        params=(
+            arg("code", "JavaScript to run as an async function body. Return a JSON-serializable value.", required=True),
+        ),
+    ),
+    Action(
+        name="highlight_element",
+        method="LOCAL",
+        path="",
+        summary="Highlight an element on screen with an outline and an optional callout label, for live tutorials",
+        description=CLIENT + " Scrolls the element into view and draws an attention outline. Call clear_highlights to remove it.",
+        handler="client",
+        requires_auth=False,
+        params=(
+            arg("selector", "CSS selector, or the element's exact visible text (e.g. a heading or link label).", required=True),
+            arg("label", "Optional callout text shown next to the element."),
+        ),
+    ),
+    Action(
+        name="clear_highlights",
+        method="LOCAL",
+        path="",
+        summary="Remove all highlights and callouts placed by highlight_element",
+        description=CLIENT,
+        handler="client",
+        requires_auth=False,
+    ),
+    Action(
+        name="show_toast",
+        method="LOCAL",
+        path="",
+        summary="Show a brief on-screen message (toast) to the user",
+        description=CLIENT,
+        handler="client",
+        requires_auth=False,
+        params=(
+            arg("text", "Message to display.", required=True),
+            arg("duration_ms", "How long to show it, in milliseconds (default 4000).", kind="integer"),
+        ),
+    ),
+    Action(
+        name="scroll_to_element",
+        method="LOCAL",
+        path="",
+        summary="Smoothly scroll an element into view",
+        description=CLIENT,
+        handler="client",
+        requires_auth=False,
+        params=(arg("selector", "CSS selector, or the element's exact visible text.", required=True),),
+    ),
+    Action(
+        name="navigate_to",
+        method="LOCAL",
+        path="",
+        summary="Send the user's browser to a URL",
+        description=(
+            CLIENT + " Use a same-origin path like /feed or /docs/index.html, or a full URL. The "
+            "page reloads; the user's Devii session and conversation persist and reconnect automatically."
+        ),
+        handler="client",
+        requires_auth=False,
+        params=(arg("url", "Path or URL to navigate to.", required=True),),
+    ),
+    Action(
+        name="reload_page",
+        method="LOCAL",
+        path="",
+        summary="Refresh the user's current page, e.g. after something changed",
+        description=CLIENT + " The Devii session and conversation persist and reconnect automatically.",
+        handler="client",
+        requires_auth=False,
+    ),
+)

devplacepy/services/devii/actions/cost_actions.py

@@ -0,0 +1,37 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from .spec import Action
+
+COST_ACTIONS: tuple[Action, ...] = (
+    Action(
+        name="usage_quota",
+        method="LOCAL",
+        path="",
+        summary="Report the current user's AI usage as a percentage of their rolling 24h quota",
+        description=(
+            "Returns ONLY the percentage of the rolling 24-hour AI quota the current user has "
+            "used, the number of turns taken today, and whether the limit is reached. It never "
+            "returns any cost, dollar amount, pricing, or spend figure. Use this for any "
+            "'how much have I used' or 'what percent of resources' question."
+        ),
+        handler="cost",
+        requires_auth=False,
+    ),
+    Action(
+        name="cost_stats",
+        method="LOCAL",
+        path="",
+        summary="Report token usage and full USD cost statistics for the current session (administrators only)",
+        description=(
+            "Administrators only. Returns this session's LLM token counts (prompt, completion, "
+            "total, cache hit/miss, reasoning), the cost in USD broken down by cache-hit input, "
+            "cache-miss input, and output, per-request averages, cache hit rate, and session "
+            "timing. Financial figures must never be shown to non-admin users."
+        ),
+        handler="cost",
+        requires_auth=True,
+        requires_admin=True,
+    ),
+)

devplacepy/services/devii/actions/dispatcher.py

@@ -0,0 +1,231 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any
+from urllib.parse import quote, urlencode
+
+from ..config import Settings
+from ..errors import (
+    AuthRequiredError,
+    DeviiError,
+    ToolInputError,
+    error_result,
+    unexpected_result,
+)
+from ..agentic.controller import AgenticController
+from ..agentic.state import record_mutation
+from ..avatar import AvatarController
+from ..chunks import ChunkController, get_store, serve_resource, wrap_if_large
+from ..cost import CostController
+from ..docs import DocsController
+from ..fetch import FetchController
+from ..http_client import PlatformClient
+from ..rsearch import RsearchController
+from ..tasks.controller import TaskController
+from ..text import format_response
+from .spec import Action, Catalog
+
+MUTATING_METHODS = ("POST", "DELETE", "PUT", "PATCH")
+
+logger = logging.getLogger("devii.dispatch")
+
+
+class Dispatcher:
+    def __init__(
+        self,
+        catalog: Catalog,
+        client: PlatformClient,
+        settings: Settings,
+        tasks: TaskController,
+        agentic: AgenticController,
+        avatar: AvatarController | None = None,
+        browser: Any = None,
+        is_admin: bool = False,
+        quota_provider: Any = None,
+    ) -> None:
+        self._actions = catalog.by_name()
+        self._client = client
+        self._settings = settings
+        self._tasks = tasks
+        self._agentic = agentic
+        self._avatar = avatar
+        self._browser = browser
+        self._is_admin = is_admin
+        self._fetch = FetchController(settings)
+        self._docs = DocsController(settings)
+        self._cost = CostController(quota_provider=quota_provider)
+        self._chunks = ChunkController(settings)
+        self._rsearch = RsearchController(settings)
+
+    async def dispatch(self, name: str, arguments: dict[str, Any]) -> str:
+        action = self._actions.get(name)
+        if action is None:
+            return error_result(ToolInputError(f"Unknown tool: {name}"))
+
+        logger.info("Dispatch %s args=%s", name, list(arguments))
+        try:
+            if action.requires_auth and not self._client.authenticated:
+                raise AuthRequiredError(
+                    "Not authenticated. Ask the user for credentials and call the login tool first.",
+                    tool=name,
+                )
+            if action.requires_admin and not self._is_admin:
+                raise AuthRequiredError(
+                    "This information is restricted to administrators.",
+                    tool=name,
+                )
+            resource_key = self._resource_key(action, arguments)
+            if resource_key:
+                cached = serve_resource(resource_key, self._settings.max_response_chars)
+                if cached is not None:
+                    logger.info("Resource cache hit for %s (%s)", name, resource_key)
+                    return cached
+            result = await self._run(action, arguments)
+            if action.handler == "chunks":
+                return result
+            return wrap_if_large(result, self._settings.max_response_chars, resource_key)
+        except DeviiError as exc:
+            logger.info("Dispatch %s failed: %s", name, exc.message)
+            return error_result(exc)
+        except Exception as exc:  # noqa: BLE001 - surfaced to the model as data
+            logger.exception("Dispatch %s crashed", name)
+            return unexpected_result(exc)
+
+    async def _run(self, action: Action, arguments: dict[str, Any]) -> str:
+        if action.handler == "status":
+            return json.dumps(
+                {
+                    "authenticated": self._client.authenticated,
+                    "user": self._client.username,
+                },
+                ensure_ascii=False,
+            )
+
+        if action.handler == "login":
+            result = await self._client.login(
+                email=self._require(arguments, "email"),
+                password=self._require(arguments, "password"),
+                remember_me=str(arguments.get("remember_me", "on")).lower() not in ("", "false", "off", "no"),
+            )
+            return json.dumps(result, ensure_ascii=False)
+
+        if action.handler == "logout":
+            return json.dumps(await self._client.logout(), ensure_ascii=False)
+
+        if action.handler == "task":
+            return await self._tasks.dispatch(action.name, arguments)
+
+        if action.handler == "agentic":
+            return await self._agentic.dispatch(action.name, arguments)
+
+        if action.handler == "fetch":
+            return await self._fetch.dispatch(action.name, arguments)
+
+        if action.handler == "docs":
+            return await self._docs.dispatch(action.name, arguments)
+
+        if action.handler == "cost":
+            return await self._cost.dispatch(action.name, arguments)
+
+        if action.handler == "chunks":
+            return await self._chunks.dispatch(action.name, arguments)
+
+        if action.handler == "rsearch":
+            return await self._rsearch.dispatch(action.name, arguments)
+
+        if action.handler == "avatar":
+            if self._avatar is None:
+                return error_result(
+                    ToolInputError(
+                        "No avatar is attached; devii's on-screen actions need the web interface."
+                    )
+                )
+            return await self._avatar.dispatch(action.name, arguments)
+
+        if action.handler == "client":
+            if self._browser is None:
+                return error_result(
+                    ToolInputError(
+                        "No browser is attached; client-side actions need the web interface."
+                    )
+                )
+            return await self._browser.dispatch(action.name, arguments)
+
+        return await self._run_http(action, arguments)
+
+    def _build_request(
+        self, action: Action, arguments: dict[str, Any]
+    ) -> tuple[str, dict[str, Any], dict[str, Any], tuple[str, str] | None]:
+        url_path = action.path
+        params: dict[str, Any] = {}
+        data: dict[str, Any] = {}
+        file_field: tuple[str, str] | None = None
+
+        for param in action.params:
+            if param.name not in arguments or arguments[param.name] is None:
+                if param.required:
+                    raise ToolInputError(
+                        f"Missing required parameter '{param.name}' for {action.name}.")
+                continue
+            value = arguments[param.name]
+            if param.location == "path":
+                url_path = url_path.replace("{" + param.name + "}", quote(str(value), safe=""))
+            elif param.location == "query":
+                params[param.name] = value
+            elif param.location == "body":
+                data[param.name] = value
+            elif param.location == "file":
+                file_field = (param.name, str(value))
+
+        if action.freeform_body:
+            extra = arguments.get("form_fields") or {}
+            if isinstance(extra, dict):
+                data.update({str(k): v for k, v in extra.items()})
+
+        return url_path, params, data, file_field
+
+    def _resource_key(self, action: Action, arguments: dict[str, Any]) -> str | None:
+        try:
+            if action.handler == "fetch" and action.name == "fetch_url":
+                url = str(arguments.get("url", "")).strip()
+                if not url:
+                    return None
+                if "://" not in url:
+                    url = "https://" + url
+                return f"fetch:{url}"
+            if action.handler == "http" and action.method == "GET":
+                url_path, params, _, _ = self._build_request(action, arguments)
+                query = urlencode(sorted((str(k), str(v)) for k, v in params.items()))
+                return f"http:GET {url_path}?{query}"
+        except ToolInputError:
+            return None
+        return None
+
+    async def _run_http(self, action: Action, arguments: dict[str, Any]) -> str:
+        url_path, params, data, file_field = self._build_request(action, arguments)
+
+        headers = {"X-Requested-With": "fetch"} if action.ajax else None
+        response = await self._client.call(
+            method=action.method,
+            path=url_path,
+            params=params or None,
+            data=data or None,
+            file_field=file_field,
+            headers=headers,
+        )
+        if action.method in MUTATING_METHODS:
+            record_mutation(action.name)
+            store = get_store()
+            if store is not None:
+                store.invalidate_resources(prefix="http:")
+        return format_response(response)
+
+    @staticmethod
+    def _require(arguments: dict[str, Any], key: str) -> str:
+        value = arguments.get(key)
+        if value is None or str(value).strip() == "":
+            raise ToolInputError(f"Missing required field '{key}'.")
+        return str(value)

devplacepy/services/devii/actions/docs_actions.py

@@ -0,0 +1,35 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from .spec import Action, Param
+
+DOCS_ACTIONS: tuple[Action, ...] = (
+    Action(
+        name="search_docs",
+        method="LOCAL",
+        path="",
+        summary="Full-text search the DevPlace API documentation",
+        description=(
+            "Searches the platform's developer documentation and returns the most relevant "
+            "sections (title and content). Use it to confirm how an endpoint, parameter, or "
+            "feature works before acting."
+        ),
+        handler="docs",
+        requires_auth=False,
+        params=(
+            Param(
+                name="query",
+                location="body",
+                description="What to look for in the documentation.",
+                required=True,
+            ),
+            Param(
+                name="max_results",
+                location="body",
+                description="Maximum number of documentation sections to return (1-10).",
+                type="integer",
+            ),
+        ),
+    ),
+)

devplacepy/services/devii/actions/fetch_actions.py

@@ -0,0 +1,36 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from .spec import Action, Param
+
+FETCH_ACTIONS: tuple[Action, ...] = (
+    Action(
+        name="fetch_url",
+        method="LOCAL",
+        path="",
+        summary="Fetch a web page and return its readable text content",
+        description=(
+            "Retrieves any http(s) URL as a real browser would (stealth headers) and returns "
+            "the page title and readable text with links preserved, safely truncated to fit "
+            "the context window. Use it to read external pages, then summarize or create posts, "
+            "gists, or articles from the content. Private and loopback addresses are refused."
+        ),
+        handler="fetch",
+        requires_auth=False,
+        params=(
+            Param(
+                name="url",
+                location="body",
+                description="The page URL to fetch (https is assumed if no scheme is given).",
+                required=True,
+            ),
+            Param(
+                name="max_chars",
+                location="body",
+                description="Optional cap on returned characters; clamped to a context-safe maximum.",
+                type="integer",
+            ),
+        ),
+    ),
+)

devplacepy/services/devii/actions/rsearch_actions.py

@@ -0,0 +1,85 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from .spec import Action, Param
+
+REMOTE_NOTE = (
+    "This is an EXTERNAL public service, not this DevPlace platform. Only call it when the user "
+    "explicitly asks to search the web or an outside source; prefer platform tools otherwise."
+)
+
+RSEARCH_ACTIONS: tuple[Action, ...] = (
+    Action(
+        name="rsearch",
+        method="LOCAL",
+        path="",
+        summary="Search the public web (and images) via the external rsearch aggregator",
+        description=(
+            "Queries multiple independent web search providers and returns ranked results "
+            "(title, URL, description, source). Set content=true to also fetch each page's "
+            "readable text, deep=true for a deeper research pass, and type=images for image "
+            "results. " + REMOTE_NOTE
+        ),
+        handler="rsearch",
+        requires_auth=False,
+        params=(
+            Param(name="query", location="body", description="The web search query.", required=True),
+            Param(name="count", location="body", description="Number of results (1-100, default 10).", type="integer"),
+            Param(name="content", location="body", description="Fetch full page content for each result.", type="boolean"),
+            Param(name="deep", location="body", description="Run a deeper research pass.", type="boolean"),
+            Param(name="type", location="body", description="Result type: 'web' (default) or 'images'."),
+        ),
+    ),
+    Action(
+        name="rsearch_answer",
+        method="LOCAL",
+        path="",
+        summary="Get an AI answer to a question, grounded in fresh public web search",
+        description=(
+            "Sends the question to an external AI that autonomously searches the public web and "
+            "returns a written answer plus the source links it used. Use for current, real-world "
+            "questions the platform cannot answer. " + REMOTE_NOTE
+        ),
+        handler="rsearch",
+        requires_auth=False,
+        params=(
+            Param(name="query", location="body", description="The question or prompt to answer.", required=True),
+            Param(name="content", location="body", description="Let the AI read full page content while answering.", type="boolean"),
+            Param(name="count", location="body", description="Max sources to consider (1-100, default 10).", type="integer"),
+        ),
+    ),
+    Action(
+        name="rsearch_chat",
+        method="LOCAL",
+        path="",
+        summary="Send a prompt to the external rsearch AI for a direct answer (no web search)",
+        description=(
+            "Direct chat completion from the external rsearch AI model, with no web search or "
+            "platform context. Set json=true to force a JSON-only answer, or pass a system message "
+            "to steer the persona. " + REMOTE_NOTE
+        ),
+        handler="rsearch",
+        requires_auth=False,
+        params=(
+            Param(name="prompt", location="body", description="The prompt to send.", required=True),
+            Param(name="json", location="body", description="Force a valid-JSON-only response.", type="boolean"),
+            Param(name="system", location="body", description="Optional system message to steer the answer."),
+        ),
+    ),
+    Action(
+        name="rsearch_describe_image",
+        method="LOCAL",
+        path="",
+        summary="Describe a public image URL using the external rsearch vision model",
+        description=(
+            "Sends an image URL to the external rsearch vision model and returns a written "
+            "description. " + REMOTE_NOTE
+        ),
+        handler="rsearch",
+        requires_auth=False,
+        params=(
+            Param(name="url", location="body", description="Public URL of the image to describe.", required=True),
+        ),
+    ),
+)

devplacepy/services/devii/actions/spec.py

@@ -0,0 +1,92 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+ParamLocation = Literal["path", "query", "body", "file"]
+
+
+@dataclass(frozen=True)
+class Param:
+    name: str
+    location: ParamLocation
+    description: str
+    required: bool = False
+    type: str = "string"
+
+
+@dataclass(frozen=True)
+class Action:
+    name: str
+    method: str
+    path: str
+    summary: str
+    description: str = ""
+    params: tuple[Param, ...] = ()
+    requires_auth: bool = True
+    requires_admin: bool = False
+    handler: Literal[
+        "http", "login", "logout", "status", "task", "agentic", "avatar", "client", "fetch",
+        "docs", "cost", "chunks", "rsearch"
+    ] = "http"
+    freeform_body: bool = False
+    ajax: bool = False
+
+    def tool_schema(self) -> dict[str, Any]:
+        properties: dict[str, Any] = {}
+        required: list[str] = []
+        for param in self.params:
+            if param.type == "array":
+                properties[param.name] = {
+                    "type": "array",
+                    "items": {"type": "object"},
+                    "description": param.description,
+                }
+            else:
+                properties[param.name] = {
+                    "type": param.type,
+                    "description": param.description,
+                }
+            if param.required:
+                required.append(param.name)
+        if self.freeform_body:
+            properties["form_fields"] = {
+                "type": "object",
+                "description": "Additional form fields as key/value string pairs.",
+                "additionalProperties": {"type": "string"},
+            }
+        text = self.summary if not self.description else f"{self.summary}. {self.description}"
+        return {
+            "type": "function",
+            "function": {
+                "name": self.name,
+                "description": text,
+                "parameters": {
+                    "type": "object",
+                    "properties": properties,
+                    "required": required,
+                    "additionalProperties": False,
+                },
+            },
+        }
+
+
+@dataclass(frozen=True)
+class Catalog:
+    actions: tuple[Action, ...] = field(default_factory=tuple)
+
+    def by_name(self) -> dict[str, Action]:
+        return {action.name: action for action in self.actions}
+
+    def tool_schemas(self) -> list[dict[str, Any]]:
+        return [action.tool_schema() for action in self.actions]
+
+    def tool_schemas_for(self, authenticated: bool, is_admin: bool = False) -> list[dict[str, Any]]:
+        return [
+            action.tool_schema()
+            for action in self.actions
+            if (authenticated or not action.requires_auth)
+            and (is_admin or not action.requires_admin)
+        ]

devplacepy/services/devii/agent.py

@@ -0,0 +1,166 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any, Optional
+
+from .actions import Dispatcher
+from .agentic import AgentState, LessonStore, react_loop
+from .agentic.loop import TraceCallback
+from .config import Settings
+from .llm import LLMClient
+
+logger = logging.getLogger("devii.agent")
+
+SYSTEM_PROMPT = (
+    "You are Devii, an agentic assistant that manages a user's account on this "
+    "DevPlace developer social network entirely through the provided tools, running a "
+    "ReAct loop with structured planning, reflection, a verification gate, and persistent "
+    "self-learning memory.\n\n"
+    "OPERATING PROTOCOL\n"
+    "1. PLAN FIRST. When a request needs any tool use, your very first tool call must be "
+    "plan() with goal, ordered steps, success_criteria, and a confidence estimate.\n"
+    "2. PERMISSIONS. You only have the tools your current access allows; account actions are "
+    "present only when the user is already signed in. Never pressure the user to log in and "
+    "never ask for credentials unprompted - just work with the tools you have. If the user "
+    "explicitly asks for something that needs an account you cannot reach, note briefly that it "
+    "requires signing in, then continue with whatever you can do. Never invent credentials.\n"
+    "3. RECALL AND ACT. Use recall() to consult lessons from past requests when unsure. "
+    "Investigate with read actions before changing anything. Independent reads may be issued "
+    "together in one turn; they run in parallel.\n"
+    "4. VERIFY. After mutating actions (create, edit, delete, vote, react, follow, admin "
+    "changes), confirm the result and call verify() before your final answer.\n"
+    "5. REFLECT. After any tool error, the harness asks you to call reflect(); diagnose the "
+    "cause and record the lesson - a reusable rule or procedure - rather than blindly retrying. "
+    "Reflect at the end of non-trivial tasks too, so you learn. Your lesson memory is PRIVATE to "
+    "this account (or, for a guest, this web session only) and is never shared with other users. "
+    "Never store credentials, passwords, API keys, tokens, or other secrets in a lesson. When the "
+    "user asks you to forget something, call forget_lessons (optionally with a query).\n"
+    "6. DELEGATE. For a self-contained sub-task, call delegate() to run it in an isolated "
+    "sub-agent that returns a concise result.\n\n"
+    "Tool results are JSON; an 'error' field means failure - read the message and recover. "
+    "If a result has 'truncated': true it includes chunk_id, remaining_chars, and next_offset; "
+    "call read_more with that chunk_id and offset to page through the rest until remaining_chars "
+    "is 0 whenever you need the full content. Use read_more to continue - never re-run the "
+    "original tool to get more of the same resource; its full content is already cached. "
+    "Platform calls return structured JSON: reads give objects with fields like uid, slug, "
+    "next_cursor, and nested authors/comments; write actions return {ok, redirect, data} where "
+    "data has the created resource's uid/slug/url. Always reuse those exact slugs/uids/urls for "
+    "follow-up calls instead of constructing them. "
+    "In the web terminal you can act on the user's own screen: get_page_context tells you where "
+    "they are and what they see; run_js executes JavaScript in their browser and returns a value; "
+    "highlight_element, show_toast, scroll_to_element and clear_highlights let you guide them with "
+    "live, on-screen tutorials; navigate_to and reload_page move or refresh their page (their "
+    "session and this conversation persist and reconnect automatically). Read the page context "
+    "before guiding, prefer the dedicated tools over raw run_js, and clear highlights when done. "
+    "Confirm destructive actions with the user first. Schedule autonomous work with "
+    "create_task and related tools. All times are UTC.\n\n"
+    "AGGREGATES AND LARGE DATA\n"
+    "For any count, total, or 'how many' / 'how active' question, call site_analytics - it returns "
+    "member totals, active users over 24h/7d/30d, signups, content totals, and top authors in a "
+    "single call. Never page through admin_list_users or any list_* endpoint to count records: "
+    "fanning out paginated calls wastes the context window and is forbidden. When the user actually "
+    "wants items (not a count), page with the cursor and stop as soon as you have enough.\n\n"
+    "NEVER GUESS - CHECK THE DOCS FIRST\n"
+    "When you are unsure about a route, endpoint, parameter, capability, or whether a page or "
+    "feature exists, call search_docs first (the documentation lists every route and endpoint), and "
+    "use get_page_context to see where the user is. Never invent a URL, never probe by "
+    "trial-and-error, and never tell the user that a page or capability does not exist without "
+    "confirming against the docs. If a guess returns a 404, that means you guessed - search the docs "
+    "instead of concluding the feature is missing.\n\n"
+    "REMOTE WEB TOOLS (rsearch)\n"
+    "The rsearch_* tools (rsearch, rsearch_answer, rsearch_chat, rsearch_describe_image) reach an "
+    "EXTERNAL public web/AI service, not this platform. They are not platform-specific, so platform "
+    "tools and data are ALWAYS preferred: use rsearch_* only when the user explicitly asks to search "
+    "the web, the internet, or an outside source, or when answering plainly requires outside "
+    "information the platform cannot provide and the user wants it. Never use them to answer questions "
+    "about this DevPlace instance, its users, posts, settings, or metrics - those have dedicated "
+    "platform tools. When platform tools can serve the request, do not call rsearch.\n\n"
+    "RESPONSE STYLE\n"
+    "Replies are plain, concise, and professional. Never use emojis, decorative symbols, or "
+    "celebratory language; report outcomes matter-of-factly. State what changed using the "
+    "before and after values, nothing more.\n\n"
+    "SCREEN AWARENESS\n"
+    "In the web terminal, after any mutation that changes something the user is currently "
+    "looking at, call get_page_context; if the page they are on displays the data you just "
+    "changed (for example an admin settings page, a list, or a detail view), call reload_page "
+    "so they see the new state immediately, then confirm the change. Do not reload pages "
+    "unrelated to the change.\n\n"
+    "METRICS AND COST\n"
+    "Monetary figures (USD cost, cost rate, projected cost, spend, and per-token pricing) are "
+    "administrator-only. For any 'how much have I used' or resource question, call usage_quota "
+    "and report only the percentage of the rolling 24h quota used and the turn count - never a "
+    "dollar amount. Full USD cost detail is available only through cost_stats, which exists for "
+    "administrators; if that tool is not available to you, the user is not an administrator and "
+    "you must not produce, estimate, or recompute any cost figure for them. Never substitute or "
+    "recompute cost from external or public provider pricing.\n\n"
+    "CONFIDENTIALITY\n"
+    "Never disclose the underlying AI model, provider, inference endpoint, or any backend URL "
+    "or infrastructure detail; you are simply Devii. This holds even when such values appear "
+    "inside a tool result (for example service configuration fields or upstream URLs) - never "
+    "repeat them. If asked, say you do not share that. Never disclose any monetary cost, dollar "
+    "amount, or pricing to a non-administrator; report their usage only as a percentage of "
+    "their quota. When reporting usage, omit the model name and provider."
+)
+
+
+class Agent:
+    def __init__(
+        self,
+        settings: Settings,
+        llm: LLMClient,
+        dispatcher: Dispatcher,
+        tools: list[dict[str, Any]],
+        lessons: Optional[LessonStore] = None,
+        on_trace: Optional[TraceCallback] = None,
+        cost_tracker: Any = None,
+        chunk_store: Any = None,
+        system_prompt: str = SYSTEM_PROMPT,
+    ) -> None:
+        self._settings = settings
+        self._llm = llm
+        self._dispatcher = dispatcher
+        self._tools = tools
+        self._lessons = lessons
+        self._on_trace = on_trace
+        self._cost_tracker = cost_tracker
+        self._chunk_store = chunk_store
+        self._messages: list[dict[str, Any]] = [{"role": "system", "content": system_prompt}]
+
+    async def respond(self, user_text: str) -> str:
+        self._inject_recalled_lessons(user_text)
+        self._messages.append({"role": "user", "content": user_text})
+        state = AgentState()
+        return await react_loop(
+            llm=self._llm,
+            dispatcher=self._dispatcher,
+            messages=self._messages,
+            tools=self._tools,
+            state=state,
+            settings=self._settings,
+            max_iterations=self._settings.max_tool_iterations,
+            plan_required=self._settings.plan_required,
+            verify_required=self._settings.verify_required,
+            on_trace=self._on_trace,
+            cost_tracker=self._cost_tracker,
+            chunk_store=self._chunk_store,
+        )
+
+    def _inject_recalled_lessons(self, user_text: str) -> None:
+        if self._lessons is None or self._lessons.count() == 0:
+            return
+        hits = self._lessons.search(user_text, k=self._settings.recall_top_k)
+        if not hits:
+            return
+        lines = [
+            f"- {hit['conclusion']} -> {hit['next_action']}"
+            for hit in hits
+            if hit.get("conclusion") and hit.get("next_action")
+        ]
+        if not lines:
+            return
+        note = "[memory] Relevant lessons from past sessions:\n" + "\n".join(lines)
+        self._messages.append({"role": "user", "content": note})
+        logger.debug("Injected %d recalled lessons", len(lines))

devplacepy/services/devii/agentic/__init__.py

@@ -0,0 +1,9 @@
+# retoor <retoor@molodetz.nl>
+
+from .actions import AGENTIC_ACTIONS
+from .controller import AgenticController
+from .lessons import LessonStore
+from .loop import react_loop
+from .state import AgentState
+
+__all__ = ["AGENTIC_ACTIONS", "AgenticController", "LessonStore", "react_loop", "AgentState"]

devplacepy/services/devii/agentic/actions.py

@@ -0,0 +1,101 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from ..actions.spec import Action, Param
+
+
+def arg(name: str, description: str, required: bool = False, kind: str = "string") -> Param:
+    return Param(name=name, location="body", description=description, required=required, type=kind)
+
+
+AGENTIC_ACTIONS: tuple[Action, ...] = (
+    Action(
+        name="plan",
+        method="LOCAL",
+        path="",
+        summary="Record the structured execution plan for the current request",
+        description="Must be the very first tool call when a request requires any tool use.",
+        handler="agentic",
+        requires_auth=False,
+        params=(
+            arg("goal", "One-line restatement of the user's goal.", required=True),
+            arg("steps", "Ordered list of step objects, each with id, action, depends_on.", required=True, kind="array"),
+            arg("success_criteria", "Concrete criteria for declaring the task complete.", required=True),
+            arg("confidence", "Self-estimate of plan correctness from 0.0 to 1.0.", kind="number"),
+        ),
+    ),
+    Action(
+        name="reflect",
+        method="LOCAL",
+        path="",
+        summary="Record a reflection and persist it as a reusable lesson for future requests",
+        description="Call after any tool error and at the end of a non-trivial task to learn from it.",
+        handler="agentic",
+        requires_auth=False,
+        params=(
+            arg("observation", "What was observed: the failure mode or notable outcome.", required=True),
+            arg("conclusion", "Diagnosis or interpretation of the observation.", required=True),
+            arg("next_action", "The chosen next step or the rule to apply in future.", required=True),
+            arg("tags", "Optional comma-separated keywords to aid later recall."),
+        ),
+    ),
+    Action(
+        name="recall",
+        method="LOCAL",
+        path="",
+        summary="Retrieve relevant lessons learned from past requests",
+        description="Search the persistent lesson memory for guidance before acting on unfamiliar work.",
+        handler="agentic",
+        requires_auth=False,
+        params=(
+            arg("query", "Natural-language or keyword query describing the current situation.", required=True),
+            arg("k", "Number of lessons to return.", kind="integer"),
+        ),
+    ),
+    Action(
+        name="forget_lessons",
+        method="LOCAL",
+        path="",
+        summary="Permanently delete learned lessons from your private memory",
+        description=(
+            "Removes lessons you previously stored with reflect(). With a query, only matching "
+            "lessons are deleted; with no query, all of them are. Use this when the user asks you "
+            "to forget something. Your lesson memory is private to this account or guest session."
+        ),
+        handler="agentic",
+        requires_auth=False,
+        params=(
+            arg("query", "Optional topic/keywords; only matching lessons are forgotten. Omit to forget everything."),
+        ),
+    ),
+    Action(
+        name="verify",
+        method="LOCAL",
+        path="",
+        summary="Confirm that a change took effect, satisfying the verification gate",
+        description=(
+            "After mutating actions (create, edit, delete, vote, follow, admin changes), confirm "
+            "the result, then call this with a summary of what you confirmed."
+        ),
+        handler="agentic",
+        requires_auth=False,
+        params=(
+            arg("summary", "What was confirmed and how.", required=True),
+            arg("confirmed", "Whether the change was confirmed successful.", kind="boolean"),
+        ),
+    ),
+    Action(
+        name="delegate",
+        method="LOCAL",
+        path="",
+        summary="Spawn a focused sub-agent to complete a self-contained sub-task in isolation",
+        description="The sub-agent runs its own plan/act/verify loop and returns a concise result.",
+        handler="agentic",
+        requires_auth=False,
+        params=(
+            arg("task", "Clear, scoped task description for the sub-agent.", required=True),
+            arg("allowed_tools", "Optional list of tool names the sub-agent may use.", kind="array"),
+        ),
+    ),
+)

devplacepy/services/devii/agentic/compaction.py

@@ -0,0 +1,59 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any
+
+logger = logging.getLogger("devii.agentic.compaction")
+
+SUMMARY_INPUT_CAP = 600_000
+SUMMARY_PROMPT = (
+    "Summarize the following assistant conversation segment as a concise factual log of "
+    "actions taken, tools called, entities created or changed, conclusions reached, and "
+    "outstanding work. Keep identifiers, slugs, uids, and decisions verbatim. Maximum 800 words.\n\n"
+    "---\n\n"
+)
+
+
+def context_size(messages: list[dict[str, Any]]) -> int:
+    return len(json.dumps(messages, default=str))
+
+
+def find_compaction_split(messages: list[dict[str, Any]], keep_tail: int) -> int:
+    if len(messages) <= keep_tail:
+        return 1
+    candidate = len(messages) - keep_tail
+    while candidate > 1:
+        if messages[candidate].get("role") == "user":
+            return candidate
+        candidate -= 1
+    return 1
+
+
+async def compact_messages(llm: Any, messages: list[dict[str, Any]], keep_tail: int) -> list[dict[str, Any]]:
+    if len(messages) < keep_tail + 3:
+        return messages
+    split = find_compaction_split(messages, keep_tail)
+    if split <= 1:
+        return messages
+    system_message = messages[0]
+    middle = messages[1:split]
+    tail = messages[split:]
+    if not middle:
+        return messages
+
+    segment = json.dumps(middle, default=str)[:SUMMARY_INPUT_CAP]
+    try:
+        summary = await llm.summarize(SUMMARY_PROMPT + segment)
+    except Exception:  # noqa: BLE001 - compaction must never break the loop
+        logger.exception("Compaction summary failed; keeping full context")
+        return messages
+
+    logger.info("Compacted %d messages into a summary", len(middle))
+    return [
+        system_message,
+        {"role": "assistant", "content": f"[compacted earlier turns]\n\n{summary}"},
+        *tail,
+    ]

devplacepy/services/devii/agentic/controller.py

@@ -0,0 +1,195 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any, Optional
+
+from ..config import Settings
+from ..errors import ToolInputError
+from .lessons import LessonStore
+from .loop import TraceCallback, react_loop
+from .state import AgentState, get_state
+
+logger = logging.getLogger("devii.agentic.controller")
+
+SUB_AGENT_SYSTEM_PROMPT = (
+    "You are a focused sub-agent spawned to complete a single scoped task on this "
+    "DevPlace platform. Begin with a plan() call. Investigate, then act with the most specific "
+    "tools available. If you change anything, confirm it and call verify() before returning. "
+    "Return a concise factual result string under 1500 characters."
+)
+NO_DELEGATE = "delegate"
+
+
+class AgenticController:
+    def __init__(self, lessons: LessonStore, settings: Settings) -> None:
+        self._lessons = lessons
+        self._settings = settings
+        self._llm: Any = None
+        self._dispatcher: Any = None
+        self._tools: list[dict[str, Any]] = []
+        self._on_trace: Optional[TraceCallback] = None
+        self._cost_tracker: Any = None
+        self._chunk_store: Any = None
+
+    def bind(
+        self,
+        llm: Any,
+        dispatcher: Any,
+        tools: list[dict[str, Any]],
+        on_trace: Optional[TraceCallback],
+        cost_tracker: Any = None,
+        chunk_store: Any = None,
+    ) -> None:
+        self._llm = llm
+        self._dispatcher = dispatcher
+        self._tools = tools
+        self._on_trace = on_trace
+        self._cost_tracker = cost_tracker
+        self._chunk_store = chunk_store
+
+    async def dispatch(self, name: str, arguments: dict[str, Any]) -> str:
+        handlers = {
+            "plan": self._plan,
+            "reflect": self._reflect,
+            "recall": self._recall,
+            "forget_lessons": self._forget,
+            "verify": self._verify,
+            "delegate": self._delegate,
+        }
+        handler = handlers.get(name)
+        if handler is None:
+            raise ToolInputError(f"Unknown agentic tool: {name}")
+        return await handler(arguments)
+
+    async def _plan(self, arguments: dict[str, Any]) -> str:
+        goal = str(arguments.get("goal", "")).strip()
+        steps = arguments.get("steps") or []
+        if not goal:
+            raise ToolInputError("plan requires a goal.")
+        if not isinstance(steps, list) or not steps:
+            raise ToolInputError("plan requires a non-empty steps list.")
+        confidence = float(arguments.get("confidence", 0.8) or 0.8)
+        state = get_state()
+        if state is not None:
+            state.plan = {
+                "goal": goal,
+                "steps": steps,
+                "success_criteria": arguments.get("success_criteria", ""),
+                "confidence": confidence,
+            }
+        advice = ""
+        if confidence < 0.6:
+            advice = "Confidence is below 0.6 - gather more context or recall() past lessons before executing."
+        return json.dumps(
+            {"status": "success", "plan_recorded": True, "step_count": len(steps), "advice": advice},
+            ensure_ascii=False,
+        )
+
+    async def _reflect(self, arguments: dict[str, Any]) -> str:
+        observation = str(arguments.get("observation", "")).strip()
+        conclusion = str(arguments.get("conclusion", "")).strip()
+        next_action = str(arguments.get("next_action", "")).strip()
+        if not (observation and conclusion and next_action):
+            raise ToolInputError("reflect requires observation, conclusion, and next_action.")
+        tags = str(arguments.get("tags", "") or "").strip()
+        record = self._lessons.add(observation, conclusion, next_action, tags)
+        state = get_state()
+        if state is not None:
+            state.reflections.append(
+                {"observation": observation, "conclusion": conclusion, "next_action": next_action}
+            )
+        return json.dumps(
+            {
+                "status": "success",
+                "lesson_uid": record["uid"],
+                "total_lessons": self._lessons.count(),
+            },
+            ensure_ascii=False,
+        )
+
+    async def _recall(self, arguments: dict[str, Any]) -> str:
+        query = str(arguments.get("query", "")).strip()
+        if not query:
+            raise ToolInputError("recall requires a query.")
+        k = int(arguments.get("k", self._settings.recall_top_k) or self._settings.recall_top_k)
+        hits = self._lessons.search(query, k=k)
+        return json.dumps({"status": "success", "count": len(hits), "lessons": hits}, ensure_ascii=False)
+
+    async def _forget(self, arguments: dict[str, Any]) -> str:
+        query = str(arguments.get("query", "") or "").strip()
+        if query:
+            removed = 0
+            for hit in self._lessons.search(query, k=50):
+                uid = hit.get("uid")
+                if uid and self._lessons.delete(uid):
+                    removed += 1
+        else:
+            removed = self._lessons.clear()
+        return json.dumps(
+            {"status": "success", "forgotten": removed, "remaining": self._lessons.count()},
+            ensure_ascii=False,
+        )
+
+    async def _verify(self, arguments: dict[str, Any]) -> str:
+        summary = str(arguments.get("summary", "")).strip()
+        if not summary:
+            raise ToolInputError("verify requires a summary of what was confirmed.")
+        confirmed = arguments.get("confirmed", True)
+        if isinstance(confirmed, str):
+            confirmed = confirmed.strip().lower() not in ("", "false", "no", "0")
+        state = get_state()
+        if state is not None and confirmed:
+            state.verified = True
+        return json.dumps(
+            {"status": "success", "verified": bool(confirmed), "summary": summary}, ensure_ascii=False
+        )
+
+    async def _delegate(self, arguments: dict[str, Any]) -> str:
+        task = str(arguments.get("task", "")).strip()
+        if not task:
+            raise ToolInputError("delegate requires a task description.")
+        if self._llm is None or self._dispatcher is None:
+            raise ToolInputError("Delegation is not available in this context.")
+        allowed = arguments.get("allowed_tools")
+        if allowed:
+            allowed_set = {str(name) for name in allowed}
+            tools = [
+                tool
+                for tool in self._tools
+                if tool["function"]["name"] in allowed_set and tool["function"]["name"] != NO_DELEGATE
+            ]
+        else:
+            tools = [tool for tool in self._tools if tool["function"]["name"] != NO_DELEGATE]
+
+        messages = [
+            {"role": "system", "content": SUB_AGENT_SYSTEM_PROMPT},
+            {"role": "user", "content": task},
+        ]
+        sub_state = AgentState()
+        result = await react_loop(
+            llm=self._llm,
+            dispatcher=self._dispatcher,
+            messages=messages,
+            tools=tools,
+            state=sub_state,
+            settings=self._settings,
+            max_iterations=self._settings.delegate_max_iterations,
+            plan_required=self._settings.plan_required,
+            verify_required=self._settings.verify_required,
+            on_trace=self._on_trace,
+            cost_tracker=self._cost_tracker,
+            chunk_store=self._chunk_store,
+        )
+        return json.dumps(
+            {
+                "status": "success",
+                "iterations": sub_state.iteration,
+                "verified": sub_state.verified,
+                "reflections": len(sub_state.reflections),
+                "result": result,
+            },
+            ensure_ascii=False,
+        )

devplacepy/services/devii/agentic/lessons.py

@@ -0,0 +1,161 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import collections
+import logging
+import math
+import re
+import uuid
+from typing import Any
+
+from ..tasks.schedule import now_utc, to_iso
+
+logger = logging.getLogger("devii.agentic.lessons")
+
+TABLE = "devii_lessons"
+TOKEN_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*|\d+")
+BM25_K1 = 1.5
+BM25_B = 0.75
+
+
+def tokenize(text: str) -> list[str]:
+    tokens = TOKEN_RE.findall((text or "").lower())
+    extra: list[str] = []
+    for token in tokens:
+        extra.extend(re.findall(r"[a-z]+", token))
+    return list(dict.fromkeys(tokens + extra))
+
+
+class LessonStore:
+    def __init__(self, db: Any, owner_kind: str, owner_id: str) -> None:
+        self._db = db
+        self._owner_kind = owner_kind
+        self._owner_id = owner_id
+        self._dirty = True
+        self._docs: list[dict[str, Any]] = []
+        self._tf: list[collections.Counter] = []
+        self._dl: list[int] = []
+        self._idf: dict[str, float] = {}
+        self._avgdl = 0.0
+        self._n = 0
+        self._ensure_indexes()
+
+    def _ensure_indexes(self) -> None:
+        if TABLE not in self._db.tables:
+            return
+        self._db[TABLE].create_index(["owner_kind", "owner_id"])
+
+    @property
+    def _table(self) -> Any:
+        return self._db[TABLE]
+
+    @property
+    def _scope(self) -> dict[str, str]:
+        return {"owner_kind": self._owner_kind, "owner_id": self._owner_id}
+
+    def count(self) -> int:
+        if TABLE not in self._db.tables:
+            return 0
+        return self._table.count(**self._scope)
+
+    def add(self, observation: str, conclusion: str, next_action: str, tags: str = "") -> dict[str, Any]:
+        record = {
+            "uid": uuid.uuid4().hex,
+            "observation": observation,
+            "conclusion": conclusion,
+            "next_action": next_action,
+            "tags": tags,
+            "created_at": to_iso(now_utc()),
+            "hits": 0,
+            **self._scope,
+        }
+        self._table.insert(record)
+        self._dirty = True
+        logger.info("Lesson stored owner=%s/%s tags=%s", self._owner_kind, self._owner_id, tags)
+        return record
+
+    def all(self) -> list[dict[str, Any]]:
+        if TABLE not in self._db.tables:
+            return []
+        return list(self._table.find(**self._scope))
+
+    def delete(self, uid: str) -> bool:
+        if TABLE not in self._db.tables:
+            return False
+        deleted = self._table.delete(uid=uid, **self._scope)
+        self._dirty = True
+        return bool(deleted)
+
+    def clear(self) -> int:
+        n = self.count()
+        if TABLE in self._db.tables:
+            self._table.delete(**self._scope)
+        self._dirty = True
+        logger.info("Cleared %d lesson(s) for owner=%s/%s", n, self._owner_kind, self._owner_id)
+        return n
+
+    def _rebuild(self) -> None:
+        rows = self.all()
+        df: collections.Counter = collections.Counter()
+        docs: list[dict[str, Any]] = []
+        tf_list: list[collections.Counter] = []
+        dl: list[int] = []
+        for row in rows:
+            text = " ".join(
+                str(row.get(field) or "")
+                for field in ("observation", "conclusion", "next_action", "tags")
+            )
+            tokens = tokenize(text)
+            if not tokens:
+                continue
+            tf = collections.Counter(tokens)
+            for term in tf:
+                df[term] += 1
+            docs.append(row)
+            tf_list.append(tf)
+            dl.append(len(tokens))
+        self._docs = docs
+        self._tf = tf_list
+        self._dl = dl
+        self._n = len(docs)
+        self._avgdl = sum(dl) / max(self._n, 1)
+        self._idf = {
+            term: math.log((self._n - freq + 0.5) / (freq + 0.5) + 1)
+            for term, freq in df.items()
+        }
+        self._dirty = False
+
+    def search(self, query: str, k: int = 3) -> list[dict[str, Any]]:
+        if self._dirty:
+            self._rebuild()
+        tokens = tokenize(query)
+        if not tokens or not self._docs:
+            return []
+        scored: list[tuple[float, int]] = []
+        for index, tf in enumerate(self._tf):
+            score = 0.0
+            for token in tokens:
+                freq = tf.get(token, 0)
+                if freq == 0:
+                    continue
+                idf = self._idf.get(token, 0.0)
+                norm = 1 - BM25_B + BM25_B * (self._dl[index] / max(self._avgdl, 1))
+                score += idf * (freq * (BM25_K1 + 1)) / (freq + BM25_K1 * norm)
+            if score > 0:
+                scored.append((score, index))
+        scored.sort(reverse=True)
+        results: list[dict[str, Any]] = []
+        for score, index in scored[:k]:
+            row = self._docs[index]
+            results.append(
+                {
+                    "uid": row.get("uid"),
+                    "observation": row.get("observation"),
+                    "conclusion": row.get("conclusion"),
+                    "next_action": row.get("next_action"),
+                    "tags": row.get("tags"),
+                    "score": round(score, 3),
+                }
+            )
+        return results

devplacepy/services/devii/agentic/loop.py

@@ -0,0 +1,242 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+from typing import Any, Callable, Optional
+
+from ..config import Settings
+from ..errors import LLMError
+from ..chunks import reset_store, set_store
+from ..cost import reset_tracker, set_tracker
+from .compaction import compact_messages, context_size
+from .state import AgentState, reset_state, set_state
+
+logger = logging.getLogger("devii.agentic.loop")
+
+TraceCallback = Callable[[str, str, str], None]
+OUTPUT_CAP_CHARS = 400_000
+
+REFLECTION_TRIGGER = (
+    "[reflection-trigger] One or more tool calls returned an error. Call reflect() with the "
+    "observation, root-cause conclusion, and next action before retrying. Do not repeat the "
+    "same call without diagnosis."
+)
+PLAN_VIOLATION = (
+    "Protocol violation: your first tool call must be plan(). Restart with a structured plan "
+    "before any other action."
+)
+VERIFICATION_GATE = (
+    "[verification-gate] You produced a final answer after performing changes without calling "
+    "verify(). Confirm the change took effect and call verify() with a summary. If verification "
+    "truly does not apply, reply explicitly starting with: 'No verification applicable: <reason>'."
+)
+ITERATION_LIMIT_MESSAGE = "[stopped] Maximum iterations reached without a final answer."
+
+
+LABEL_KEYS = (
+    "post_slug",
+    "project_slug",
+    "gist_slug",
+    "news_slug",
+    "username",
+    "comment_uid",
+    "notification_uid",
+    "attachment_uid",
+    "poll_uid",
+    "receiver_uid",
+    "uid",
+    "name",
+    "q",
+    "query",
+    "title",
+    "goal",
+    "task",
+    "email",
+)
+LABEL_MAX_CHARS = 60
+
+
+def call_label(call: dict[str, Any]) -> str:
+    raw = call.get("function", {}).get("arguments") or "{}"
+    try:
+        arguments = json.loads(raw) if isinstance(raw, str) else raw
+    except (ValueError, TypeError):
+        return ""
+    if not isinstance(arguments, dict):
+        return ""
+
+    def clean(value: Any) -> str:
+        return str(value).replace("\n", " ").strip()[:LABEL_MAX_CHARS]
+
+    if arguments.get("target_uid"):
+        target_type = arguments.get("target_type")
+        target = clean(arguments["target_uid"])
+        return f"{target_type}/{target}" if target_type else target
+    for key in LABEL_KEYS:
+        value = arguments.get(key)
+        if value not in (None, ""):
+            return clean(value)
+    for value in arguments.values():
+        if isinstance(value, (str, int, float, bool)) and clean(value):
+            return clean(value)
+    return ""
+
+
+def normalize(message: dict[str, Any]) -> dict[str, Any]:
+    normalized: dict[str, Any] = {
+        "role": message.get("role", "assistant"),
+        "content": message.get("content") or "",
+    }
+    if message.get("tool_calls"):
+        normalized["tool_calls"] = message["tool_calls"]
+    return normalized
+
+
+def _is_error(result: str) -> bool:
+    try:
+        return isinstance(json.loads(result), dict) and json.loads(result).get("error") is not None
+    except (ValueError, TypeError):
+        return False
+
+
+def _summary(result: str) -> str:
+    try:
+        parsed = json.loads(result)
+    except (ValueError, TypeError):
+        return ""
+    if not isinstance(parsed, dict):
+        return ""
+    if parsed.get("error"):
+        return str(parsed.get("message") or parsed.get("error"))
+    if "status" in parsed:
+        return str(parsed["status"])
+    return ""
+
+
+async def _run_tool_call(dispatcher: Any, call: dict[str, Any]) -> str:
+    function = call.get("function", {})
+    name = function.get("name", "")
+    raw_arguments = function.get("arguments") or "{}"
+    try:
+        arguments = json.loads(raw_arguments) if isinstance(raw_arguments, str) else raw_arguments
+        if not isinstance(arguments, dict):
+            raise ValueError("arguments must be an object")
+    except ValueError as exc:
+        return json.dumps({"error": "tool_input_error", "message": f"Invalid arguments: {exc}"})
+    return await dispatcher.dispatch(name, arguments)
+
+
+async def react_loop(
+    llm: Any,
+    dispatcher: Any,
+    messages: list[dict[str, Any]],
+    tools: list[dict[str, Any]],
+    state: AgentState,
+    settings: Settings,
+    max_iterations: int,
+    plan_required: bool,
+    verify_required: bool,
+    on_trace: Optional[TraceCallback] = None,
+    cost_tracker: Any = None,
+    chunk_store: Any = None,
+) -> str:
+    def trace(event: str, name: str = "", detail: str = "") -> None:
+        if on_trace is not None:
+            on_trace(event, name, detail)
+
+    token = set_state(state)
+    cost_token = set_tracker(cost_tracker) if cost_tracker is not None else None
+    chunk_token = set_store(chunk_store) if chunk_store is not None else None
+    final_content = ""
+    try:
+        while state.iteration < max_iterations:
+            state.iteration += 1
+
+            if context_size(messages) > settings.context_compact_threshold:
+                trace("compact")
+                messages[:] = await compact_messages(llm, messages, settings.context_keep_tail)
+
+            try:
+                message = await llm.complete(messages, tools)
+            except LLMError as exc:
+                logger.info("LLM error: %s", exc.message)
+                return f"[model error] {exc.message}"
+
+            messages.append(normalize(message))
+            tool_calls = message.get("tool_calls") or []
+
+            if tool_calls:
+                if plan_required and state.plan is None and tool_calls[0]["function"]["name"] != "plan":
+                    trace("plan-gate")
+                    for call in tool_calls:
+                        messages.append(
+                            {
+                                "role": "tool",
+                                "tool_call_id": call.get("id", ""),
+                                "name": call.get("function", {}).get("name", ""),
+                                "content": json.dumps({"error": "protocol", "message": PLAN_VIOLATION}),
+                            }
+                        )
+                    continue
+
+                for call in tool_calls:
+                    trace("call", call.get("function", {}).get("name", ""), call_label(call))
+
+                results = await asyncio.gather(
+                    *[_run_tool_call(dispatcher, call) for call in tool_calls]
+                )
+
+                any_error = False
+                for call, result in zip(tool_calls, results):
+                    if len(result) > OUTPUT_CAP_CHARS:
+                        result = result[:OUTPUT_CAP_CHARS] + f"\n...[truncated {len(result)} chars]"
+                    name = call.get("function", {}).get("name", "")
+                    messages.append(
+                        {
+                            "role": "tool",
+                            "tool_call_id": call.get("id", ""),
+                            "name": name,
+                            "content": result,
+                        }
+                    )
+                    label = call_label(call)
+                    summary = _summary(result)
+                    detail = " ".join(part for part in (label, f"({summary})" if summary else "") if part)
+                    if _is_error(result):
+                        any_error = True
+                        trace("err", name, detail)
+                    else:
+                        trace("ok", name, detail)
+
+                if any_error:
+                    trace("reflect-trigger")
+                    messages.append({"role": "user", "content": REFLECTION_TRIGGER})
+                continue
+
+            content = message.get("content")
+            if content:
+                if (
+                    verify_required
+                    and not state.verified
+                    and not state.gate_triggered
+                    and state.mutations
+                ):
+                    state.gate_triggered = True
+                    trace("verify-gate")
+                    messages.append({"role": "user", "content": VERIFICATION_GATE})
+                    continue
+                final_content = content
+                break
+
+        if state.iteration >= max_iterations and not final_content:
+            return ITERATION_LIMIT_MESSAGE
+        return final_content
+    finally:
+        if chunk_token is not None:
+            reset_store(chunk_token)
+        if cost_token is not None:
+            reset_tracker(cost_token)
+        reset_state(token)

devplacepy/services/devii/agentic/state.py

@@ -0,0 +1,37 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import contextvars
+from dataclasses import dataclass, field
+from typing import Any, Optional
+
+_active_state: contextvars.ContextVar = contextvars.ContextVar("devii_agent_state", default=None)
+
+
+@dataclass
+class AgentState:
+    plan: Optional[dict[str, Any]] = None
+    reflections: list[dict[str, str]] = field(default_factory=list)
+    iteration: int = 0
+    verified: bool = False
+    gate_triggered: bool = False
+    mutations: set[str] = field(default_factory=set)
+
+
+def get_state() -> Optional[AgentState]:
+    return _active_state.get()
+
+
+def set_state(state: AgentState) -> contextvars.Token:
+    return _active_state.set(state)
+
+
+def reset_state(token: contextvars.Token) -> None:
+    _active_state.reset(token)
+
+
+def record_mutation(name: str) -> None:
+    state = _active_state.get()
+    if state is not None:
+        state.mutations.add(name)

devplacepy/services/devii/avatar/__init__.py

@@ -0,0 +1,5 @@
+# retoor <retoor@molodetz.nl>
+
+from .controller import AvatarController
+
+__all__ = ["AvatarController"]

devplacepy/services/devii/avatar/controller.py

@@ -0,0 +1,56 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any, Awaitable, Callable, Optional
+
+logger = logging.getLogger("devii.avatar")
+
+RequestSink = Callable[[str, dict[str, Any]], Awaitable[Any]]
+
+PREFIX = "avatar_"
+UNAVAILABLE = {
+    "status": "unavailable",
+    "message": (
+        "No avatar is attached to this session. devii's on-screen actions are only available "
+        "in the web interface (devii-terminal with a devii-avatar)."
+    ),
+}
+
+
+class AvatarController:
+    def __init__(self) -> None:
+        self._request: Optional[RequestSink] = None
+
+    def bind(self, request: RequestSink) -> None:
+        self._request = request
+        logger.debug("Avatar controller bound to a session")
+
+    def unbind(self) -> None:
+        self._request = None
+        logger.debug("Avatar controller unbound")
+
+    @property
+    def attached(self) -> bool:
+        return self._request is not None
+
+    async def dispatch(self, name: str, arguments: dict[str, Any]) -> str:
+        action = name[len(PREFIX):] if name.startswith(PREFIX) else name
+        if self._request is None:
+            return json.dumps(UNAVAILABLE, ensure_ascii=False)
+        try:
+            result = await self._request(action, arguments)
+        except Exception as exc:  # noqa: BLE001 - surfaced to the model as data
+            logger.exception("Avatar action %s failed", action)
+            return json.dumps({"status": "error", "action": action, "message": str(exc)})
+
+        if isinstance(result, dict) and result.get("error"):
+            return json.dumps(
+                {"status": "error", "action": action, "message": str(result["error"])},
+                ensure_ascii=False,
+            )
+        return json.dumps(
+            {"status": "success", "action": action, "result": result}, ensure_ascii=False
+        )

devplacepy/services/devii/chunks/__init__.py

@@ -0,0 +1,23 @@
+# retoor <retoor@molodetz.nl>
+
+from .controller import ChunkController
+from .store import (
+    ChunkStore,
+    chunk_envelope,
+    get_store,
+    reset_store,
+    serve_resource,
+    set_store,
+    wrap_if_large,
+)
+
+__all__ = [
+    "ChunkController",
+    "ChunkStore",
+    "chunk_envelope",
+    "get_store",
+    "set_store",
+    "reset_store",
+    "serve_resource",
+    "wrap_if_large",
+]

devplacepy/services/devii/chunks/controller.py

@@ -0,0 +1,61 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from ..config import Settings
+from ..errors import ToolInputError
+from .store import get_store
+
+
+class ChunkController:
+    def __init__(self, settings: Settings) -> None:
+        self._settings = settings
+
+    async def dispatch(self, name: str, arguments: dict[str, Any]) -> str:
+        if name != "read_more":
+            raise ToolInputError(f"Unknown chunk tool: {name}")
+        store = get_store()
+        if store is None:
+            return json.dumps(
+                {"status": "unavailable", "message": "No chunk store is active for this session."}
+            )
+        chunk_id = str(arguments.get("chunk_id", "")).strip()
+        if not chunk_id:
+            raise ToolInputError("read_more requires a chunk_id.")
+        text = store.get(chunk_id)
+        if text is None:
+            return json.dumps(
+                {
+                    "status": "not_found",
+                    "message": (
+                        f"No stored content for chunk_id '{chunk_id}'. It may have expired; "
+                        "re-run the original tool to get a fresh chunk_id."
+                    ),
+                }
+            )
+
+        offset = max(0, int(arguments.get("offset", 0) or 0))
+        length = int(arguments.get("length", self._settings.max_response_chars) or self._settings.max_response_chars)
+        length = max(1, min(length, self._settings.max_response_chars))
+
+        total = len(text)
+        piece = text[offset : offset + length]
+        next_offset = offset + len(piece)
+        has_more = next_offset < total
+        return json.dumps(
+            {
+                "status": "success",
+                "chunk_id": chunk_id,
+                "offset": offset,
+                "shown_chars": len(piece),
+                "total_chars": total,
+                "remaining_chars": max(0, total - next_offset),
+                "next_offset": next_offset if has_more else None,
+                "has_more": has_more,
+                "content": piece,
+            },
+            ensure_ascii=False,
+        )

devplacepy/services/devii/chunks/store.py

@@ -0,0 +1,130 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import contextvars
+import json
+import logging
+import uuid
+from collections import OrderedDict
+from typing import Optional
+
+logger = logging.getLogger("devii.chunks")
+
+STORE_MAX_ENTRIES = 16
+STORE_MAX_CHARS = 8_000_000
+
+_active_store: contextvars.ContextVar = contextvars.ContextVar("devii_chunk_store", default=None)
+
+
+class ChunkStore:
+    def __init__(self) -> None:
+        self._items: "OrderedDict[str, str]" = OrderedDict()
+        self._resources: dict[str, str] = {}
+
+    def put(self, text: str, resource_key: Optional[str] = None) -> str:
+        chunk_id = uuid.uuid4().hex[:12]
+        self._items[chunk_id] = text[:STORE_MAX_CHARS]
+        self._items.move_to_end(chunk_id)
+        if resource_key:
+            self._resources[resource_key] = chunk_id
+        while len(self._items) > STORE_MAX_ENTRIES:
+            evicted, _ = self._items.popitem(last=False)
+            self._drop_resource_for(evicted)
+        logger.debug("Stored chunkable result %s (%d chars)", chunk_id, len(text))
+        return chunk_id
+
+    def get(self, chunk_id: str) -> Optional[str]:
+        text = self._items.get(chunk_id)
+        if text is not None:
+            self._items.move_to_end(chunk_id)
+        return text
+
+    def get_by_resource(self, resource_key: str) -> Optional[tuple[str, str]]:
+        chunk_id = self._resources.get(resource_key)
+        if chunk_id is None:
+            return None
+        text = self._items.get(chunk_id)
+        if text is None:
+            del self._resources[resource_key]
+            return None
+        self._items.move_to_end(chunk_id)
+        return chunk_id, text
+
+    def invalidate_resources(self, prefix: str = "") -> None:
+        if not prefix:
+            self._resources.clear()
+            return
+        for key in [key for key in self._resources if key.startswith(prefix)]:
+            del self._resources[key]
+
+    def _drop_resource_for(self, chunk_id: str) -> None:
+        for key in [key for key, value in self._resources.items() if value == chunk_id]:
+            del self._resources[key]
+
+
+def get_store() -> Optional[ChunkStore]:
+    return _active_store.get()
+
+
+def set_store(store: ChunkStore) -> contextvars.Token:
+    return _active_store.set(store)
+
+
+def reset_store(token: contextvars.Token) -> None:
+    _active_store.reset(token)
+
+
+def chunk_envelope(text: str, chunk_id: str, max_chars: int) -> str:
+    total = len(text)
+    shown = text[:max_chars]
+    return json.dumps(
+        {
+            "truncated": True,
+            "chunk_id": chunk_id,
+            "total_chars": total,
+            "shown_chars": len(shown),
+            "remaining_chars": total - len(shown),
+            "next_offset": len(shown),
+            "hint": (
+                f"Output truncated. Call read_more with chunk_id='{chunk_id}' and "
+                f"offset={len(shown)} to read the next part (repeat until remaining_chars is 0). "
+                "Do not re-run the original tool; the full content is already cached."
+            ),
+            "content": shown,
+        },
+        ensure_ascii=False,
+    )
+
+
+def wrap_if_large(result: str, max_chars: int, resource_key: Optional[str] = None) -> str:
+    if len(result) <= max_chars:
+        return result
+    store = get_store()
+    if store is None:
+        total = len(result)
+        shown = result[:max_chars]
+        return json.dumps(
+            {
+                "truncated": True,
+                "total_chars": total,
+                "shown_chars": len(shown),
+                "remaining_chars": total - len(shown),
+                "hint": "Output truncated; no chunk store is available to fetch the rest.",
+                "content": shown,
+            },
+            ensure_ascii=False,
+        )
+    chunk_id = store.put(result, resource_key)
+    return chunk_envelope(result, chunk_id, max_chars)
+
+
+def serve_resource(resource_key: str, max_chars: int) -> Optional[str]:
+    store = get_store()
+    if store is None:
+        return None
+    hit = store.get_by_resource(resource_key)
+    if hit is None:
+        return None
+    chunk_id, text = hit
+    return chunk_envelope(text, chunk_id, max_chars)

devplacepy/services/devii/cli.py

@@ -0,0 +1,271 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import logging
+from dataclasses import replace
+from typing import Any, Optional
+
+import dataset
+
+from . import console
+from .actions import Dispatcher
+from .agent import Agent
+from .agentic import AgenticController, LessonStore
+from .chunks import ChunkStore
+from .config import Settings, load_settings
+from .cost import CostTracker
+from .errors import AuthRequiredError
+from .http_client import PlatformClient
+from .llm import LLMClient
+from .registry import CATALOG
+from .tasks import Scheduler, TaskController, TaskStore
+
+logger = logging.getLogger("devii.cli")
+
+EXIT_COMMANDS = {"/exit", "/quit", "/q"}
+WELCOME = "Devii - DevPlace assistant"
+LOGIN_REQUEST = (
+    "Welcome. I manage your DevPlace account, but you must log in first. "
+    "Please tell me your email and password to continue."
+)
+GOODBYE = "Session ended."
+RESULT_NOTICE_CHARS = 20000
+CLI_TASKS_DB = "devii_tasks.db"
+
+
+def _label(row: dict[str, Any]) -> str:
+    return row.get("label") or row.get("uid", "task")
+
+
+def _make_event_handler() -> Any:
+    def handle(kind: str, row: dict[str, Any], payload: str) -> None:
+        name = _label(row)
+        if kind == "start":
+            console.notice(f"\n[task {name}] running...")
+        elif kind in ("done", "finished"):
+            suffix = " (completed, no further runs)" if kind == "finished" else ""
+            console.notice(f"[task {name}] finished{suffix}:")
+            console.assistant(payload[:RESULT_NOTICE_CHARS])
+        elif kind == "error":
+            console.warn(f"[task {name}] failed: {payload}")
+
+    return handle
+
+
+TRACE_GLYPHS = {
+    "call": "↳",
+    "ok": "✓",
+    "err": "✗",
+    "plan-gate": "plan required first",
+    "reflect-trigger": "reflecting on error",
+    "verify-gate": "verification gate",
+    "compact": "context compaction",
+}
+
+
+def _trace(event: str, name: str = "", detail: str = "") -> None:
+    glyph = TRACE_GLYPHS.get(event, event)
+    if event in ("call", "ok", "err"):
+        suffix = f" {detail}" if detail else ""
+        console.notice(f"  {glyph} {name}{suffix}")
+    else:
+        console.notice(f"  ! {glyph}")
+
+
+def _make_executor(
+    settings: Settings,
+    llm: LLMClient,
+    dispatcher: Dispatcher,
+    lessons: LessonStore,
+    cost_tracker: CostTracker,
+    chunk_store: ChunkStore,
+) -> Any:
+    tools = CATALOG.tool_schemas()
+
+    async def execute(prompt: str) -> str:
+        worker = Agent(
+            settings, llm, dispatcher, tools,
+            lessons=lessons, cost_tracker=cost_tracker, chunk_store=chunk_store,
+        )
+        return await worker.respond(prompt)
+
+    return execute
+
+
+async def _bootstrap_auth(client: PlatformClient, settings: Settings) -> str:
+    if client.authenticated:
+        logger.info("Authenticated via API key")
+        return "Authenticated via API key. How can I help?"
+    if settings.login_email and settings.login_password:
+        try:
+            await client.login(settings.login_email, settings.login_password)
+            return f"Logged in as {client.username}. How can I help?"
+        except AuthRequiredError as exc:
+            console.warn(f"[auth] automatic login failed: {exc.message}")
+    return LOGIN_REQUEST
+
+
+def _resolve_owner(settings: Settings) -> tuple[str, str]:
+    """Resolve the local DevPlace user behind the CLI's credentials so the CLI shares the
+    same per-user memory, tasks, and admin analytics as the web session for that account."""
+    try:
+        from devplacepy.database import db, get_table
+        if "users" not in db.tables:
+            return "user", "cli"
+        users = get_table("users")
+        user = None
+        if settings.platform_api_key:
+            user = users.find_one(api_key=settings.platform_api_key)
+        if user is None and settings.login_email:
+            user = users.find_one(email=settings.login_email.lower())
+        if user:
+            return "user", user["uid"]
+    except Exception:  # noqa: BLE001 - no local DB / remote-only: fall back to a standalone store
+        return "user", "cli"
+    return "user", "cli"
+
+
+def _build_stores(owner_kind: str, owner_id: str):
+    if owner_id == "cli":
+        owned_db = dataset.connect(f"sqlite:///{CLI_TASKS_DB}")
+    else:
+        from devplacepy.database import db as owned_db  # share the platform DB for this account
+    return TaskStore(owned_db, owner_kind, owner_id), LessonStore(owned_db, owner_kind, owner_id)
+
+
+async def run(settings: Settings, prompt: Optional[str] = None) -> None:
+    client = PlatformClient(settings.base_url, settings.timeout_seconds, settings.platform_api_key)
+    llm = LLMClient(settings)
+    greeting = await _bootstrap_auth(client, settings)
+
+    owner_kind, owner_id = _resolve_owner(settings)
+    store, lessons = _build_stores(owner_kind, owner_id)
+    controller = TaskController(store)
+    cost_tracker = CostTracker()
+    chunk_store = ChunkStore()
+    agentic = AgenticController(lessons, settings)
+    dispatcher = Dispatcher(CATALOG, client, settings, controller, agentic, is_admin=True)
+    tools = CATALOG.tool_schemas_for(client.authenticated, is_admin=True)
+    agentic.bind(
+        llm=llm, dispatcher=dispatcher, tools=tools,
+        on_trace=_trace, cost_tracker=cost_tracker, chunk_store=chunk_store,
+    )
+    agent = Agent(
+        settings, llm, dispatcher, tools,
+        lessons=lessons, on_trace=_trace, cost_tracker=cost_tracker, chunk_store=chunk_store,
+    )
+
+    if prompt is not None:
+        try:
+            reply = await agent.respond(prompt)
+            print(reply)
+        finally:
+            await client.aclose()
+            await llm.aclose()
+        return
+
+    scheduler = Scheduler(
+        store,
+        _make_executor(settings, llm, dispatcher, lessons, cost_tracker, chunk_store),
+        _make_event_handler(),
+        settings.scheduler_tick_seconds,
+    )
+
+    console.banner(WELCOME)
+    console.notice(f"Endpoint {settings.base_url} | model {settings.ai_model}")
+    console.notice(
+        "Type /exit to quit. Agentic ReAct loop with planning, self-learning memory, and "
+        "background tasks is active.\n"
+    )
+    console.assistant(greeting)
+
+    scheduler.start()
+    try:
+        while True:
+            try:
+                user_text = (await console.prompt("you> ")).strip()
+            except (EOFError, KeyboardInterrupt):
+                print()
+                break
+
+            if not user_text:
+                continue
+            if user_text.lower() in EXIT_COMMANDS:
+                break
+
+            try:
+                reply = await agent.respond(user_text)
+            except Exception as exc:  # noqa: BLE001 - keep the chat alive
+                logger.exception("Unhandled error while responding")
+                console.warn(f"[error] {exc}")
+                continue
+
+            console.assistant(reply)
+    finally:
+        await scheduler.stop()
+        await client.aclose()
+        await llm.aclose()
+        console.notice(GOODBYE)
+
+
+def _parse_args(argv: Optional[list[str]] = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        prog="devii",
+        description="Agentic chat assistant that manages a DevPlace account.",
+    )
+    parser.add_argument("-p", "--prompt", help="Run a single prompt non-interactively and exit.")
+    parser.add_argument("--api-key", help="DevPlace API key for automatic authentication (Bearer).")
+    parser.add_argument("--email", help="Login email for automatic basic-auth login.")
+    parser.add_argument("--password", help="Login password for automatic basic-auth login.")
+    parser.add_argument("--basic-auth", help="Shorthand credentials in the form email:password.")
+    parser.add_argument("--base-url", help="Override the platform base URL.")
+    parser.add_argument("--ai-url", help="Override the model endpoint URL.")
+    parser.add_argument("--ai-key", help="Override the model API key.")
+    parser.add_argument("--model", help="Override the model name.")
+    return parser.parse_args(argv)
+
+
+def _settings_from_args(args: argparse.Namespace) -> Settings:
+    settings = load_settings()
+    email = args.email or settings.login_email
+    password = args.password or settings.login_password
+    if args.basic_auth and ":" in args.basic_auth:
+        email, password = args.basic_auth.split(":", 1)
+    overrides: dict[str, Any] = {"login_email": email, "login_password": password}
+    if args.api_key:
+        overrides["platform_api_key"] = args.api_key
+    if args.base_url:
+        overrides["base_url"] = args.base_url.rstrip("/")
+    if args.ai_url:
+        overrides["ai_url"] = args.ai_url
+    if args.ai_key:
+        overrides["ai_key"] = args.ai_key
+    if args.model:
+        overrides["ai_model"] = args.model
+    return replace(settings, **overrides)
+
+
+NOISY_LOGGERS = ("httpcore", "httpx", "httpcore.http11", "httpcore.connection", "asyncio", "urllib3")
+
+
+def main(argv: Optional[list[str]] = None) -> None:
+    args = _parse_args(argv)
+    settings = _settings_from_args(args)
+    logging.basicConfig(
+        level=getattr(logging, settings.log_level, logging.WARNING),
+        format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+    )
+    for name in NOISY_LOGGERS:
+        logging.getLogger(name).setLevel(logging.WARNING)
+    logger.info("Starting Devii")
+    try:
+        asyncio.run(run(settings, prompt=args.prompt))
+    except KeyboardInterrupt:
+        pass
+
+
+if __name__ == "__main__":
+    main()

devplacepy/services/devii/client/__init__.py

@@ -0,0 +1,5 @@
+# retoor <retoor@molodetz.nl>
+
+from .controller import ClientController
+
+__all__ = ["ClientController"]

devplacepy/services/devii/client/controller.py

@@ -0,0 +1,58 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any, Awaitable, Callable, Optional
+
+logger = logging.getLogger("devii.client")
+
+RequestSink = Callable[[str, dict[str, Any]], Awaitable[Any]]
+
+UNAVAILABLE = {
+    "status": "unavailable",
+    "message": (
+        "No browser is attached to this session. Client-side actions only work in the web "
+        "terminal, with a live connection open."
+    ),
+}
+EVAL_DISABLED = {
+    "status": "disabled",
+    "message": "JavaScript execution is disabled for Devii by the administrator.",
+}
+
+
+class ClientController:
+    def __init__(self, allow_eval: bool = True) -> None:
+        self._request: Optional[RequestSink] = None
+        self._allow_eval = allow_eval
+
+    def bind(self, request: RequestSink) -> None:
+        self._request = request
+        logger.debug("Client controller bound to a session")
+
+    def unbind(self) -> None:
+        self._request = None
+        logger.debug("Client controller unbound")
+
+    @property
+    def attached(self) -> bool:
+        return self._request is not None
+
+    async def dispatch(self, name: str, arguments: dict[str, Any]) -> str:
+        if name == "run_js" and not self._allow_eval:
+            return json.dumps(EVAL_DISABLED, ensure_ascii=False)
+        if self._request is None:
+            return json.dumps(UNAVAILABLE, ensure_ascii=False)
+        try:
+            result = await self._request(name, arguments)
+        except Exception as exc:  # noqa: BLE001 - surfaced to the model as data
+            logger.exception("Client action %s failed", name)
+            return json.dumps({"status": "error", "action": name, "message": str(exc)}, ensure_ascii=False)
+
+        if isinstance(result, dict) and result.get("error"):
+            return json.dumps(
+                {"status": "error", "action": name, "message": str(result["error"])}, ensure_ascii=False
+            )
+        return json.dumps({"status": "success", "action": name, "result": result}, ensure_ascii=False)

devplacepy/services/devii/config.py

@@ -0,0 +1,195 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import os
+import uuid
+from dataclasses import dataclass
+
+from devplacepy.config import INTERNAL_GATEWAY_URL, INTERNAL_MODEL
+
+
+DEFAULT_AI_URL = INTERNAL_GATEWAY_URL
+DEFAULT_AI_MODEL = INTERNAL_MODEL
+DEFAULT_BASE_URL = "http://127.0.0.1:10500"
+CONTEXT_WINDOW_TOKENS = 1_048_576
+MAX_OUTPUT_TOKENS = 384_000
+SYSTEM_RESERVE_TOKENS = 64_000
+CHARS_PER_TOKEN = 3
+CONTEXT_INPUT_BUDGET_TOKENS = CONTEXT_WINDOW_TOKENS - MAX_OUTPUT_TOKENS - SYSTEM_RESERVE_TOKENS
+
+MIN_TIMEOUT_SECONDS = 300.0
+DEFAULT_TIMEOUT_SECONDS = 300.0
+DEFAULT_MAX_RESPONSE_CHARS = 200_000
+DEFAULT_MAX_TOOL_ITERATIONS = 40
+DEFAULT_DELEGATE_MAX_ITERATIONS = 25
+DEFAULT_CONTEXT_COMPACT_THRESHOLD = CONTEXT_INPUT_BUDGET_TOKENS * CHARS_PER_TOKEN
+DEFAULT_CONTEXT_KEEP_TAIL = 12
+DEFAULT_RECALL_TOP_K = 3
+DEFAULT_FETCH_MAX_CHARS = 200_000
+DEFAULT_FETCH_TIMEOUT_SECONDS = 300.0
+DEFAULT_FETCH_MAX_BYTES = 8_000_000
+DEFAULT_RSEARCH_URL = "https://rsearch.app.molodetz.nl"
+DEFAULT_RSEARCH_TIMEOUT_SECONDS = 300.0
+
+
+@dataclass(frozen=True)
+class Settings:
+    ai_url: str
+    ai_key: str
+    ai_model: str
+    base_url: str
+    timeout_seconds: float
+    max_response_chars: int
+    max_tool_iterations: int
+    log_level: str
+    log_path: str
+    tasks_db_path: str
+    scheduler_tick_seconds: float
+    lessons_db_path: str
+    delegate_max_iterations: int
+    context_compact_threshold: int
+    context_keep_tail: int
+    recall_top_k: int
+    plan_required: bool
+    verify_required: bool
+    platform_api_key: str
+    login_email: str
+    login_password: str
+    fetch_max_chars: int
+    fetch_timeout_seconds: float
+    fetch_max_bytes: int
+    fetch_allow_private: bool
+    allow_eval: bool
+    rsearch_enabled: bool
+    rsearch_url: str
+    rsearch_timeout_seconds: float
+    daily_limit_usd: float
+
+
+def _default_ai_key() -> str:
+    env_key = os.environ.get("DEVII_AI_KEY", "")
+    if env_key:
+        return env_key
+    try:
+        from devplacepy.database import internal_gateway_key
+        key = internal_gateway_key()
+        if key:
+            return key
+    except Exception:  # noqa: BLE001 - no local DB / remote-only: fall back to a random key
+        pass
+    return str(uuid.uuid4())
+
+
+def load_settings() -> Settings:
+    return Settings(
+        ai_url=os.environ.get("DEVII_AI_URL", DEFAULT_AI_URL),
+        ai_key=_default_ai_key(),
+        ai_model=os.environ.get("DEVII_AI_MODEL", DEFAULT_AI_MODEL),
+        base_url=os.environ.get("DEVII_BASE_URL", DEFAULT_BASE_URL).rstrip("/"),
+        timeout_seconds=float(os.environ.get("DEVII_TIMEOUT", DEFAULT_TIMEOUT_SECONDS)),
+        max_response_chars=int(os.environ.get("DEVII_MAX_RESPONSE", DEFAULT_MAX_RESPONSE_CHARS)),
+        max_tool_iterations=int(os.environ.get("DEVII_MAX_TOOL_ITERATIONS", DEFAULT_MAX_TOOL_ITERATIONS)),
+        log_level=os.environ.get("DEVII_LOG_LEVEL", "WARNING").upper(),
+        log_path=os.environ.get("DEVII_LOG_PATH", "devii.log"),
+        tasks_db_path=os.environ.get("DEVII_TASKS_DB", "devii_tasks.db"),
+        scheduler_tick_seconds=float(os.environ.get("DEVII_SCHEDULER_TICK", "1.0")),
+        lessons_db_path=os.environ.get("DEVII_LESSONS_DB", "devii_lessons.db"),
+        delegate_max_iterations=int(
+            os.environ.get("DEVII_DELEGATE_MAX_ITERATIONS", DEFAULT_DELEGATE_MAX_ITERATIONS)
+        ),
+        context_compact_threshold=int(
+            os.environ.get("DEVII_CONTEXT_COMPACT_THRESHOLD", DEFAULT_CONTEXT_COMPACT_THRESHOLD)
+        ),
+        context_keep_tail=int(os.environ.get("DEVII_CONTEXT_KEEP_TAIL", DEFAULT_CONTEXT_KEEP_TAIL)),
+        recall_top_k=int(os.environ.get("DEVII_RECALL_TOP_K", DEFAULT_RECALL_TOP_K)),
+        plan_required=os.environ.get("DEVII_PLAN_REQUIRED", "1").lower() not in ("0", "false", "no"),
+        verify_required=os.environ.get("DEVII_VERIFY_REQUIRED", "1").lower() not in ("0", "false", "no"),
+        platform_api_key=os.environ.get("DEVII_PLATFORM_API_KEY", ""),
+        login_email=os.environ.get("DEVII_LOGIN_EMAIL", ""),
+        login_password=os.environ.get("DEVII_LOGIN_PASSWORD", ""),
+        fetch_max_chars=int(os.environ.get("DEVII_FETCH_MAX_CHARS", DEFAULT_FETCH_MAX_CHARS)),
+        fetch_timeout_seconds=float(
+            os.environ.get("DEVII_FETCH_TIMEOUT", DEFAULT_FETCH_TIMEOUT_SECONDS)
+        ),
+        fetch_max_bytes=int(os.environ.get("DEVII_FETCH_MAX_BYTES", DEFAULT_FETCH_MAX_BYTES)),
+        fetch_allow_private=os.environ.get("DEVII_FETCH_ALLOW_PRIVATE", "0").lower()
+        in ("1", "true", "yes", "on"),
+        allow_eval=os.environ.get("DEVII_ALLOW_EVAL", "1").lower() in ("1", "true", "yes", "on"),
+        rsearch_enabled=os.environ.get("DEVII_RSEARCH_ENABLED", "1").lower() in ("1", "true", "yes", "on"),
+        rsearch_url=os.environ.get("DEVII_RSEARCH_URL", DEFAULT_RSEARCH_URL).rstrip("/"),
+        rsearch_timeout_seconds=float(
+            os.environ.get("DEVII_RSEARCH_TIMEOUT", DEFAULT_RSEARCH_TIMEOUT_SECONDS)
+        ),
+        daily_limit_usd=0.0,
+    )
+
+
+FIELD_AI_URL = "devii_ai_url"
+FIELD_AI_MODEL = "devii_ai_model"
+FIELD_AI_KEY = "devii_ai_key"
+FIELD_PLAN_REQUIRED = "devii_plan_required"
+FIELD_VERIFY_REQUIRED = "devii_verify_required"
+FIELD_MAX_ITERATIONS = "devii_max_iterations"
+FIELD_USER_DAILY_USD = "devii_user_daily_usd"
+FIELD_GUEST_DAILY_USD = "devii_guest_daily_usd"
+FIELD_ADMIN_DAILY_USD = "devii_admin_daily_usd"
+FIELD_GUESTS_ENABLED = "devii_guests_enabled"
+FIELD_PRICE_CACHE_HIT = "devii_price_cache_hit"
+FIELD_PRICE_CACHE_MISS = "devii_price_cache_miss"
+FIELD_PRICE_OUTPUT = "devii_price_output"
+FIELD_ALLOW_EVAL = "devii_allow_eval"
+FIELD_TIMEOUT = "devii_timeout"
+FIELD_FETCH_TIMEOUT = "devii_fetch_timeout"
+FIELD_RSEARCH_ENABLED = "devii_rsearch_enabled"
+FIELD_RSEARCH_URL = "devii_rsearch_url"
+FIELD_RSEARCH_TIMEOUT = "devii_rsearch_timeout"
+
+LESSONS_DB_PATH = os.environ.get("DEVII_LESSONS_DB", "devii_lessons.db")
+
+
+def effective_daily_limit(config: dict, owner_kind: str, is_admin: bool = False) -> float:
+    if is_admin:
+        return float(config.get(FIELD_ADMIN_DAILY_USD, 0.0) or 0.0)
+    if owner_kind == "guest":
+        return float(config.get(FIELD_GUEST_DAILY_USD, 0.05) or 0.0)
+    return float(config.get(FIELD_USER_DAILY_USD, 1.0) or 0.0)
+
+
+def build_settings(
+    config: dict, base_url: str, api_key: str, owner_kind: str = "guest", is_admin: bool = False
+) -> Settings:
+    configured_key = config[FIELD_AI_KEY] or str(uuid.uuid4())
+    ai_key = api_key if (owner_kind == "user" and api_key) else configured_key
+    return Settings(
+        ai_url=config[FIELD_AI_URL] or DEFAULT_AI_URL,
+        ai_key=ai_key,
+        ai_model=config[FIELD_AI_MODEL] or DEFAULT_AI_MODEL,
+        base_url=(base_url or DEFAULT_BASE_URL).rstrip("/"),
+        timeout_seconds=float(config.get(FIELD_TIMEOUT) or DEFAULT_TIMEOUT_SECONDS),
+        max_response_chars=DEFAULT_MAX_RESPONSE_CHARS,
+        max_tool_iterations=int(config[FIELD_MAX_ITERATIONS]),
+        log_level="INFO",
+        log_path="devii.log",
+        tasks_db_path="",
+        scheduler_tick_seconds=1.0,
+        lessons_db_path=LESSONS_DB_PATH,
+        delegate_max_iterations=DEFAULT_DELEGATE_MAX_ITERATIONS,
+        context_compact_threshold=DEFAULT_CONTEXT_COMPACT_THRESHOLD,
+        context_keep_tail=DEFAULT_CONTEXT_KEEP_TAIL,
+        recall_top_k=DEFAULT_RECALL_TOP_K,
+        plan_required=bool(config[FIELD_PLAN_REQUIRED]),
+        verify_required=bool(config[FIELD_VERIFY_REQUIRED]),
+        platform_api_key=api_key,
+        login_email="",
+        login_password="",
+        fetch_max_chars=DEFAULT_FETCH_MAX_CHARS,
+        fetch_timeout_seconds=float(config.get(FIELD_FETCH_TIMEOUT) or DEFAULT_FETCH_TIMEOUT_SECONDS),
+        fetch_max_bytes=DEFAULT_FETCH_MAX_BYTES,
+        fetch_allow_private=False,
+        allow_eval=bool(config.get(FIELD_ALLOW_EVAL, True)),
+        rsearch_enabled=bool(config.get(FIELD_RSEARCH_ENABLED, True)),
+        rsearch_url=(config.get(FIELD_RSEARCH_URL) or DEFAULT_RSEARCH_URL).rstrip("/"),
+        rsearch_timeout_seconds=float(config.get(FIELD_RSEARCH_TIMEOUT) or DEFAULT_RSEARCH_TIMEOUT_SECONDS),
+        daily_limit_usd=effective_daily_limit(config, owner_kind, is_admin),
+    )

devplacepy/services/devii/console.py

@@ -0,0 +1,44 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import asyncio
+import sys
+
+RESET = "\033[0m"
+BOLD = "\033[1m"
+CYAN = "\033[36m"
+GREEN = "\033[32m"
+YELLOW = "\033[33m"
+GREY = "\033[90m"
+
+
+def _supports_color() -> bool:
+    return sys.stdout.isatty()
+
+
+def paint(text: str, color: str) -> str:
+    if not _supports_color():
+        return text
+    return f"{color}{text}{RESET}"
+
+
+def banner(text: str) -> None:
+    print(paint(text, BOLD + CYAN))
+
+
+def assistant(text: str) -> None:
+    print(paint("agent> ", GREEN) + text)
+
+
+def notice(text: str) -> None:
+    print(paint(text, GREY))
+
+
+def warn(text: str) -> None:
+    print(paint(text, YELLOW))
+
+
+async def prompt(label: str) -> str:
+    rendered = paint(label, BOLD + CYAN)
+    return await asyncio.to_thread(input, rendered)