Add devplacepy/static/uploads/ to .gitignore

.coveragerc

@@ -0,0 +1,12 @@
+# retoor <retoor@molodetz.nl>
+[run]
+source = devplacepy
+parallel = true
+sigterm = true
+omit =
+    tests/*
+    sitecustomize.py
+
+[report]
+show_missing = true
+skip_covered = false

.env.example

@@ -0,0 +1,43 @@
+# Copy to .env and adjust. Loaded by docker-compose (env_file) and by the app
+# at startup (python-dotenv). .env is git-ignored; this example is committed.
+
+# Session signing key. CHANGE THIS for any real deployment.
+SECRET_KEY=change-me
+
+# Database. Leave unset to use the shared project-root devplace.db (the Docker
+# app container bind-mounts ./ to /app, so it reads and writes the same file as
+# `make dev`). Set only to point at a different SQLite file.
+# DEVPLACE_DATABASE_URL=sqlite:////app/devplace.db
+
+# Persistent runtime data (zip archives, container workspaces). Lives OUTSIDE the
+# package and is never served via /static. Defaults to <repo>/var. The docker
+# daemon must be able to bind-mount this dir for container /app mounts; point it
+# at a persistent volume in production.
+# DEVPLACE_DATA_DIR=/var/lib/devplace
+
+# Container Manager (admin-only, enabled via docker-compose.containers.yml).
+# Host the /p/<slug> ingress proxy dials to reach a published container port.
+# On the host: 127.0.0.1 (default). Containerized app reaching host ports:
+# host.docker.internal.
+# DEVPLACE_CONTAINER_PROXY_HOST=host.docker.internal
+# GID of /var/run/docker.sock on the host (getent group docker | cut -d: -f3),
+# so the UID-1000 app can use the socket.
+# DOCKER_GID=999
+
+# Public origin for absolute URLs (SEO, canonical links, push). Empty = derive
+# from the request.
+DEVPLACE_SITE_URL=
+
+# Host port the nginx front door binds.
+PORT=10500
+
+# nginx upload ceiling. Must be >= the admin-configurable max_upload_size_mb.
+NGINX_MAX_BODY_SIZE=50m
+
+# Optional nginx micro-cache for proxied GETs.
+NGINX_CACHE_ENABLED=false
+NGINX_CACHE_MAX_SIZE=1g
+
+# Run the app container as this host user so shared files keep dev ownership.
+DEVPLACE_UID=1000
+DEVPLACE_GID=1000

.gitea/workflows/test.yaml

@@ -18,17 +18,34 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pip install -e .
-          pip install playwright
+          pip install -e ".[dev]"
           python -m playwright install chromium --with-deps
 
-      - name: Run integration tests
+      - name: Run integration tests with coverage
+        env:
+          COVERAGE_PROCESS_START: ${{ github.workspace }}/.coveragerc
+          PLAYWRIGHT_HEADLESS: "1"
         run: |
-          python -m pytest tests/ -v --tb=line -x
+          python -m coverage run -m pytest tests/ -p no:xdist --tb=line
+
+      - name: Build coverage report
+        if: always()
+        run: |
+          python -m coverage combine
+          python -m coverage report
+          python -m coverage html
+
+      - name: Publish coverage HTML
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage-html
+          path: htmlcov/
 
       - name: Upload test screenshots
         if: failure()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: failure-screenshots
           path: /tmp/devplace_test_screenshots/
+

.gitignore

@@ -1,7 +1,30 @@
+.cache
+.local
+.devplace_bots/
 __pycache__/
 *.py[cod]
 *.egg-info/
 .env
 devplace.db*
+devplace-services.lock
+devplace-init.lock
+.vapid.lock
+notification-private.pem
+notification-private.pkcs8.pem
+notification-public.pem
 .pytest_cache/
 .opencode
+devii_*.db
+devii_*.db-shm
+devii_*.db-wal
+devii.log
+webdata/
+# Uploaded/downloaded files - never track in git
+devplacepy/static/uploads/
+# Runtime data dir (container workspaces, zip artifacts) - never inside the package
+var/
+
+# coverage
+.coverage
+.coverage.*
+htmlcov/

Dockerfile

@@ -1,22 +1,40 @@
-FROM python:3.11-slim
+FROM python:3.13-slim
 
 WORKDIR /app
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    curl \
+    curl ca-certificates \
     && rm -rf /var/lib/apt/lists/*
 
+# Optional: the docker CLI so the (admin-only) container manager can drive the host
+# docker daemon. Off by default; the container compose override turns it on.
+ARG INSTALL_DOCKER_CLI=false
+RUN if [ "$INSTALL_DOCKER_CLI" = "true" ]; then \
+        install -m 0755 -d /etc/apt/keyrings && \
+        curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc && \
+        chmod a+r /etc/apt/keyrings/docker.asc && \
+        echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian bookworm stable" > /etc/apt/sources.list.d/docker.list && \
+        apt-get update && apt-get install -y --no-install-recommends docker-ce-cli && \
+        rm -rf /var/lib/apt/lists/* ; \
+    fi
+
 COPY pyproject.toml .
 
 COPY devplacepy/ devplacepy/
 
-RUN pip install --no-cache-dir .
+ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
 
-RUN mkdir -p /app/data /app/devplacepy/static/uploads/attachments
+RUN pip install --no-cache-dir ".[bots]" \
+    && python -m playwright install --with-deps chromium \
+    && chmod -R a+rx /ms-playwright
+
+RUN mkdir -p /app/devplacepy/static/uploads/attachments
 
 EXPOSE 10500
 
+ENV DEVPLACE_WEB_WORKERS=2
+
 HEALTHCHECK --interval=30s --timeout=10s --retries=3 --start-period=10s \
     CMD curl -f http://localhost:10500/ || exit 1
 
-CMD ["uvicorn", "devplacepy.main:app", "--host", "0.0.0.0", "--port", "10500", "--workers", "4", "--backlog", "8192"]
+CMD ["uvicorn", "devplacepy.main:app", "--host", "0.0.0.0", "--port", "10500", "--workers", "2", "--backlog", "8192", "--proxy-headers", "--forwarded-allow-ips", "*"]

Makefile

@@ -5,45 +5,69 @@ LOCUST_DB ?= $(LOCUST_DB_DIR)/datastore.db
 LOCUST_USERS ?= 20
 LOCUST_SPAWN_RATE ?= 5
 LOCUST_RUN_TIME ?= 120s
+DEVPLACE_RATE_LIMIT ?= 1000000
 
-.PHONY: install dev clean test test-headed demo locust locust-headless
+.PHONY: install dev clean test test-headed coverage coverage-headed coverage-html locust locust-headless
 
 install:
 	pip install -e .
 
 dev:
-	uvicorn devplacepy.main:app --reload --host 0.0.0.0 --port 10500 --backlog 4096
+	uvicorn devplacepy.main:app --reload --reload-dir devplacepy --host 0.0.0.0 --port 10500 --backlog 4096
 
 prod:
-	uvicorn devplacepy.main:app --host 0.0.0.0 --port 10500 --workers 2 --backlog 8192
+	DEVPLACE_WEB_WORKERS=2 uvicorn devplacepy.main:app --host 0.0.0.0 --port 10500 --workers 2 --backlog 8192 --proxy-headers --forwarded-allow-ips '*'
 
 test:
-	PLAYWRIGHT_HEADLESS=1 python -m pytest tests/test_landing.py tests/test_auth.py tests/test_feed.py tests/test_post.py tests/test_profile.py tests/test_projects.py tests/test_messages.py tests/test_notifications.py tests/test_services.py tests/test_seo.py -v --tb=line -x
+	PLAYWRIGHT_HEADLESS=1 python -m pytest tests/ -v --tb=line -x
 
 test-headed:
-	PLAYWRIGHT_HEADLESS=0 python -m pytest tests/test_landing.py tests/test_auth.py tests/test_feed.py tests/test_post.py tests/test_profile.py tests/test_projects.py tests/test_messages.py tests/test_notifications.py tests/test_services.py -v --tb=line -x
+	PLAYWRIGHT_HEADLESS=0 python -m pytest tests/ -v --tb=line -x
 
-demo:
-	PLAYWRIGHT_HEADLESS=0 python -m pytest tests/test_demo.py -v -s --tb=line -x
+coverage:
+	rm -f .coverage .coverage.*
+	COVERAGE_PROCESS_START=$(CURDIR)/.coveragerc PLAYWRIGHT_HEADLESS=1 \
+		python -m coverage run -m pytest tests/ -p no:xdist --tb=line
+	python -m coverage combine
+	python -m coverage report
+
+coverage-headed:
+	rm -f .coverage .coverage.*
+	COVERAGE_PROCESS_START=$(CURDIR)/.coveragerc PLAYWRIGHT_HEADLESS=0 \
+		python -m coverage run -m pytest tests/ -p no:xdist --tb=line
+	python -m coverage combine
+	python -m coverage report
+
+coverage-html: coverage
+	python -m coverage html
+	@echo "Report written to htmlcov/index.html"
 
 locust:
 	export DEVPLACE_DATABASE_URL="sqlite:///$(LOCUST_DB)"; \
+	export DEVPLACE_RATE_LIMIT=$(DEVPLACE_RATE_LIMIT); \
+	fuser -k $(LOCUST_PORT)/tcp 2>/dev/null || true; \
+	sleep 1; \
 	mkdir -p $(LOCUST_DB_DIR); \
 	rm -f $(LOCUST_DB); \
 	uvicorn devplacepy.main:app --host 127.0.0.1 --port $(LOCUST_PORT) --backlog 8192 > /tmp/devplace_locust_server.log 2>&1 & \
 	PID=$$!; \
-	while ! curl -s http://127.0.0.1:$(LOCUST_PORT)/ > /dev/null 2>&1; do sleep 0.5; done; \
+	while kill -0 $$PID 2>/dev/null && ! curl -s http://127.0.0.1:$(LOCUST_PORT)/ > /dev/null 2>&1; do sleep 0.5; done; \
+	if ! kill -0 $$PID 2>/dev/null; then echo "Server failed to start (port $(LOCUST_PORT) busy?). See /tmp/devplace_locust_server.log"; exit 1; fi; \
 	locust --host http://127.0.0.1:$(LOCUST_PORT) --web-port $(LOCUST_WEB_PORT); \
 	kill $$PID 2>/dev/null || true; \
 	rm -rf $(LOCUST_DB_DIR)
 
 locust-headless:
 	export DEVPLACE_DATABASE_URL="sqlite:///$(LOCUST_DB)"; \
+	export DEVPLACE_RATE_LIMIT=$(DEVPLACE_RATE_LIMIT); \
+	fuser -k $(LOCUST_PORT)/tcp 2>/dev/null || true; \
+	sleep 1; \
 	mkdir -p $(LOCUST_DB_DIR); \
 	rm -f $(LOCUST_DB); \
 	uvicorn devplacepy.main:app --host 127.0.0.1 --port $(LOCUST_PORT) --backlog 8192 > /tmp/devplace_locust_server.log 2>&1 & \
 	PID=$$!; \
-	while ! curl -s http://127.0.0.1:$(LOCUST_PORT)/ > /dev/null 2>&1; do sleep 0.5; done; \
+	while kill -0 $$PID 2>/dev/null && ! curl -s http://127.0.0.1:$(LOCUST_PORT)/ > /dev/null 2>&1; do sleep 0.5; done; \
+	if ! kill -0 $$PID 2>/dev/null; then echo "Server failed to start (port $(LOCUST_PORT) busy?). See /tmp/devplace_locust_server.log"; exit 1; fi; \
 	locust --host http://127.0.0.1:$(LOCUST_PORT) --headless -u $(LOCUST_USERS) -r $(LOCUST_SPAWN_RATE) --run-time $(LOCUST_RUN_TIME) --html=$(LOCUST_DB_DIR)/report.html; \
 	kill $$PID 2>/dev/null || true; \
 	rm -rf $(LOCUST_DB_DIR)
@@ -54,19 +78,45 @@ clean:
 	rm -rf devplacepy.egg-info
 	rm -rf .venv
 
-.PHONY: docker-build docker-up docker-down docker-logs docker-clean
+# Container Manager works out of the box: the overlay installs the docker CLI in
+# the image and mounts the host socket. DOCKER_GID is read straight from the
+# socket so the UID-1000 app can use it; the data dir is the project's own var/
+# at its real host path, so the DooD bind-mount (host == container path) holds
+# with no /srv dir and no sudo.
+COMPOSE := docker compose -f docker-compose.yml -f docker-compose.containers.yml
+DEVPLACE_DATA_DIR ?= $(CURDIR)/var
+DOCKER_GID ?= $(shell stat -c '%g' /var/run/docker.sock 2>/dev/null)
+export DEVPLACE_DATA_DIR
+export DOCKER_GID
 
-docker-build:
-	docker compose build
+.PHONY: docker-build docker-up docker-down docker-logs docker-clean docker-prep ppy
 
-docker-up:
-	docker compose up -d
+# Build the single shared container image every instance runs. Build once;
+# rebuild only when ppy.Dockerfile, the sudo shim, or pagent change.
+ppy:
+	docker build --network=host -f ppy.Dockerfile -t ppy:latest devplacepy/services/containers/files
+
+docker-prep:
+	mkdir -p $(DEVPLACE_DATA_DIR)
+
+docker-build: docker-prep
+	$(COMPOSE) build
+
+docker-up: docker-prep
+	$(COMPOSE) up -d
 
 docker-down:
-	docker compose down
+	$(COMPOSE) down
 
 docker-logs:
-	docker compose logs -f
+	$(COMPOSE) logs -f
 
 docker-clean:
-	docker compose down -v
+	$(COMPOSE) down -v
+
+docker-bup: docker-build docker-up
+
+deploy:
+	git checkout production
+	git merge master
+	git push origin production

README.md

@@ -1,4 +1,4 @@
-# DevPlace — The Developer Social Network
+# DevPlace - The Developer Social Network
 
 Server-rendered social network for developers. FastAPI backend serving Jinja2 templates with ES6 interactivity. Avatar generation via Multiavatar (local, no network). SQLite via `dataset` with WAL mode and concurrency tuning.
 
@@ -11,7 +11,6 @@ make install        # pip install -e .
 make dev            # uvicorn --reload on port 10500
 make test           # Playwright integration + unit tests, headless, fail-fast
 make test-headed    # same tests in visible browser
-make demo           # full-journey GUI demo (headed)
 ```
 
 Open `http://localhost:10500`.
@@ -20,13 +19,13 @@ Open `http://localhost:10500`.
 
 | Layer | Technology |
 |-------|-----------|
-| Backend | Python 3.13+, FastAPI, Uvicorn (single worker) |
+| Backend | Python 3.13+, FastAPI, Uvicorn (multi-worker in production) |
 | Templates | Jinja2 (server-side rendered) |
 | Frontend | Pure ES6 JavaScript, one class per file |
 | Database | SQLite via `dataset` (auto-sync schema, `uid` PKs, WAL mode, 30s busy timeout) |
-| Auth | Session cookies, SHA256+SALT via passlib |
+| Auth | Session cookie, API key (`X-API-KEY`/Bearer), or HTTP Basic; PBKDF2-SHA256 via passlib |
 | Avatars | Multiavatar (local SVG generation, no external API, <5ms) |
-| Validation | `hawk` (Python/JS/CSS/HTML) |
+| Coverage | `coverage.py` (`.coveragerc`, subprocess-aware) |
 | Load testing | Locust (locustfile.py) |
 
 ## Project structure
@@ -38,9 +37,10 @@ devplacepy/
   database.py         # dataset connection, index creation
   templating.py       # Shared Jinja2 environment + globals
   avatar.py           # Multiavatar generation, URL builder
-  utils.py            # Password hashing, session mgmt, time_ago
+  utils.py            # Password hashing, session mgmt, time_ago, notification hook
   models.py           # Pydantic schemas
-  routers/            # One file per domain (auth, feed, posts, ...)
+  push.py             # Web push crypto, VAPID keys, encrypt/send/register
+  routers/            # One file per domain (auth, feed, posts, push, ...)
   templates/          # Jinja2 HTML templates
   static/css/         # Page-specific CSS files
   static/js/          # Application.js (ES6 module)
@@ -56,59 +56,470 @@ devplacepy/
 | `/news` | Developer news listing, detail page with comments |
 | `/posts` | Post detail, creation |
 | `/comments` | Comment creation, deletion |
-| `/projects` | Project listing, creation |
+| `/projects` | Project listing, creation, and per-project visibility toggles: `POST /projects/{slug}/private` (owner-only visibility) and `POST /projects/{slug}/readonly` (immutable files) |
+| `/projects/{slug}/files` | Per-project filesystem: directory and file CRUD, upload, inline editing, and line-range operations (`lines` read, `replace-lines`, `insert-lines`, `delete-lines`, `append`) for surgical edits to large text files (public read, owner write; all writes refused while the project is read-only) |
+| `/zips` | Zip job status (`/zips/{uid}`) and archive download (`/zips/{uid}/download`); archives are queued via `/projects/{slug}/zip` and `/projects/{slug}/files/zip` |
+| `/forks` | Fork job status (`/forks/{uid}`); forks are queued via `/projects/{slug}/fork`. Any signed-in user can fork a project they can view into a new project they own; once the job finishes the response carries the new project URL |
+| `/projects/{slug}/containers` | Admin per-project container manager: Dockerfile CRUD with immutable versions, async image builds, and container instance creation. Reachable from the project page via the admin-only **Containers** button |
+| `/admin/containers` | Admin **Containers** section: a list of every container instance across all projects (`/admin/containers`), each linking to a dedicated instance detail page (`/admin/containers/{uid}`) with lifecycle controls, live logs and metrics, an interactive terminal, schedules, ingress, and workspace sync |
+| `/p/{slug}` | Public ingress proxy (HTTP + WebSocket) to a running container instance's published port, opt-in per instance via `ingress_slug` |
 | `/profile` | Profile view, editing |
 | `/messages` | Direct messaging |
-| `/notifications` | Notification list, mark read |
+| `/notifications` | Notification list, mark read, live unread counts (`/notifications/counts`) |
 | `/votes` | Upvote/downvote on posts, comments, projects |
+| `/reactions` | Emoji reactions on posts, comments, gists, projects |
+| `/bookmarks` | Save/unsave content; `/bookmarks/saved` personal list |
+| `/polls` | Vote on post-attached polls |
 | `/follow` | Follow/unfollow users |
+| `/leaderboard` | Contributor ranking by total stars earned |
 | `/avatar` | Multiavatar proxy with in-memory cache |
 | `/bugs` | Bug reports listing, creation |
-| `/services` | Background service monitoring (status, logs) |
+| `/admin/services` | Background service management (start/stop, config, status, logs) |
 | `/admin` | Admin panel (user management, news curation, settings) |
+| `/docs` | Developer documentation site with a complete, interactive HTTP API reference |
+| `/openai` | OpenAI-compatible LLM gateway service (`/openai/v1/chat/completions`, `/openai/v1/*`) |
+| `/devii` | Devii agentic assistant: WebSocket terminal (`/devii/ws`), standalone page, usage (`/devii/usage`), session bootstrap |
 | `(none)` | `/robots.txt`, `/sitemap.xml` (SEO) |
+| `(none)` | `/push.json` (VAPID key + subscribe), `/service-worker.js`, `/manifest.json` (push + PWA) |
+
+## Gamification
+
+Member progression is driven by activity and peer recognition.
+
+- **Stars** are the net vote score (`upvotes - downvotes`) on a post, project, or gist. A member's total stars is the sum across all their content and is the basis for ranking.
+- **XP and levels.** Members earn XP for contributing: posting (10), commenting (2), publishing a project (15) or gist (5), receiving an upvote (5), and gaining a follower (5). Each level requires 100 XP (`level = 1 + xp // 100`). The profile shows the current level and progress to the next.
+- **Badges** are awarded once per milestone: first post/comment/project/gist, 10 posts (Prolific), 25 stars (Rising Star), 100 stars (Star Author), 10 followers (Popular), a 7-day activity streak (On Fire), and reaching levels 5 and 10. Badges render with an icon and description on the profile.
+- **Leaderboard** (`/leaderboard`) ranks the top 50 members by total stars (single page, no pagination); a member's own rank is shown on their profile.
+- **Contribution heatmap and streaks.** Each profile shows a 12-month activity heatmap and the current/longest daily streak, derived from post/comment/gist/project timestamps (no extra storage).
+- **Social graph listings.** Each profile has Followers and Following tabs that paginate the follow graph (25 per page) and show a follow/unfollow control for each person. The same data is available as JSON at `GET /profile/{username}/followers` and `GET /profile/{username}/following`.
+- **Reward notifications** fire when a member levels up or earns a badge.
+
+## Engagement
+
+- **Emoji reactions** - a fixed palette of reactions on posts, comments, gists, and projects, separate from voting and carrying no ranking weight.
+- **Polls** - a post can carry a poll (question plus up to six options); results appear as live bars once the viewer votes, one vote per member. A poll can be attached when the post is created or added later by editing a post that has none.
+- **Bookmarks** - save posts, gists, projects, and news to a personal list at `/bookmarks/saved`.
+- **Private projects** - an owner can mark a project private so it is visible only to them (and administrators) and excluded from listings, profiles, search, the sitemap, and zip access. Set at creation or toggled later from the project page.
+- **Read-only projects** - an owner can mark a project read-only, making its entire virtual filesystem immutable: every write, edit, line-edit, move, delete, and upload is refused from all paths (the web UI, the HTTP API, the Devii agent, and container workspace sync) until read-only is turned off. Devii may toggle read-only only after the user explicitly confirms.
+
+XP awards are wired at the existing content-creation, vote, and follow hook points in the routers and centralized in `award_xp()` / `check_milestone_badges()` (`devplacepy/utils.py`). Existing accounts have their XP and levels backfilled once from prior activity at startup (`init_db()`).
 
 ## Configuration
 
 | Env var | Default | Purpose |
 |---------|---------|---------|
 | `DEVPLACE_DATABASE_URL` | `sqlite:///devplace.db` | Database connection string |
+| `DEVPLACE_DATA_DIR` | `<repo>/var` | Persistent runtime data dir, outside the package and not served via `/static` (zip archives, container workspaces). Point at a volume in production |
 | `SECRET_KEY` | hardcoded fallback | Session signing key |
+| `DEVPLACE_VAPID_SUB` | `mailto:retoor@molodetz.nl` | Contact address in the VAPID JWT `sub` claim |
+| `DEVPLACE_INTERNAL_BASE_URL` | `http://localhost:10500` | Base URL the platform's own services dial for the AI gateway |
+| `DEEPSEEK_API_KEY` / `OPENROUTER_API_KEY` | unset | Upstream provider keys; migrated into the gateway settings on first boot |
+
+### Runtime settings
+
+Operational behavior is tunable live from `/admin/settings` (stored in `site_settings`, no redeploy). The Operational group covers:
+
+| Setting | Default | Effect |
+|---------|---------|--------|
+| `site_url` | empty | Public origin for absolute links (SEO, container ingress `/p/<slug>`); empty derives from the request or `DEVPLACE_SITE_URL` |
+| `rate_limit_per_minute` | `60` | Mutating requests allowed per IP per window |
+| `rate_limit_window_seconds` | `60` | Rolling window for the request limit |
+| `session_max_age_days` | `7` | Standard session cookie lifetime |
+| `session_remember_days` | `30` | Remember-me session cookie lifetime |
+| `registration_open` | `1` | When `0`, new sign-ups are rejected |
+| `maintenance_mode` | `0` | When `1`, non-admins see the maintenance page; admins retain access |
+| `maintenance_message` | scheduled-maintenance text | Message shown during maintenance |
+| `customization_enabled` | `1` | When `0`, no user CSS/JS customization is injected on any page |
+| `customization_js_enabled` | `1` | When `0`, user custom CSS is still served but custom JavaScript is suppressed |
+
+Numeric values are floored to safe minimums so an invalid entry cannot lock out writes or stall services. Consumers read via `get_setting`/`get_int_setting`, which fall back to these defaults when a row is absent.
+
+## Authentication
+
+The website uses a `session` cookie. For automation, every page and action also
+accepts three header-based methods, resolved centrally in `get_current_user`
+(`utils.py`) so they work everywhere with no per-route changes:
+
+- **API key** - `X-API-KEY: <key>`
+- **Bearer** - `Authorization: Bearer <key>`
+- **HTTP Basic** - `Authorization: Basic base64(username-or-email:password)`
+
+Every user has an `api_key` (a uuid7), set at signup and backfilled for existing
+users on startup (and via `devplace apikey backfill`). The key is shown on the
+profile page to its owner (and to admins for any user) with a live **Regenerate**
+button; regenerating invalidates the old key immediately. Invalid credentials
+return `401`; requests with no credentials behave as anonymous (browser redirect to
+login). Full details and copy-paste `curl` examples live at `/docs/authentication.html`.
+
+```bash
+curl -H "X-API-KEY: <your-key>" https://your-host/notifications
+curl -u <username>:<password> -X POST https://your-host/follow/<username>
+```
+
+CLI: `devplace apikey get <username>` / `reset <username>` / `backfill`.
+
+## Content negotiation (HTML or JSON)
+
+Every endpoint that renders a page or returns a redirect also speaks JSON, so anything the
+website does is automatable from the same URLs. A request gets JSON when it sends
+`Accept: application/json` or `Content-Type: application/json`; a normal browser navigation
+(`Accept: text/html`) always gets HTML, so existing behaviour is unchanged (the legacy
+`X-Requested-With: fetch` AJAX header still drives the four engagement endpoints only). JSON responses are defined by Pydantic models in `devplacepy/schemas.py` and built
+from the same context the templates use (sensitive user fields like `email`/`api_key`/
+`password_hash` are never exposed). Page GETs return the page payload; form actions return a
+uniform envelope `{ "ok": true, "redirect": "…", "data": {…} }`; errors return
+`{ "error": { "status", "message" } }` (validation → `422` with a `fields` map, unauthenticated
+JSON → `401`, non-admin → `403`). The core lives in `devplacepy/responses.py`
+(`wants_json`, `respond`, `action_result`). Full details: `/docs/conventions.html`.
+
+```bash
+curl -H "Accept: application/json" https://your-host/feed
+curl -H "Accept: application/json" -X POST -d "content=hi&title=T&topic=devlog" https://your-host/posts/create
+```
+
+## Documentation site
+
+`/docs` serves a server-rendered docs site with a feed-style left sidebar
+(`routers/docs.py`). `/docs/index.html` is the start page and `/docs/authentication.html`
+documents the auth methods; both are curated markdown pages rendered through the shared
+content renderer (markdown + highlight.js) with the current host and, when logged in, your
+own API key and username filled in.
+
+Every other endpoint is documented from a single source of truth, the `API_GROUPS` registry
+in `devplacepy/docs_api.py`. Each group becomes a reference page listing its endpoints with
+copy-paste cURL, JavaScript, and Python examples and an interactive "try it" panel
+(`static/js/ApiTester.js`) that executes the real call from the browser with your API key
+pre-filled. Each panel has a response-format selector (defaulting to JSON) that drives the
+`Accept` header for both the live request and the generated snippets, an always-visible
+**Expected** tab showing the modeled response, and a **Live response** tab for the real
+result. To document a new endpoint, add an entry to `docs_api.py`; the page, sidebar link,
+examples, runner, and expected-response sample are generated automatically. FastAPI's
+built-in Swagger is disabled so `/docs` belongs to this site.
+
+Operator pages are admin-only: `/docs/admin.html` (user/news/settings administration) and
+`/docs/services.html` (Background Services) are hidden from the sidebar and return 404 for
+non-admins. The Background Services page is generated live from the service registry
+(`build_services_group()` over `service_manager.describe_all()`), so every registered service
+and its full configuration are documented automatically - including future services.
 
 ## Background Services
 
-`devplacepy/services/` provides a generic async service framework. Services start on server boot, run at configurable intervals, and are gracefully cancelled on shutdown.
+`devplacepy/services/` provides a generic async service framework. `/admin/services` is a slim index listing each service with its description and live status; clicking a service opens its detail page where everything for that service lives - start/stop, enable-on-boot, run-now, clear logs, adjustable log size, and live status/metrics - organized into **Overview / Configuration / Logs** tabs (configuration grouped into sections). State is stored in the database (desired state in `site_settings`, observed state in the `service_state` table), so control and status are correct even with multiple workers where only one runs the services.
 
 ### Service framework
 
-- **`BaseService`** — abstract class with async run loop, `deque(maxlen=20)` log buffer, `name`, `interval_seconds`
-- **`ServiceManager`** — singleton that registers, starts, and stops all services
-- **`NewsService`** — fetches news from `news.app.molodetz.nl/api`, grades with AI, stores articles >= threshold
+- **`ConfigField`** - declarative parameter spec (type, default, validation, secret) a service uses to declare its editable settings
+- **`BaseService`** - abstract class with a reconciling run loop that honors the persisted `enabled`/command/interval state, plus `config_fields`, `get_config()`, `describe()`, and a log buffer
+- **`ServiceManager`** - singleton: `register`, `describe_all`, `set_enabled`, `send_command`, `save_config`, `supervise`, `shutdown_all`
+- **`NewsService`** - fetches news from `news.app.molodetz.nl/api`, grades with AI, stores articles >= threshold
+- **`BotsService`** - Playwright fleet of AI personas that browse and interact with a DevPlace instance, with live cost/usage metrics (opt-in; install the `bots` extra)
+- **`GatewayService`** - OpenAI-compatible LLM gateway at `/openai/v1/*`, forwarding to DeepSeek (default) with optional vision augmentation; the single point of truth for AI that every other service routes through (enabled by default)
+- **`JobService` / `ZipService` / `ForkService`** - generic async job framework (`services/jobs/`) for heavy, blocking work run off the request path; `ZipService` builds project zip archives in a subprocess, `ForkService` copies a project into a new project owned by the forking user
+- **`ContainerService`** - the admin container manager (`services/containers/`): a reconciling supervisor for container instances, all running one shared prebuilt image
+
+### Container manager (admin only)
+
+`services/containers/` runs supervised container instances from the web UI, the HTTP API, and Devii. It drives the `docker` CLI via async subprocesses behind a pluggable `Backend` interface (a `DockerCliBackend` plus a `FakeBackend` for tests). **There is no in-app image building.** Every instance runs ONE shared prebuilt image, `ppy:latest` (override `DEVPLACE_CONTAINER_IMAGE`), built once with **`make ppy`** from `ppy.Dockerfile`: a Python + Playwright base with a broad set of common libraries preinstalled, the `pravda` (uid 1000) user, the sudo superclone, and `pagent` at `/usr/bin/pagent.py` all baked in. Creating an instance is then an instant `docker run` (it fails fast with a clear error if the `ppy` image has not been built yet). `ContainerService` reconciles desired instance state against `docker ps` each tick (containers are labeled `devplace.instance=<uid>`, so orphans are reaped and no state is lost), applies restart policies, fires cron/interval/one-time schedules, and samples metrics. The container's `/app` is bind-mounted to a persistent project workspace (materialized from the project files) and can be synced back; projects that need extra packages use runtime `pip install` (pravda owns the site-packages, no sudo needed) or add the library to `ppy.Dockerfile` and rerun `make ppy`. A running instance can be **published** with an `ingress_slug`, making its service reachable (HTTP and WebSocket) at `/p/<slug>` through DevPlace. The manager is reached two ways: the admin **Containers** sidebar entry (`/admin/containers`) lists every instance across all projects and opens a dedicated detail page per instance, and each project page carries an admin-only **Containers** button to its own instance manager.
+
+**Security:** this requires mounting the Docker socket, which grants host root. Every run, exec, lifecycle, and schedule operation is administrator-only; `--privileged` is never used and all docker calls are argument-list subprocesses. The service is disabled by default; an admin enables **Containers** on `/admin/services`. CLI: `devplace containers list | reconcile | prune | prune-builds | gc-workspaces` (`prune-builds` is a one-time cleanup that removes legacy per-project images and the old dockerfiles/builds tables).
+
+**Runtime data** (container workspaces and zip archives) lives in `DEVPLACE_DATA_DIR` (default `var/`), **outside the package and never served via `/static`**. The docker daemon must be able to bind-mount the data dir for `/app`.
+
+### Async job framework and zip downloads
+
+`services/jobs/` is the standard way to run blocking work asynchronously and hand the caller a result URL. A shared `jobs` table is the queue (discriminated by `kind`); `queue.enqueue()` inserts a `pending` row from any worker, the lock-owning worker processes jobs in `JobService.run_once` (reap, recover orphans, refill up to a concurrency limit, prune expired), and status is polled from the database. Retention is built in: each job service deletes its own expired artifacts via a `cleanup` hook (default 7 days, admin-configurable). To add a kind, subclass `JobService`, set `kind`, and implement `process()` and `cleanup()`.
+
+`ZipService` archives a project (or a file/folder subtree) into `{crc32}.{slug}.zip` using the standalone `zip_worker` subprocess, with a CRC32 over the output. The frontend `app.zipDownloader` (any `data-zip-download` element, plus the files context menu) enqueues, polls `/zips/{uid}`, and downloads from `/zips/{uid}/download`. The job uid is the capability token, so anyone with the link can download. CLI: `devplace zips prune` (delete expired) / `devplace zips clear` (delete all).
+
+`ForkService` copies a project into a new project owned by the forking user. The **Fork** button on the project page (any signed-in user) prompts for a name; the job creates the destination project, duplicates the entire virtual filesystem, and records a directional `project_forks` relation so each fork shows a "Forked from X" link. The frontend `app.projectForker` enqueues, polls `/forks/{uid}`, and redirects to the new project once it is done; on failure the partially created project is rolled back. The forked project is permanent, so retention removes only the job tracking row. CLI: `devplace forks prune` / `devplace forks clear` (job rows only).
 
 ### Adding a service
 
-Create a class extending `BaseService`, override `run_once()`, register in `main.py`:
+Create a class extending `BaseService`, declare its `config_fields`, override `run_once()` (read parameters via `self.get_config()`), and register in `main.py`:
 ```python
 service_manager.register(YourService())
 ```
-The service appears at `/services` with live status and log tail.
+The service appears at `/admin/services` with controls, a generated config form, live status, and the log tail - no template, JS, or router changes.
 
 ### News service
 
 Fetches news from a configurable API, grades each article via AI, stores ALL articles in the `news` table (nothing is silently skipped). Articles with grade >= the configurable threshold are auto-published; the rest go to draft. Admin can manually publish/draft, toggle for landing page appearance, and delete. Images are extracted from article URLs.
 
-Configuration via admin site settings:
+Configuration on the Services tab (`/admin/services`):
 
-| Setting key | Default | Purpose |
-|-------------|---------|---------|
+| Parameter | Default | Purpose |
+|-----------|---------|---------|
 | `news_grade_threshold` | `7` | Minimum AI grade for auto-publish |
 | `news_api_url` | `https://news.app.molodetz.nl/api` | News source |
-| `news_ai_url` | `https://openai.app.molodetz.nl/v1/chat/completions` | AI grading endpoint |
-| `news_ai_model` | `molodetz` | AI model |
+| `news_ai_url` | `http://localhost:10500/openai/v1/chat/completions` | AI grading endpoint (the internal gateway) |
+| `news_ai_model` | `molodetz` | Generic model name; the gateway maps it to the real model |
+| `news_ai_key` | internal key | AI API key (`NEWS_AI_KEY` env, then the auto-generated gateway internal key) |
+| `news_service_interval` | `3600` | Seconds between fetch cycles (min 60) |
 
 News articles have detail pages at `/news/{slug}` with full comment support (same component as posts/projects). The landing page can display curated articles toggled from admin.
 
-CLI: `devplace news clear` — delete all news from local database.
+CLI: `devplace news clear` - delete all news from local database.
+
+### Bots service
+
+A Playwright-driven fleet of AI personas (`devplacepy/services/bot/`) that browse and interact with a DevPlace instance: posting, commenting, voting, reacting, creating gists/projects, filing bugs, following, and messaging. It is the former standalone `dpbot.py`, refactored into a package and managed entirely from the Services tab. Disabled by default.
+
+Install the extra and the browser binary, then enable it on `/admin/services`:
+
+```bash
+pip install -e ".[bots]"
+playwright install chromium
+```
+
+Configuration on the Services tab:
+
+| Parameter | Default | Purpose |
+|-----------|---------|---------|
+| `bot_fleet_size` | `1` | Number of concurrent bot accounts |
+| `bot_headless` | enabled | Run Chromium headless |
+| `bot_max_actions` | `0` | Stop a bot after N actions (0 = forever) |
+| `bot_base_url` | `https://pravda.education` | Target DevPlace instance |
+| `bot_api_url` | `http://localhost:10500/openai/v1/chat/completions` | LLM endpoint (the internal gateway) |
+| `bot_news_api` | `https://news.app.molodetz.nl/api` | Article source |
+| `bot_model` | `molodetz` | Generic model name; the gateway maps it to the real model |
+| `bot_api_key` | internal key | LLM key (defaults to the auto-generated gateway internal key) |
+| `bot_input_cost_per_1m` / `bot_output_cost_per_1m` | `0.27` / `1.10` | Token pricing for live cost tracking |
+
+The Services tab shows live fleet metrics (bots running, total cost, LLM calls, tokens, posts/comments/votes) and a per-bot breakdown that update on the page poll. Each bot keeps a stable identity and cumulative cost in `~/.devplace_bots/state_slot{N}.json`.
+
+### LLM gateway service
+
+`GatewayService` (`devplacepy/services/openai_gateway/`) exposes an OpenAI-compatible
+endpoint at `/openai/v1/chat/completions` (plus a transparent `/openai/v1/*`
+passthrough). It forwards to the configured upstream (DeepSeek by default) and, when a
+request carries image content, first describes the image(s) via a vision model
+(OpenRouter/Gemma by default) and inlines the description as text - so a vision-less
+upstream can still answer. Enabled by default and configurable on `/admin/services`.
+
+It is the **single point of truth for AI**: the news, bots, and guest Devii sessions call this
+gateway by default instead of an external provider, send the generic model name `molodetz`,
+and authenticate with an auto-generated internal key. The real provider URLs, models, and keys
+live only here, so providers and backends are switched in one place. `DEEPSEEK_API_KEY` and
+`OPENROUTER_API_KEY` are migrated into the editable settings on first boot, and the value in use
+is shown. The internal base URL the other services dial is `DEVPLACE_INTERNAL_BASE_URL`
+(default `http://localhost:10500`).
+
+Authenticated users may call the gateway with their own API key (the `Allow users` setting,
+on by default), and Devii operating a signed-in user authenticates its LLM calls with that
+user's key. Every call is therefore attributed to the user, so their full AI spend (through
+Devii or direct API calls) is tracked and limitable per user. Administrators see a per-user
+"AI usage (last 24h)" panel on each profile page - request volume, success and error rates,
+token totals, cost, average latency and throughput, a per-model breakdown, a 30-day cost
+projection, and a quota bar - served from `GET /admin/users/{uid}/ai-usage`. A non-admin user
+sees only one figure on their own profile: the percentage of their daily AI quota used so far.
+The same rule governs the Devii assistant: monetary figures (cost, pricing, spend, limit) are
+disclosed only to administrators, while members and guests can see only the percentage of their
+24-hour quota used. The `/devii/usage` endpoint returns that percentage and the day's turn count to
+everyone, and includes dollar figures only for administrators.
+
+Configuration on the Services tab:
+
+| Parameter | Default | Purpose |
+|-----------|---------|---------|
+| `gateway_upstream_url` | `https://api.deepseek.com/chat/completions` | Where requests are forwarded |
+| `gateway_model` | `deepseek-v4-flash` | Real model sent upstream (what `molodetz` maps to); 1M-token context, 384K max output |
+| `gateway_force_model` | on | Override the client-requested model (and the `molodetz` alias) |
+| `gateway_api_key` | migrated from env | Upstream key, shown and editable (auto-migrated from `DEEPSEEK_API_KEY`/`OPENROUTER_API_KEY`) |
+| `gateway_instances` | `4` | Max concurrent upstream forwards per worker (pool + semaphore) |
+| `gateway_timeout` | `300` | Upstream timeout (seconds); minimum five minutes; also bounds the vision describe-image call |
+| `gateway_vision_enabled` / `_url` / `_model` / `_key` / `_cache_size` | on / OpenRouter / Gemma / env / 256 | Image-description augmentation |
+| `gateway_require_auth` | on | When off, the gateway is open |
+| `gateway_allow_admins` / `gateway_allow_users` | on / off | Which DevPlace users may call it (any auth scheme) |
+| `gateway_access_key` | empty | A standalone key (sent as `X-API-KEY`/Bearer) that always grants access |
+| `gateway_internal_key` | auto (uuid4) | Auto-generated on boot; DevPlace's own services authenticate with this. Clear and restart to rotate |
+| `gateway_price_cache_hit_per_m` / `_cache_miss_per_m` / `_output_per_m` | 0.0028 / 0.14 / 0.28 | Chat cost per 1M tokens, used when the upstream returns no native cost (DeepSeek) |
+| `gateway_vision_price_input_per_m` / `_output_per_m` | 0 / 0 | Vision cost per 1M tokens, used only when the vision upstream returns no native cost |
+| `gateway_max_retries` / `gateway_retry_backoff_ms` | 2 / 250 | Retry attempts and linear backoff on timeout, connection error, or upstream 5xx |
+| `gateway_circuit_threshold` / `gateway_circuit_cooldown_seconds` | 5 / 30 | Consecutive failures before the circuit breaker opens, and its cooldown |
+| `gateway_usage_retention_hours` | 720 | How long per-call usage rows are kept before pruning (30 days) |
+| `gateway_model_context_map` | JSON map | Model to max-context-tokens map for context-window utilization tracking |
+
+Callers authenticate with the static `gateway_access_key`, the auto-generated
+`gateway_internal_key` (used by the platform's own services), or - when allowed - their
+DevPlace credentials via any scheme (`X-API-KEY`, Bearer, Basic). The endpoint is
+exempt from the per-IP rate limit and the maintenance gate, and its concurrency is
+bounded by `gateway_instances`; scale further with uvicorn workers. The Services tab
+shows live request/error/latency/vision-call counters.
+
+#### Usage and cost metrics
+
+Every upstream call (chat, vision, and passthrough) is recorded to `gateway_usage_ledger`
+with its tokens, cost, latency breakdown, status, and caller. Cost is taken from the
+upstream native `cost` field when present (OpenRouter) and computed from the configured
+per-million pricing otherwise (DeepSeek, which reports no cost). The admin **AI usage**
+page (`/admin/ai-usage`) reports per-hour and 24h metrics - request volume and throughput,
+token usage with averages and percentiles (p50/p90/p95/p99), latency (upstream round-trip,
+gateway overhead, semaphore queue wait, connection establishment), error rates by category,
+cost (per model, per caller, input vs output, projected monthly burn, caching savings),
+caller behavior, and an hourly breakdown - divided and combined. The gateway also retries
+transient upstream failures and opens a circuit breaker after repeated failures. Time to
+first token and inter-token latency are not reported because the gateway forwards
+non-streaming to the upstream.
+
+```bash
+curl -H "X-API-KEY: <gateway_access_key>" \
+  -d '{"messages":[{"role":"user","content":"hi"}]}' \
+  https://your-host/openai/v1/chat/completions
+```
+
+### Devii assistant service
+
+`DeviiService` (`devplacepy/services/devii/`) is an in-platform agentic assistant exposed
+as a WebSocket terminal at `/devii/ws`, reachable from the **Devii** item in the user
+dropdown menu and embedded on the documentation page for guests. It runs a ReAct loop
+(planning, reflection, a verification gate, self-learning memory, autonomous tasks) over a
+declarative tool catalog. A signed-in user's Devii operates their **own** DevPlace account -
+the agent authenticates to this instance with the user's `api_key` - while guests get a
+sandbox session. Disabled by default; enable and configure it on `/admin/services`.
+
+Sessions are durable. A logged-in user's conversation is keyed by user uid, shared live
+across every tab, window, and device (each turn is broadcast to all connected sockets), and
+persisted to `devii_conversations` so it survives a server restart - on reconnect the prior
+conversation is rehydrated. Guests get a non-persistent session keyed by a `devii_guest`
+cookie. To keep one logical session and one cost/broadcast source per user, the `/devii/ws`
+endpoint is served only by the worker that holds the background-service lock; other workers
+close the socket so the client reconnects to the owning worker.
+
+Spend is capped per owner over a rolling 24 hours. Every turn appends a row to
+`devii_usage_ledger` (the authoritative source for the cap - the in-memory cost tracker is
+display-only) and an audit row to `devii_turns`. The cap is checked before each turn.
+
+Configuration on the Services tab:
+
+| Parameter | Default | Purpose |
+|-----------|---------|---------|
+| `devii_ai_url` | `https://openai.app.molodetz.nl/v1/chat/completions` | OpenAI-compatible reasoning endpoint |
+| `devii_ai_model` | `molodetz` | Model name |
+| `devii_ai_key` | env fallback (`DEVII_AI_KEY`) | AI API key |
+| `devii_base_url` | this instance's origin | Platform Devii drives via each user's API key |
+| `devii_plan_required` / `devii_verify_required` | on / on | Enforce plan-first and verify-after-mutation |
+| `devii_max_iterations` | `40` | Tool-loop iterations per turn |
+| `devii_timeout` | `300` | Read timeout (seconds) for each AI chat-completions call; minimum five minutes |
+| `devii_fetch_timeout` | `300` | Read timeout (seconds) for the fetch tool; minimum five minutes |
+| `devii_user_daily_usd` | `1.0` | Rolling 24h spend cap per signed-in user |
+| `devii_guest_daily_usd` | `0.05` | Rolling 24h spend cap per guest |
+| `devii_admin_daily_usd` | `0.0` | Rolling 24h spend cap per administrator (`0` = unlimited; admins are exempt by default) |
+| `devii_guests_enabled` | on | Allow anonymous guests |
+| `devii_price_cache_hit` / `_cache_miss` / `_output` | `0.0028` / `0.14` / `0.28` | USD per 1M tokens (drives the cap) |
+| `devii_rsearch_enabled` | on | Enable the external web search tools (`rsearch_*`) |
+| `devii_rsearch_url` | `https://rsearch.app.molodetz.nl` | Base URL of the web search service those tools call |
+| `devii_rsearch_timeout` | `300` | Read timeout (seconds) for `rsearch_*` calls; web-grounded answers can take minutes; minimum five minutes |
+
+Beyond the platform tools, Devii has external **web search** tools - `rsearch` (web/image
+search), `rsearch_answer` (a web-grounded AI answer with sources), `rsearch_chat` (direct AI
+chat), and `rsearch_describe_image` (vision). These reach an external public service rather
+than this platform, so platform tools are always preferred; Devii uses them only when the user
+explicitly asks to search the web or an outside source. They are gated by `devii_rsearch_enabled`
+and the service URL is configurable via `devii_rsearch_url`.
+
+A `devii` console script ships the same agent as an interactive terminal:
+
+```bash
+pip install -e .
+devii --api-key <your DevPlace api_key> --base-url https://your-host
+devii -p "List my unread notifications as a bullet list."   # one-shot
+```
+
+### Site customization (per-user CSS/JS)
+
+Each user can reshape the site to taste by injecting their own **CSS** (look) and
+**JavaScript** (behaviour), scoped either to a single **page type** (every post page, every
+project page) or **globally** (every page). Customizations apply only to that user's own
+sessions; a signed-out guest gets customizations scoped to their `devii_guest` session cookie.
+This is the user's own code running in their own browser, like a userscript, so it never affects
+anyone else's view.
+
+It is configured conversationally through Devii. Ask for a change and Devii previews it live in
+your browser, then before saving it asks whether to apply it to the current page type or the
+whole site, and only persists once you confirm. Overrides are stored in the `user_customizations`
+table (one row per owner, scope, and language; cached with cross-worker invalidation) and injected
+server-side: custom CSS is the last `<style>` in `<head>` so it overrides the design system, and
+custom JavaScript runs after the application boots with access to the global `app`. Devii tools:
+`customize_list`, `customize_get`, `customize_set_css`, `customize_set_js`, and `customize_reset`
+(restores the default look and behaviour). Operators can disable the feature site-wide with the
+`customization_enabled` / `customization_js_enabled` settings.
+
+### User-defined tools (vibe new functionality)
+
+Users can invent new Devii capabilities by describing them in natural language, with no code. Telling
+Devii "when I say woeii, search the web for a cat picture matching my description and set it as my
+page background" makes it call `tool_create` and store a virtual tool. The tool then appears in
+Devii's own toolset, and on the next turn "woeii orange" calls it: the handler re-runs Devii itself
+(a self-evaluating sub-agent) on the stored prompt plus the user's input, with full tool access, and
+returns the result. Each tool takes a single free-form `input` string; the stored prompt directs what
+to do with it.
+
+Virtual tools are owner-scoped (persistent in the DB for signed-in users in `devii_virtual_tools`,
+session-only for guests) and never run in anyone else's session. The full CRUD is exposed as Devii
+tools: `tool_create`, `tool_list`, `tool_get`, `tool_update` (including enable/disable), and
+`tool_delete`. A general `eval(prompt)` tool runs Devii on any prompt and returns the result; it is
+the same engine that powers virtual tools. Self-evaluation depth is bounded so a tool cannot loop by
+calling itself.
+
+## Push notifications & PWA
+
+Authenticated users can receive native web push notifications, and the site is an
+installable Progressive Web App. Push uses only standard libraries (`cryptography`,
+`PyJWT`, `httpx`) against the Web Push Protocol - no third-party push wrapper.
+
+### Events
+
+Every event that already produces an in-app notification also sends a web push,
+because both share a single funnel - `create_notification()` in `utils.py`:
+
+| Event | Recipient |
+|-------|-----------|
+| Direct message received | receiver |
+| Comment on your post | post author |
+| Reply to your comment | comment author |
+| `@mention` in any content | mentioned user |
+| Upvote on your content | content owner |
+| New follower | followed user |
+
+`create_notification` schedules delivery as a fire-and-forget async task, so a dead
+subscription or push-service error never blocks the triggering request. Delivery
+(`push.notify_user`) iterates a user's subscriptions, encrypts the payload
+(legacy `aesgcm` content encoding), and POSTs to each endpoint; subscriptions that
+return `404`/`410` are soft-deleted.
+
+### VAPID keys
+
+The server identity is three PEM files generated once at startup in the repository
+root: `notification-private.pem`, `notification-private.pkcs8.pem`,
+`notification-public.pem`. They are git-ignored.
+
+**These keys are the application's identity to the push services. If they are lost or
+regenerated, every existing subscription becomes permanently undeliverable.** Persist
+them across deployments and back them up; do not regenerate them.
+
+### Opt-in
+
+The browser requires the first permission prompt to originate from a user gesture, so
+opt-in is exposed as a button in both the top navigation (bell-with-slash icon) and on
+the `/notifications` page. After opt-in, the subscription is refreshed silently on
+every page load. `PushManager.js` owns registration, subscription, and the opt-in UI.
+
+### PWA
+
+`manifest.json` (192/512 and maskable icons), `service-worker.js`, and an install
+button (`PwaInstaller.js`) make the app installable. The service worker uses a
+network-first strategy for navigations and falls back to `static/offline.html` when
+offline. Installation requires a secure origin (HTTPS, or `localhost` for development).
+
+| File | Role |
+|------|------|
+| `devplacepy/push.py` | VAPID keys, payload encryption, send, register |
+| `devplacepy/routers/push.py` | `/push.json`, `/service-worker.js`, `/manifest.json` |
+| `static/js/PushManager.js` | Service-worker registration + subscribe + opt-in UI |
+| `static/js/PwaInstaller.js` | `beforeinstallprompt` capture + install button |
+| `static/service-worker.js` | Receives push, shows notification, offline fallback |
+| `static/manifest.json` | PWA manifest (icons, display, theme) |
+| `static/offline.html` | Offline fallback page |
 
 ## Database
 
@@ -123,12 +534,14 @@ PRAGMA temp_store=MEMORY;      -- temp tables in memory
 PRAGMA mmap_size=268435456;    -- 256MB memory map for reads
 ```
 
-All indexes are created via `CREATE INDEX IF NOT EXISTS` wrapped in try/except — safe to run on every startup regardless of table state.
+All indexes are created via `CREATE INDEX IF NOT EXISTS` wrapped in try/except - safe to run on every startup regardless of table state.
+
+SQLite is synchronous by design and will never be made async. It is more than fast enough for this platform: the database is a local file with WAL, a 30s busy timeout, and a 256MB memory map, so queries are sub-millisecond. The synchronous database layer is a deliberate architectural decision, not a limitation, and is not subject to change.
 
 ## Testing
 
-- **148 tests** across 14 files: Playwright integration + unit tests
-- Playwright (NOT pytest-playwright plugin — conflicts, uninstall it)
+- **419 tests** across 34 files: Playwright integration + in-process unit tests
+- Playwright (NOT pytest-playwright plugin - conflicts, uninstall it)
 - Server starts as subprocess on port 10501 with isolated temp database
 - Test users `alice_test` / `bob_test` seeded via HTTP at session start
 - Tests stop at first failure (`-x` flag)
@@ -137,34 +550,97 @@ All indexes are created via `CREATE INDEX IF NOT EXISTS` wrapped in try/except
 
 ### Key test patterns
 
-- Every `page.goto()` and `page.wait_for_url()` uses `wait_until="domcontentloaded"` — avatar images don't block test execution
+- Every `page.goto()` and `page.wait_for_url()` uses `wait_until="domcontentloaded"` - avatar images don't block test execution
 - Session-scoped browser context with per-test cookie clearing
 - `page.locator(...).wait_for(state="visible")` preferred over bare selectors
 
 ## Avatars
 
-Uses [Multiavatar](https://github.com/multiavatar/multiavatar-python) — generates deterministic SVG avatars locally from a seed string (the username). No external API calls, no network dependency. Generation takes <5ms. Results are cached in-memory (cleared on server restart).
+Uses [Multiavatar](https://github.com/multiavatar/multiavatar-python) - generates deterministic SVG avatars locally from a seed string (the username). No external API calls, no network dependency. Generation takes <5ms. Results are cached in-memory (cleared on server restart).
 
 Avatar URL format: `/avatar/multiavatar/{username}?size={size}`
 
+## Deployment
+
+Production runs on a single host via Docker Compose: an **app** container (Uvicorn, 2 workers) behind an **nginx** container that terminates HTTP, serves static assets, and proxies the rest.
+
+### Shared database and uploads
+
+The app container bind-mounts the host project directory (`./` to `/app`) and runs as the host user (`DEVPLACE_UID`/`DEVPLACE_GID`, default `1000`). Because `config.py` resolves the database to an absolute path under the project root, the container reads and writes the **same `devplace.db`** as `make dev`, plus the same `static/uploads/`, `devii_lessons.db`, `devii_tasks.db`, VAPID keys, and `devplace-services.lock`. SQLite WAL mode allows the concurrent access, and the file lock (`fcntl.flock` on `devplace-services.lock`) guarantees only one process - dev or prod - runs the background services (news, bots, Devii hub), so they never double-fire.
+
+This requires prod and dev to be on the **same host/filesystem**; SQLite is a local-file database and cannot be shared across machines. No `DEVPLACE_DATABASE_URL` override is set, so nothing diverges.
+
+### First deploy
+
+```bash
+cp .env.example .env        # set SECRET_KEY; adjust PORT, DEVPLACE_SITE_URL
+make docker-build           # build the image (installs the docker CLI)
+make docker-up              # start (creates ./var, mounts the socket)
+```
+
+Open `http://<host>:${PORT}` (default 10500). `make docker-logs` tails output; `make docker-down` stops.
+
+`make docker-build` and `make docker-up` always apply the `docker-compose.containers.yml` overlay, so the admin-only Container Manager works with no extra setup. Mounting the docker socket grants the app **root on the host**; the feature itself stays disabled until an admin enables **Containers** on `/admin/services` (and the `ppy` image is built once with `make ppy`), but treat the whole deployment as trusted-admins-only. Always update through the make targets - a plain `docker compose up -d` drops the overlay and silently removes the socket and CLI.
+
+### Updating
+
+```bash
+git pull
+make docker-up              # restart with new code (bind-mounted, no rebuild)
+make docker-build && \
+  make docker-up            # only when dependencies in pyproject.toml change
+```
+
+The `make deploy` target fast-forwards the `production` branch (`git checkout production && git merge master && git push origin production`); pull that branch on the server.
+
+### Container Manager wiring (what the overlay does)
+
+The Container Manager drives the host Docker daemon, so `make docker-build`/`make docker-up` apply `docker-compose.containers.yml` automatically. The make targets derive everything the overlay needs - `DOCKER_GID` from the socket and `DEVPLACE_DATA_DIR` as the project's own `./var` at its real host path - so no `sudo`, no `/srv` dir, and no manual `.env` editing are required. (Override `DEVPLACE_DATA_DIR` or `DOCKER_GID` on the make command line to point elsewhere.)
+
+What the overlay (`docker-compose.containers.yml`) changes:
+
+- **Docker CLI in the image** via the `INSTALL_DOCKER_CLI=true` build arg (the base image stays lean).
+- **Docker socket** mounted into the app container. This grants the app **root on the host** - every build/run/exec is admin-only, `--privileged` is never used, and all docker calls are argument-list subprocesses, but treat the whole feature as trusted-admins-only.
+- **Socket permissions:** the app runs as UID 1000, so the overlay adds the host `docker` group via `group_add`. `make` reads the gid straight from `/var/run/docker.sock` (`stat -c '%g'`), the exact group that owns the socket.
+- **Data dir at a consistent path (critical).** When the app (in its container) runs `docker run -v <path>:/app`, the daemon resolves `<path>` against the **host**, not the app container. So the workspace/data dir must be mounted at the **same absolute path** on host and in the container - the make targets set `DEVPLACE_DATA_DIR` to the project's `./var` (an absolute host path) and mount it at that identical path on both sides. (Build contexts go through the docker API as a tarball, so they can stay in the container's temp dir - only the `/app` bind mount needs path consistency.)
+- **Ingress reach:** published container ports live on the **host**, so the overlay sets `DEVPLACE_CONTAINER_PROXY_HOST=host.docker.internal` (with `extra_hosts: host-gateway`) so the `/p/<slug>` proxy can reach them. On a bare-metal `make prod` deploy the app is already on the host, so the default `127.0.0.1` works and no overlay is needed (just install the docker CLI and run the services).
+
+Then enable **Container builds** and **Containers** on `/admin/services`. Builds default to `--network=host` (configurable on the service) so pip can reach PyPI; set the build network to empty to use the docker default.
+
+### nginx specifics
+
+- **WebSockets:** `/devii/ws` (the Devii terminal) and `/p/` (container ingress) are proxied with `Upgrade`/`Connection` headers and 1h timeouts; `/p/` also disables buffering for streaming.
+- **Uploads:** `/static/uploads/` is served with `X-Content-Type-Options: nosniff` and a `Content-Disposition` that mirrors the app's `UploadStaticFiles` control: known-safe media (images, video, audio) is served `inline` so it embeds and plays in the page, while every other type is forced to `attachment` so it downloads instead of rendering (stored-XSS defense). SVG is deliberately excluded from the inline set.
+- **Upload size:** `NGINX_MAX_BODY_SIZE` (default `50m`) must be **>= the admin-configurable `max_upload_size_mb`** (Admin -> Settings), or large uploads are rejected with HTTP 413 before reaching the app.
+- **Micro-cache:** off by default; enable with `NGINX_CACHE_ENABLED=true`.
+
+### Bare-metal alternative
+
+`make prod` runs the same app without containers (`uvicorn ... --workers 2 --proxy-headers`) from the project root, sharing the identical database and files. Note it binds port 10500, so it conflicts with the Docker front door on the same port - run one, or set a different `PORT`.
+
+### Multi-worker safety
+
+The app runs correctly under multiple Uvicorn workers (independent processes sharing only the DB and lock files). Cross-worker invalidation of the auth and settings caches is coordinated through a `cache_state` version table (propagation bounded to ~2s); the rate limiter enforces `ceil(limit / DEVPLACE_WEB_WORKERS)` per process so the aggregate honours the configured value - **set `DEVPLACE_WEB_WORKERS` to match `--workers`** (done in `make prod` and the Dockerfile). First-boot DB init and VAPID key generation are `flock`-serialized. Full detail and the design rules are in the admin docs at `/docs/production-concurrency.html`.
+
 ## CI/CD
 
 Gitea Actions workflow at `.gitea/workflows/test.yaml`:
-- Runs on push/PR to main
-- Sets up Python 3.13, installs dependencies + Playwright
-- Validates all source files with `hawk`
-- Runs all tests with fail-fast
-- Uploads failure screenshots as artifacts
+- Runs on push/PR to `master`
+- Sets up Python 3.13, installs dependencies + Playwright Chromium
+- Runs the full test suite under coverage (`coverage run -m pytest -p no:xdist`)
+- Builds and publishes the coverage HTML as an artifact on every run
+- Uploads failure screenshots as artifacts when a test fails
+
+Changes are promoted through automated DTAP streets: Development (`make dev`), Test (the CI suite + coverage on `master`), Acceptance (the `master` to `production` promotion via `make deploy`), and Production (the Docker Compose stack on the host). Only CI-green `master` commits reach `production`.
 
 ## Feature workflow
 
 1. Implement the feature (router + template + CSS + JS)
-2. `hawk .` — validate all source files
-3. `make test` — run all tests (fail-fast)
+2. Validate each touched language (Python compiles/imports, JS parses, CSS and HTML balance)
+3. `make test` - run all tests (fail-fast)
 4. Add Playwright tests in `tests/test_*.py` for new functionality
-5. For visual features: `falcon take --output /tmp/verify.png && falcon describe /tmp/verify.png`
-6. Update `AGENTS.md` and `README.md` if new conventions were introduced
+5. Update `AGENTS.md` and `README.md` if new conventions were introduced
 
 ## License
 
-MIT — DevPlace
+MIT - DevPlace

devplacepy/attachments.py

@@ -1,33 +1,113 @@
 import logging
-from datetime import datetime
+from datetime import datetime, timezone
 from pathlib import Path
 from PIL import Image
 from io import BytesIO
-from devplacepy.database import get_table
+from devplacepy.database import get_table, db, get_setting
 from devplacepy.config import STATIC_DIR
 from devplacepy.utils import generate_uid
 
 logger = logging.getLogger(__name__)
-ATTACHMENTS_DIR = STATIC_DIR / "uploads" / "attachments"
+
+UPLOADS_DIR = STATIC_DIR / "uploads"
+ATTACHMENTS_DIR = UPLOADS_DIR / "attachments"
 THUMBNAIL_SIZE = (200, 200)
 THUMBNAIL_QUALITY = 80
 IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".tiff"}
 THUMBNAIL_EXTENSIONS = IMAGE_EXTENSIONS - {".gif"}
+POST_IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".webp"}
+
+ALLOWED_UPLOAD_TYPES = {
+    ".jpg": "image/jpeg",
+    ".jpeg": "image/jpeg",
+    ".png": "image/png",
+    ".gif": "image/gif",
+    ".webp": "image/webp",
+    ".bmp": "image/bmp",
+    ".tiff": "image/tiff",
+    ".pdf": "application/pdf",
+    ".zip": "application/zip",
+    ".mp4": "video/mp4",
+    ".webm": "video/webm",
+    ".ogv": "video/ogg",
+    ".mov": "video/quicktime",
+    ".m4v": "video/x-m4v",
+    ".mp3": "audio/mpeg",
+    ".txt": "text/plain",
+    ".py": "text/x-python",
+    ".js": "text/javascript",
+    ".css": "text/css",
+    ".md": "text/markdown",
+}
+
+FILE_ICONS = {
+    ".pdf": "\U0001F4C4",
+    ".zip": "\U0001F4E6",
+    ".gz": "\U0001F4E6",
+    ".tar": "\U0001F4E6",
+    ".rar": "\U0001F4E6",
+    ".7z": "\U0001F4E6",
+    ".mp4": "\U0001F3AC",
+    ".webm": "\U0001F3AC",
+    ".ogv": "\U0001F3AC",
+    ".mov": "\U0001F3AC",
+    ".m4v": "\U0001F3AC",
+    ".mp3": "\U0001F3B5",
+    ".py": "\U0001F4BB",
+    ".js": "\U0001F4BB",
+    ".ts": "\U0001F4BB",
+    ".html": "\U0001F4BB",
+    ".css": "\U0001F4BB",
+    ".json": "\U0001F4BB",
+    ".md": "\U0001F4BB",
+    ".csv": "\U0001F4CA",
+    ".xls": "\U0001F4CA",
+    ".xlsx": "\U0001F4CA",
+    ".doc": "\U0001F4DD",
+    ".docx": "\U0001F4DD",
+    ".txt": "\U0001F4C4",
+    ".exe": "\u2699",
+    ".bin": "\u2699",
+}
+DEFAULT_FILE_ICON = "\U0001F4CE"
 
-def _get_setting(key, default):
-    row = get_table("site_settings").find_one(key=key)
-    return row["value"] if row else default
 
 def _get_max_upload_bytes():
-    return int(_get_setting("max_upload_size_mb", "10")) * 1024 * 1024
+    return int(get_setting("max_upload_size_mb", "10")) * 1024 * 1024
+
+
+def allowed_extensions():
+    raw = get_setting("allowed_file_types", "").strip()
+    if raw:
+        return {
+            ext if ext.startswith(".") else f".{ext}"
+            for ext in (part.strip().lower() for part in raw.split(","))
+            if ext
+        }
+    return set(ALLOWED_UPLOAD_TYPES)
+
+
+def is_extension_allowed(ext):
+    return ext in allowed_extensions()
+
 
 def _directory_for(uid):
     return f"{uid[:2]}/{uid[2:4]}"
 
+
 def _detect_mime(file_bytes, original_filename):
     ext = Path(original_filename).suffix.lower()
-    m = {".jpg":"image/jpeg",".jpeg":"image/jpeg",".png":"image/png",".gif":"image/gif",".webp":"image/webp",".svg":"image/svg+xml",".bmp":"image/bmp",".tiff":"image/tiff",".pdf":"application/pdf",".zip":"application/zip",".mp4":"video/mp4",".mp3":"audio/mpeg",".txt":"text/plain",".py":"text/x-python",".js":"text/javascript",".html":"text/html",".css":"text/css",".md":"text/markdown"}
-    return m.get(ext, "application/octet-stream")
+    return ALLOWED_UPLOAD_TYPES.get(ext, "application/octet-stream")
+
+
+def _image_dimensions(file_bytes):
+    try:
+        img = Image.open(BytesIO(file_bytes))
+        return img.width, img.height
+    except Exception as e:
+        logger.warning(f"Could not read image dimensions: {e}")
+        return None, None
+
 
 def _generate_thumbnail(file_bytes, thumb_path):
     try:
@@ -45,95 +125,202 @@ def _generate_thumbnail(file_bytes, thumb_path):
         logger.warning(f"Thumbnail generation failed: {e}")
         return None
 
+
+def save_inline_image(file_bytes, original_filename):
+    if len(file_bytes) > _get_max_upload_bytes():
+        logger.warning(f"Inline image too large: {original_filename}")
+        return None
+    ext = Path(original_filename).suffix.lower()
+    if ext not in POST_IMAGE_EXTENSIONS:
+        logger.warning(f"Unsupported inline image type: {ext}")
+        return None
+    UPLOADS_DIR.mkdir(parents=True, exist_ok=True)
+    filename = f"{generate_uid()}{ext}"
+    (UPLOADS_DIR / filename).write_bytes(file_bytes)
+    logger.info(f"Inline image saved: {filename}")
+    return filename
+
+
+def delete_inline_image(filename):
+    if not filename:
+        return
+    try:
+        (UPLOADS_DIR / filename).unlink(missing_ok=True)
+    except Exception as e:
+        logger.warning(f"Failed to delete inline image {filename}: {e}")
+
+
 def store_attachment(file_bytes, original_filename, user_uid):
     if len(file_bytes) > _get_max_upload_bytes():
         return None
+    ext = Path(original_filename).suffix.lower()
+    if not is_extension_allowed(ext):
+        return None
+
     uid = generate_uid()
-    ext = Path(original_filename).suffix.lower() or ".bin"
     stored_name = f"{uid}{ext}"
     directory = _directory_for(uid)
     file_dir = ATTACHMENTS_DIR / directory
     file_dir.mkdir(parents=True, exist_ok=True)
     (file_dir / stored_name).write_bytes(file_bytes)
+
     mime = _detect_mime(file_bytes, original_filename)
-    is_img = mime.startswith("image/")
-    img_w, img_h = None, None
-    thumb = None
-    if is_img:
-        try:
-            img = Image.open(BytesIO(file_bytes))
-            img_w, img_h = img.width, img.height
-            if ext not in (".gif",):
-                thumb_name = f"{uid}_thumb.jpg"
-                r = _generate_thumbnail(file_bytes, file_dir / thumb_name)
-                if r: thumb = r
-        except: pass
+    is_image = mime.startswith("image/")
+    image_width, image_height = None, None
+    thumbnail = None
+    if is_image:
+        image_width, image_height = _image_dimensions(file_bytes)
+        if ext not in (".gif",):
+            thumbnail = _generate_thumbnail(file_bytes, file_dir / f"{uid}_thumb.jpg")
+
     get_table("attachments").insert({
-        "uid": uid, "target_type": "", "target_uid": "", "user_uid": user_uid,
-        "original_filename": original_filename, "stored_name": stored_name,
-        "directory": directory, "file_size": len(file_bytes), "mime_type": mime,
-        "image_width": img_w, "image_height": img_h,
-        "has_thumbnail": 1 if thumb else 0,
-        "created_at": datetime.utcnow().isoformat(),
+        "uid": uid,
+        "target_type": "",
+        "target_uid": "",
+        "user_uid": user_uid,
+        "original_filename": original_filename,
+        "stored_name": stored_name,
+        "directory": directory,
+        "file_size": len(file_bytes),
+        "mime_type": mime,
+        "image_width": image_width,
+        "image_height": image_height,
+        "has_thumbnail": 1 if thumbnail else 0,
+        "thumbnail_name": thumbnail,
+        "created_at": datetime.now(timezone.utc).isoformat(),
     })
-    return {"uid": uid, "original_filename": original_filename, "file_size": len(file_bytes), "mime_type": mime, "url": f"/static/uploads/attachments/{directory}/{stored_name}", "thumbnail_url": f"/static/uploads/attachments/{directory}/{thumb}" if thumb else None, "has_thumbnail": thumb is not None, "is_image": is_img}
+    return {
+        "uid": uid,
+        "original_filename": original_filename,
+        "file_size": len(file_bytes),
+        "mime_type": mime,
+        "url": f"/static/uploads/attachments/{directory}/{stored_name}",
+        "thumbnail_url": f"/static/uploads/attachments/{directory}/{thumbnail}" if thumbnail else None,
+        "has_thumbnail": thumbnail is not None,
+        "is_image": is_image,
+        "is_video": mime.startswith("video/"),
+    }
+
 
 def link_attachments(uids, target_type, target_uid):
-    if not uids: return
-    atts = get_table("attachments")
-    for auid in uids:
-        auid = auid.strip()
-        if not auid: continue
-        e = atts.find_one(uid=auid)
-        if e: atts.update({"id": e["id"], "uid": auid, "target_type": target_type, "target_uid": target_uid}, ["id"])
+    flat = [uid.strip() for raw in uids or [] for uid in str(raw).split(",") if uid.strip()]
+    if not flat:
+        return
+    placeholders = ",".join(f":p{i}" for i in range(len(flat)))
+    params = {f"p{i}": uid for i, uid in enumerate(flat)}
+    db.query(
+        f"UPDATE attachments SET target_type=:tt, target_uid=:tu WHERE uid IN ({placeholders})",
+        tt=target_type, tu=target_uid, **params,
+    )
+
+
+def _unlink_attachment_files(row):
+    stored_name = row.get("stored_name", "")
+    directory = row.get("directory", "")
+    if not (stored_name and directory):
+        return
+    file_path = ATTACHMENTS_DIR / directory / stored_name
+    try:
+        file_path.unlink(missing_ok=True)
+    except Exception as e:
+        logger.warning(f"Failed to delete attachment file {file_path}: {e}")
+    for thumb_path in (ATTACHMENTS_DIR / directory).glob(f"{Path(stored_name).stem}_thumb.*"):
+        try:
+            thumb_path.unlink(missing_ok=True)
+        except Exception as e:
+            logger.warning(f"Failed to delete thumbnail {thumb_path}: {e}")
+
+
+def _delete_attachment_row(row):
+    _unlink_attachment_files(row)
+    get_table("attachments").delete(id=row["id"])
+
 
 def delete_attachment(uid):
-    atts = get_table("attachments")
-    att = atts.find_one(uid=uid)
-    if not att: return
-    sn, d = att.get("stored_name",""), att.get("directory","")
-    if sn and d:
-        fp = ATTACHMENTS_DIR / d / sn
-        try: fp.unlink(missing_ok=True)
-        except: pass
-        for tp in (ATTACHMENTS_DIR / d).glob(f"{Path(sn).stem}_thumb.*"):
-            try: tp.unlink(missing_ok=True)
-            except: pass
-    atts.delete(id=att["id"])
+    row = get_table("attachments").find_one(uid=uid)
+    if row:
+        _delete_attachment_row(row)
 
-def delete_target_attachments(tt, tu):
-    for a in get_table("attachments").find(target_type=tt, target_uid=tu):
-        delete_attachment(a["uid"])
 
-def get_attachments(tt, tu):
-    try: rows = list(get_table("attachments").find(target_type=tt, target_uid=tu, order_by=["created_at"]))
-    except: return []
-    return [_r2a(r) for r in rows]
+def delete_target_attachments(target_type, target_uid):
+    for row in get_table("attachments").find(target_type=target_type, target_uid=target_uid):
+        _delete_attachment_row(row)
 
-def get_attachments_batch(tt, uids):
-    if not uids: return {}
-    from devplacepy.database import db
-    if "attachments" not in db.tables: return {u:[] for u in uids}
-    ph = ",".join(f":p{i}" for i in range(len(uids)))
-    try:
-        rows = db.query(f"SELECT * FROM attachments WHERE target_type=:tt AND target_uid IN ({ph}) ORDER BY created_at", tt=tt, **{f"p{i}":u for i,u in enumerate(uids)})
-    except: return {u:[] for u in uids}
-    r = {u:[] for u in uids}
+
+def delete_attachments_for(target_type, target_uids):
+    uids = [uid for uid in target_uids if uid]
+    if not uids or "attachments" not in db.tables:
+        return
+    placeholders = ",".join(f":p{i}" for i in range(len(uids)))
+    params = {f"p{i}": uid for i, uid in enumerate(uids)}
+    rows = list(db.query(
+        f"SELECT * FROM attachments WHERE target_type=:tt AND target_uid IN ({placeholders})",
+        tt=target_type, **params,
+    ))
+    if not rows:
+        return
     for row in rows:
-        if row["target_uid"] in r: r[row["target_uid"]].append(_r2a(row))
-    return r
+        _unlink_attachment_files(row)
+    ids = ",".join(str(row["id"]) for row in rows)
+    db.query(f"DELETE FROM attachments WHERE id IN ({ids})")
 
-def _r2a(r):
-    sn, d = r.get("stored_name",""), r.get("directory","")
-    ts = f"{Path(sn).stem}_thumb.jpg" if r.get("has_thumbnail") else None
-    return {"uid":r["uid"],"original_filename":r.get("original_filename",""),"file_size":r.get("file_size",0),"mime_type":r.get("mime_type",""),"url":f"/static/uploads/attachments/{d}/{sn}","thumbnail_url":f"/static/uploads/attachments/{d}/{ts}" if ts else None,"has_thumbnail":bool(r.get("has_thumbnail")),"is_image":r.get("mime_type","").startswith("image/")}
 
-def format_file_size(b):
-    if b < 1024: return f"{b} B"
-    if b < 1048576: return f"{b/1024:.1f} KB"
-    return f"{b/1048576:.1f} MB"
+def get_attachments(target_type, target_uid):
+    if "attachments" not in db.tables:
+        return []
+    rows = list(get_table("attachments").find(target_type=target_type, target_uid=target_uid, order_by=["created_at"]))
+    return [_row_to_attachment(r) for r in rows]
 
-def file_icon_emoji(fn):
-    ext = Path(fn).suffix.lower()
-    m = {".pdf":"\U0001F4C4",".zip":"\U0001F4E6",".gz":"\U0001F4E6",".tar":"\U0001F4E6",".rar":"\U0001F4E6",".7z":"\U0001F4E6",".mp4":"\U0001F3AC",".mp3":"\U0001F3B5",".py":"\U0001F4BB",".js":"\U0001F4BB",".ts":"\U0001F4BB",".html":"\U0001F4BB",".css":"\U0001F4BB",".json":"\U0001F4BB",".md":"\U0001F4BB",".csv":"\U0001F4CA",".xls":"\U0001F4CA",".xlsx":"\U0001F4CA",".doc":"\U0001F4DD",".docx":"\U0001F4DD",".txt":"\U0001F4C4",".exe":"\u2699",".bin":"\u2699"}
-    return m.get(ext, "\U0001F4CE")
+
+def get_attachments_batch(target_type, uids):
+    if not uids:
+        return {}
+    if "attachments" not in db.tables:
+        return {uid: [] for uid in uids}
+    placeholders = ",".join(f":p{i}" for i in range(len(uids)))
+    params = {f"p{i}": uid for i, uid in enumerate(uids)}
+    rows = db.query(
+        f"SELECT * FROM attachments WHERE target_type=:tt AND target_uid IN ({placeholders}) ORDER BY created_at",
+        tt=target_type, **params,
+    )
+    result = {uid: [] for uid in uids}
+    for row in rows:
+        if row["target_uid"] in result:
+            result[row["target_uid"]].append(_row_to_attachment(row))
+    return result
+
+
+def _row_to_attachment(row):
+    stored_name = row.get("stored_name", "")
+    directory = row.get("directory", "")
+    thumb_name = None
+    if row.get("has_thumbnail"):
+        thumb_name = row.get("thumbnail_name")
+        if not thumb_name:
+            stem = Path(stored_name).stem
+            png = f"{stem}_thumb.png"
+            thumb_name = png if (ATTACHMENTS_DIR / directory / png).exists() else f"{stem}_thumb.jpg"
+    return {
+        "uid": row["uid"],
+        "original_filename": row.get("original_filename", ""),
+        "file_size": row.get("file_size", 0),
+        "mime_type": row.get("mime_type", ""),
+        "url": f"/static/uploads/attachments/{directory}/{stored_name}",
+        "thumbnail_url": f"/static/uploads/attachments/{directory}/{thumb_name}" if thumb_name else None,
+        "has_thumbnail": bool(row.get("has_thumbnail")),
+        "is_image": row.get("mime_type", "").startswith("image/"),
+        "is_video": row.get("mime_type", "").startswith("video/"),
+    }
+
+
+def format_file_size(size):
+    if size < 1024:
+        return f"{size} B"
+    if size < 1048576:
+        return f"{size / 1024:.1f} KB"
+    return f"{size / 1048576:.1f} MB"
+
+
+def file_icon_emoji(filename):
+    ext = Path(filename).suffix.lower()
+    return FILE_ICONS.get(ext, DEFAULT_FILE_ICON)

devplacepy/avatar.py

@@ -9,11 +9,12 @@ def avatar_url(style: str, seed: str, size: int = 128) -> str:
 
 def generate_avatar_svg(seed: str) -> str:
     try:
-        from multiavatar import multiavatar
-        svg = multiavatar(seed)
+        from multiavatar.multiavatar import multiavatar
+        svg = multiavatar(seed, None, None)
         if svg and svg.strip().startswith("<svg"):
             return svg
     except Exception as e:
+        logger.exception(e)
         logger.warning(f"Avatar generation failed for {seed}: {e}")
     initial = seed[:1].upper() if seed else "?"
     return (

devplacepy/cache.py

@@ -0,0 +1,36 @@
+import time
+from collections import OrderedDict
+
+
+class TTLCache:
+    def __init__(self, ttl: int, max_size: int = 0):
+        self.ttl = ttl
+        self.max_size = max_size
+        self._store = OrderedDict()
+
+    def get(self, key):
+        entry = self._store.get(key)
+        if entry is None:
+            return None
+        value, expiry = entry
+        if time.time() >= expiry:
+            self._store.pop(key, None)
+            return None
+        self._store.move_to_end(key)
+        return value
+
+    def set(self, key, value):
+        self._store[key] = (value, time.time() + self.ttl)
+        self._store.move_to_end(key)
+        if self.max_size and len(self._store) > self.max_size:
+            self._store.popitem(last=False)
+
+    def pop(self, key):
+        self._store.pop(key, None)
+
+    def clear(self):
+        self._store.clear()
+
+    def items(self):
+        now = time.time()
+        return [(key, value) for key, (value, expiry) in self._store.items() if now < expiry]

devplacepy/cli.py

@@ -1,6 +1,7 @@
 import argparse
 import sys
 from devplacepy.database import get_table
+from devplacepy.utils import strip_html
 
 
 def cmd_role_get(args):
@@ -28,11 +29,68 @@ def cmd_role_set(args):
     print(f"User '{args.username}' role set to '{role}'")
 
 
+def cmd_apikey_get(args):
+    users = get_table("users")
+    user = users.find_one(username=args.username)
+    if not user:
+        print(f"User '{args.username}' not found")
+        sys.exit(1)
+    print(user.get("api_key", "") or "")
+
+
+def cmd_apikey_reset(args):
+    from devplacepy.utils import generate_uid, clear_user_cache
+    users = get_table("users")
+    user = users.find_one(username=args.username)
+    if not user:
+        print(f"User '{args.username}' not found")
+        sys.exit(1)
+    new_key = generate_uid()
+    users.update({"uid": user["uid"], "api_key": new_key}, ["uid"])
+    clear_user_cache(user["uid"])
+    print(new_key)
+
+
+def cmd_apikey_backfill(args):
+    from devplacepy.database import backfill_api_keys
+    updated = backfill_api_keys()
+    print(f"Assigned API keys to {updated} user(s) without one")
+
+
+def cmd_devii_reset_quota(args):
+    from devplacepy.database import db
+    table_name = "devii_usage_ledger"
+    if table_name not in db.tables:
+        print(f"Table '{table_name}' does not exist, nothing to reset")
+        return
+    table = db[table_name]
+    if args.all:
+        count = table.count()
+        table.delete()
+        print(f"Reset all AI quotas ({count} ledger rows deleted)")
+        return
+    if args.guests:
+        count = table.count(owner_kind="guest")
+        table.delete(owner_kind="guest")
+        print(f"Reset all guest AI quotas ({count} ledger rows deleted)")
+        return
+    if not args.username:
+        print("Provide a username, or --guests, or --all")
+        sys.exit(1)
+    user = get_table("users").find_one(username=args.username)
+    if not user:
+        print(f"User '{args.username}' not found")
+        sys.exit(1)
+    count = table.count(owner_kind="user", owner_id=user["uid"])
+    table.delete(owner_kind="user", owner_id=user["uid"])
+    print(f"Reset AI quota for '{args.username}' ({count} ledger rows deleted)")
+
+
 def cmd_news_clear(args):
     from devplacepy.database import db
     for table in ("news", "news_images", "news_sync"):
         if table in db.tables:
-            count = len(list(db[table].all()))
+            count = db[table].count()
             db[table].delete()
             print(f"Deleted {count} rows from '{table}'")
         else:
@@ -40,45 +98,175 @@ def cmd_news_clear(args):
     print("News data cleared")
 
 
-def cmd_attachments_prune(args):
+def cmd_news_sanitize(args):
     from devplacepy.database import db
-    from devplacepy.config import STATIC_DIR
-    import os
-    deleted_records = 0
-    deleted_files = 0
-    freed_bytes = 0
+    if "news" not in db.tables:
+        print("News table does not exist")
+        return
+    news_table = db["news"]
+    updated = 0
+    for row in news_table.all():
+        desc = (strip_html(row.get("description", "") or ""))[:5000]
+        content = (strip_html(row.get("content", "") or ""))[:10000]
+        if desc != row.get("description", "") or content != row.get("content", ""):
+            news_table.update({"id": row["id"], "description": desc, "content": content}, ["id"])
+            updated += 1
+    print(f"Sanitized {updated} news article(s)")
 
-    if "attachments" in db.tables:
-        orphans = list(db["attachments"].find(resource_uid=""))
-        orphans += list(db["attachments"].find(resource_type=""))
-        seen = set()
-        unique_orphans = []
-        for o in orphans:
-            if o["uid"] not in seen:
-                seen.add(o["uid"])
-                unique_orphans.append(o)
 
-        for att in unique_orphans:
-            sp = att.get("storage_path", "")
-            if sp:
-                fp = STATIC_DIR / "uploads" / sp
-                try:
-                    if fp.exists():
-                        freed_bytes += fp.stat().st_size
-                        fp.unlink()
-                        deleted_files += 1
-                    parent = fp.parent
-                    if parent.exists() and not any(parent.iterdir()):
-                        parent.rmdir()
-                    grandparent = parent.parent
-                    if grandparent.exists() and not any(grandparent.iterdir()):
-                        grandparent.rmdir()
-                except Exception as e:
-                    print(f"  Error deleting {sp}: {e}")
-            db["attachments"].delete(id=att["id"])
-            deleted_records += 1
+def cmd_attachments_prune(args):
+    from datetime import datetime, timezone, timedelta
+    from devplacepy.database import db
+    from devplacepy.attachments import delete_attachment
 
-    print(f"Pruned {deleted_records} orphan records, {deleted_files} files, {freed_bytes / 1024:.1f} KB freed")
+    if "attachments" not in db.tables:
+        print("Attachments table does not exist")
+        return
+
+    cutoff = (datetime.now(timezone.utc) - timedelta(hours=args.hours)).isoformat()
+    orphans = [
+        att for att in db["attachments"].find(target_type="", target_uid="")
+        if att.get("created_at", "") < cutoff
+    ]
+    for att in orphans:
+        delete_attachment(att["uid"])
+    print(f"Pruned {len(orphans)} orphan attachment(s) older than {args.hours}h")
+
+
+def _remove_zip_artifacts(job):
+    import shutil
+    from pathlib import Path
+    from devplacepy.services.jobs.zip_service import STAGING_DIR
+    local_path = (job.get("result") or {}).get("local_path")
+    if local_path:
+        Path(local_path).unlink(missing_ok=True)
+    shutil.rmtree(STAGING_DIR / job["uid"], ignore_errors=True)
+
+
+def cmd_zips_prune(args):
+    from datetime import datetime, timezone
+    from devplacepy.services.jobs import queue
+    now = datetime.now(timezone.utc)
+    removed = 0
+    for job in queue.list_jobs(kind="zip", status=queue.DONE):
+        expires_at = job.get("expires_at")
+        if not expires_at:
+            continue
+        try:
+            expiry = datetime.fromisoformat(expires_at)
+        except (ValueError, TypeError):
+            continue
+        if expiry < now:
+            _remove_zip_artifacts(job)
+            get_table("jobs").delete(uid=job["uid"])
+            removed += 1
+    print(f"Pruned {removed} expired zip job(s)")
+
+
+def cmd_zips_clear(args):
+    from devplacepy.services.jobs import queue
+    jobs = queue.list_jobs(kind="zip")
+    for job in jobs:
+        _remove_zip_artifacts(job)
+        get_table("jobs").delete(uid=job["uid"])
+    print(f"Cleared {len(jobs)} zip job(s) and their archives")
+
+
+def cmd_forks_prune(args):
+    from datetime import datetime, timezone
+    from devplacepy.services.jobs import queue
+    now = datetime.now(timezone.utc)
+    removed = 0
+    for job in queue.list_jobs(kind="fork", status=queue.DONE):
+        expires_at = job.get("expires_at")
+        if not expires_at:
+            continue
+        try:
+            expiry = datetime.fromisoformat(expires_at)
+        except (ValueError, TypeError):
+            continue
+        if expiry < now:
+            get_table("jobs").delete(uid=job["uid"])
+            removed += 1
+    print(f"Pruned {removed} expired fork job(s)")
+
+
+def cmd_forks_clear(args):
+    from devplacepy.services.jobs import queue
+    jobs = queue.list_jobs(kind="fork")
+    for job in jobs:
+        get_table("jobs").delete(uid=job["uid"])
+    print(f"Cleared {len(jobs)} fork job(s)")
+
+
+def cmd_containers_list(args):
+    from devplacepy.services.containers import store
+    instances = store.all_instances()
+    if not instances:
+        print("No container instances")
+        return
+    for inst in instances:
+        print(f"{inst['uid'][:8]} {inst.get('name', ''):24.24} {inst.get('status', ''):10} "
+              f"desired={inst.get('desired_state', '')} policy={inst.get('restart_policy', '')}")
+
+
+def cmd_containers_reconcile(args):
+    import asyncio
+    from devplacepy.services.containers.service import ContainerService
+    asyncio.run(ContainerService().run_once())
+    print("Reconcile pass complete")
+
+
+def cmd_containers_prune(args):
+    import asyncio
+    from devplacepy.services.containers.runtime import get_backend
+    from devplacepy.services.containers.service import ContainerService
+
+    async def run():
+        await ContainerService().run_once()
+        await get_backend().image_prune()
+
+    asyncio.run(run())
+    print("Reaped orphans and pruned dangling images")
+
+
+def cmd_containers_prune_builds(args):
+    import asyncio
+    from devplacepy.database import db, get_table
+    from devplacepy.services.containers.runtime import get_backend
+
+    async def run():
+        backend = get_backend()
+        removed = 0
+        if "builds" in db.tables:
+            for build in list(get_table("builds").find()):
+                tag = build.get("image_tag")
+                if tag:
+                    await backend.remove_image(tag)
+                    removed += 1
+        for table in ("builds", "dockerfile_versions", "dockerfiles"):
+            if table in db.tables:
+                get_table(table).delete()
+        return removed
+
+    removed = asyncio.run(run())
+    print(f"Removed {removed} legacy per-project image(s) and cleared the dockerfiles/builds tables")
+
+
+def cmd_containers_gc_workspaces(args):
+    import shutil
+    from pathlib import Path
+    from devplacepy import config
+    from devplacepy.services.containers import store
+    active = {inst["project_uid"] for inst in store.all_instances()}
+    base = Path(config.CONTAINER_WORKSPACES_DIR)
+    removed = 0
+    if base.is_dir():
+        for child in base.iterdir():
+            if child.is_dir() and child.name not in active:
+                shutil.rmtree(child, ignore_errors=True)
+                removed += 1
+    print(f"Removed {removed} unused workspace director{'y' if removed == 1 else 'ies'}")
 
 
 def main():
@@ -97,16 +285,60 @@ def main():
     role_set.add_argument("role", choices=["member", "admin"])
     role_set.set_defaults(func=cmd_role_set)
 
+    apikey = sub.add_parser("apikey", help="Manage user API keys")
+    apikey_sub = apikey.add_subparsers(title="action", dest="action")
+    apikey_get = apikey_sub.add_parser("get", help="Print a user's API key")
+    apikey_get.add_argument("username")
+    apikey_get.set_defaults(func=cmd_apikey_get)
+    apikey_reset = apikey_sub.add_parser("reset", help="Regenerate a user's API key")
+    apikey_reset.add_argument("username")
+    apikey_reset.set_defaults(func=cmd_apikey_reset)
+    apikey_backfill = apikey_sub.add_parser("backfill", help="Assign API keys to users that lack one")
+    apikey_backfill.set_defaults(func=cmd_apikey_backfill)
+
     news = sub.add_parser("news", help="News management")
     news_sub = news.add_subparsers(title="action", dest="action")
     news_clear = news_sub.add_parser("clear", help="Delete all news from local database")
     news_clear.set_defaults(func=cmd_news_clear)
+    news_sanitize = news_sub.add_parser("sanitize", help="Strip HTML from all existing news descriptions and content")
+    news_sanitize.set_defaults(func=cmd_news_sanitize)
 
     attachments = sub.add_parser("attachments", help="Attachment management")
     att_sub = attachments.add_subparsers(title="action", dest="action")
     att_prune = att_sub.add_parser("prune", help="Remove orphaned attachment records and files")
+    att_prune.add_argument("--hours", type=int, default=24, help="Only prune orphans older than this many hours")
     att_prune.set_defaults(func=cmd_attachments_prune)
 
+    devii = sub.add_parser("devii", help="Devii assistant management")
+    devii_sub = devii.add_subparsers(title="action", dest="action")
+    devii_reset = devii_sub.add_parser("reset-quota", help="Reset the rolling 24h AI spend quota")
+    devii_reset.add_argument("username", nargs="?", help="Reset the quota for a single user")
+    devii_reset.add_argument("--guests", action="store_true", help="Reset every guest quota")
+    devii_reset.add_argument("--all", action="store_true", help="Reset every quota (users and guests)")
+    devii_reset.set_defaults(func=cmd_devii_reset_quota)
+
+    zips = sub.add_parser("zips", help="Zip archive job management")
+    zips_sub = zips.add_subparsers(title="action", dest="action")
+    zips_prune = zips_sub.add_parser("prune", help="Delete expired zip archives and their job rows")
+    zips_prune.set_defaults(func=cmd_zips_prune)
+    zips_clear = zips_sub.add_parser("clear", help="Delete every zip archive and job row")
+    zips_clear.set_defaults(func=cmd_zips_clear)
+
+    forks = sub.add_parser("forks", help="Fork job management")
+    forks_sub = forks.add_subparsers(title="action", dest="action")
+    forks_prune = forks_sub.add_parser("prune", help="Delete expired completed fork job rows (forked projects persist)")
+    forks_prune.set_defaults(func=cmd_forks_prune)
+    forks_clear = forks_sub.add_parser("clear", help="Delete every fork job row (forked projects persist)")
+    forks_clear.set_defaults(func=cmd_forks_clear)
+
+    containers = sub.add_parser("containers", help="Container manager")
+    containers_sub = containers.add_subparsers(title="action", dest="action")
+    containers_sub.add_parser("list", help="List container instances").set_defaults(func=cmd_containers_list)
+    containers_sub.add_parser("reconcile", help="Run one reconcile pass").set_defaults(func=cmd_containers_reconcile)
+    containers_sub.add_parser("prune", help="Reap orphan containers and dangling images").set_defaults(func=cmd_containers_prune)
+    containers_sub.add_parser("prune-builds", help="Remove legacy per-project images and clear the dockerfiles/builds tables").set_defaults(func=cmd_containers_prune_builds)
+    containers_sub.add_parser("gc-workspaces", help="Remove workspace dirs with no instances").set_defaults(func=cmd_containers_gc_workspaces)
+
     args = parser.parse_args()
     if hasattr(args, "func"):
         args.func(args)

devplacepy/config.py

@@ -9,5 +9,28 @@ STATIC_DIR = BASE_DIR / "devplacepy" / "static"
 TEMPLATES_DIR = BASE_DIR / "devplacepy" / "templates"
 DATABASE_URL = environ.get("DEVPLACE_DATABASE_URL", f"sqlite:///{BASE_DIR / 'devplace.db'}")
 SECRET_KEY = environ.get("SECRET_KEY", "devplace-secret-key-change-in-production")
-SESSION_MAX_AGE = 86400 * 7
+SECONDS_PER_DAY = 86400
+SESSION_MAX_AGE = SECONDS_PER_DAY * 7
+SESSION_MAX_AGE_REMEMBER = SECONDS_PER_DAY * 30
 PORT = 10500
+SITE_URL = environ.get("DEVPLACE_SITE_URL", "").rstrip("/")
+
+INTERNAL_BASE_URL = environ.get("DEVPLACE_INTERNAL_BASE_URL", f"http://localhost:{PORT}").rstrip("/")
+INTERNAL_GATEWAY_URL = f"{INTERNAL_BASE_URL}/openai/v1/chat/completions"
+INTERNAL_MODEL = "molodetz"
+
+SERVICE_LOCK_FILE = BASE_DIR / "devplace-services.lock"
+INIT_LOCK_FILE = BASE_DIR / "devplace-init.lock"
+
+# Container data lives OUTSIDE the package (never served via /static, never watched by --reload).
+DATA_DIR = Path(environ.get("DEVPLACE_DATA_DIR", str(BASE_DIR / "var")))
+CONTAINER_WORKSPACES_DIR = DATA_DIR / "container_workspaces"
+# Every container instance runs this one prebuilt image (built once via `make ppy`).
+CONTAINER_IMAGE = environ.get("DEVPLACE_CONTAINER_IMAGE", "ppy:latest")
+# Host the /p/<slug> ingress proxy dials to reach a published container port.
+CONTAINER_PROXY_HOST = environ.get("DEVPLACE_CONTAINER_PROXY_HOST", "127.0.0.1")
+
+VAPID_PRIVATE_KEY_FILE = BASE_DIR / "notification-private.pem"
+VAPID_PRIVATE_KEY_PKCS8_FILE = BASE_DIR / "notification-private.pkcs8.pem"
+VAPID_PUBLIC_KEY_FILE = BASE_DIR / "notification-public.pem"
+VAPID_SUB = environ.get("DEVPLACE_VAPID_SUB", "mailto:retoor@molodetz.nl")

devplacepy/constants.py

@@ -1 +1,5 @@
 TOPICS = ["devlog", "showcase", "question", "rant", "fun", "random", "signals"]
+
+REACTION_EMOJI = ["\U0001F44D", "❤️", "\U0001F680", "\U0001F389", "\U0001F602", "\U0001F440", "\U0001F525", "\U0001F92F"]
+
+DEVII_GUEST_COOKIE = "devii_guest"

devplacepy/content.py

@@ -0,0 +1,187 @@
+# retoor <retoor@molodetz.nl>
+import logging
+from typing import Any
+from datetime import datetime, timezone
+from fastapi.responses import RedirectResponse
+from devplacepy.database import (
+    get_table,
+    resolve_by_slug,
+    get_users_by_uids,
+    get_vote_counts,
+    get_user_votes,
+    get_reactions_by_targets,
+    get_user_bookmarks,
+    get_poll_for_post,
+    delete_engagement,
+    load_comments,
+    db,
+)
+
+BOOKMARKABLE_TYPES = {"post", "gist", "project", "news"}
+REACTABLE_TYPES = {"post", "comment", "gist", "project"}
+from devplacepy.attachments import delete_attachments_for, delete_inline_image, get_attachments, link_attachments
+from devplacepy.utils import time_ago, generate_uid, make_combined_slug, award_rewards, create_mention_notifications
+
+logger = logging.getLogger(__name__)
+
+
+def is_owner(item: dict | None, user: dict | None) -> bool:
+    return bool(item and user and item["user_uid"] == user["uid"])
+
+
+def can_view_project(project: dict | None, user: dict | None) -> bool:
+    from devplacepy.utils import is_admin
+    if not project:
+        return False
+    if not project.get("is_private"):
+        return True
+    return is_owner(project, user) or is_admin(user)
+
+
+def canonical_redirect(area: str, item: dict, requested: str) -> RedirectResponse | None:
+    canonical = item.get("slug") or item["uid"]
+    if requested == canonical:
+        return None
+    return RedirectResponse(url=f"/{area}/{canonical}", status_code=301)
+
+
+def first_image_url(item: dict, attachments: list | None) -> str | None:
+    inline = item.get("image")
+    if inline:
+        return f"/static/uploads/{inline}"
+    for attachment in attachments or []:
+        if attachment.get("is_image"):
+            return attachment["url"]
+    return None
+
+
+def create_content_item(table_name: str, target_type: str, user: dict, fields: dict, slug_source: str, xp: int, badge: str, mention_text: str, attachment_uids: list | None) -> tuple[str, str]:
+    uid = generate_uid()
+    slug = make_combined_slug(slug_source, uid)
+    get_table(table_name).insert({
+        "uid": uid,
+        "user_uid": user["uid"],
+        "slug": slug,
+        "stars": 0,
+        "created_at": datetime.now(timezone.utc).isoformat(),
+        **fields,
+    })
+    award_rewards(user["uid"], xp, badge)
+    if attachment_uids:
+        link_attachments(attachment_uids, target_type, uid)
+    create_mention_notifications(mention_text, user["uid"], f"/{table_name}/{slug}")
+    logger.info(f"{target_type} {uid} created by {user['username']}")
+    return uid, slug
+
+
+def detail_context(request, user: dict | None, detail: dict, key: str, seo_ctx: dict, extra: dict | None = None) -> dict:
+    context = {
+        **seo_ctx,
+        "request": request,
+        "user": user,
+        key: detail["item"],
+        "author": detail["author"],
+        "is_owner": detail["is_owner"],
+        "star_count": detail["star_count"],
+        "my_vote": detail["my_vote"],
+        "time_ago": detail["time_ago"],
+        "comments": detail["comments"],
+        "attachments": detail["attachments"],
+        "reactions": detail.get("reactions", {"counts": {}, "mine": []}),
+        "bookmarked": detail.get("bookmarked", False),
+        "poll": detail.get("poll"),
+    }
+    if extra:
+        context.update(extra)
+    return context
+
+
+def edit_content_item(request, table_name: str, user: dict, slug: str, update_fields: dict, redirect_fail: str):
+    from devplacepy.responses import action_result, wants_json, json_error
+    table = get_table(table_name)
+    item = resolve_by_slug(table, slug)
+    if not is_owner(item, user):
+        if wants_json(request):
+            return json_error(403, "Not allowed")
+        return RedirectResponse(url=redirect_fail, status_code=302)
+    update_fields = {**update_fields, "updated_at": datetime.now(timezone.utc).isoformat()}
+    table.update({"uid": item["uid"], **update_fields}, ["uid"])
+    logger.info(f"{table_name} {item['uid']} edited by {user['username']}")
+    url = f"/{table_name}/{item['slug'] or item['uid']}"
+    return action_result(request, url, data={"uid": item["uid"], "slug": item.get("slug"), "url": url})
+
+
+def delete_content_item(request, table_name: str, target_type: str, user: dict, slug: str, redirect_url: str, inline_image_field: str | None = None):
+    from devplacepy.responses import action_result, wants_json, json_error
+    table = get_table(table_name)
+    item = resolve_by_slug(table, slug)
+    if not is_owner(item, user):
+        if wants_json(request):
+            return json_error(403, "Not allowed")
+        return RedirectResponse(url=redirect_url, status_code=302)
+    if is_owner(item, user):
+        delete_attachments_for(target_type, [item["uid"]])
+        comment_uids = []
+        if "comments" in db.tables:
+            comments = get_table("comments")
+            comment_uids = [comment["uid"] for comment in comments.find(target_uid=item["uid"])]
+            delete_attachments_for("comment", comment_uids)
+            comments.delete(target_uid=item["uid"])
+        if "votes" in db.tables:
+            votes = get_table("votes")
+            votes.delete(target_uid=item["uid"])
+            if comment_uids:
+                votes.delete(votes.table.columns.target_uid.in_(comment_uids), target_type="comment")
+        delete_engagement(target_type, [item["uid"]])
+        if comment_uids:
+            delete_engagement("comment", comment_uids)
+        if target_type == "project":
+            from devplacepy.project_files import delete_all_project_files
+            from devplacepy.database import delete_fork_relations
+            delete_all_project_files(item["uid"])
+            delete_fork_relations(item["uid"])
+        if inline_image_field:
+            delete_inline_image(item.get(inline_image_field))
+        table.delete(id=item["id"])
+        logger.info(f"{table_name} {item['uid']} deleted by {user['username']}")
+    return action_result(request, redirect_url)
+
+
+def load_detail(table_name: str, target_type: str, slug: str, user: dict | None) -> dict | None:
+    item = resolve_by_slug(get_table(table_name), slug)
+    if not item:
+        return None
+    author = get_users_by_uids([item["user_uid"]]).get(item["user_uid"])
+    ups, downs = get_vote_counts([item["uid"]])
+    reactions = get_reactions_by_targets(target_type, [item["uid"]], user).get(item["uid"], {"counts": {}, "mine": []}) if target_type in REACTABLE_TYPES else {"counts": {}, "mine": []}
+    bookmarked = bool(user) and target_type in BOOKMARKABLE_TYPES and item["uid"] in get_user_bookmarks(user["uid"], target_type, [item["uid"]])
+    return {
+        "item": item,
+        "author": author,
+        "is_owner": bool(user and user["uid"] == item["user_uid"]),
+        "star_count": ups.get(item["uid"], 0) - downs.get(item["uid"], 0),
+        "my_vote": get_user_votes(user["uid"], [item["uid"]]).get(item["uid"], 0) if user else 0,
+        "comments": load_comments(target_type, item["uid"], user),
+        "attachments": get_attachments(target_type, item["uid"]),
+        "time_ago": time_ago(item["created_at"]),
+        "reactions": reactions,
+        "bookmarked": bookmarked,
+        "poll": get_poll_for_post(item["uid"], user) if target_type == "post" else None,
+    }
+
+
+def enrich_items(items: list, key: str, authors: dict, extra_maps: dict[str, Any] | None = None, ts_field: str = "created_at", user: dict | None = None) -> list:
+    extra_maps = extra_maps or {}
+    my_votes = get_user_votes(user["uid"], [item["uid"] for item in items]) if user else {}
+    enriched = []
+    for item in items:
+        entry = {
+            key: item,
+            "author": authors.get(item["user_uid"]),
+            "time_ago": time_ago(item[ts_field]),
+            "my_vote": my_votes.get(item["uid"], 0),
+        }
+        for name, source in extra_maps.items():
+            entry[name] = source(item) if callable(source) else source.get(item["uid"], 0)
+        enriched.append(entry)
+    return enriched

devplacepy/customization.py

@@ -0,0 +1,77 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import json
+import logging
+
+from markupsafe import Markup
+from starlette.requests import Request
+
+from devplacepy.constants import DEVII_GUEST_COOKIE
+from devplacepy.database import get_custom_overrides, get_setting
+from devplacepy.utils import get_current_user
+
+logger = logging.getLogger("customization")
+
+EMPTY = Markup("")
+
+
+def page_type_for(request: Request) -> str:
+    route = request.scope.get("route")
+    path = getattr(route, "path", None)
+    if path:
+        return path
+    return request.url.path
+
+
+def owner_for(request: Request) -> tuple[str, str] | None:
+    user = get_current_user(request)
+    if user:
+        return "user", user["uid"]
+    guest = request.cookies.get(DEVII_GUEST_COOKIE)
+    if guest:
+        return "guest", guest
+    return None
+
+
+def _overrides_for(request: Request) -> dict:
+    if get_setting("customization_enabled", "1") != "1":
+        return {"css": "", "js": ""}
+    owner = owner_for(request)
+    if owner is None:
+        return {"css": "", "js": ""}
+    return get_custom_overrides(owner[0], owner[1], page_type_for(request))
+
+
+def custom_css_tag(request: Request) -> Markup:
+    try:
+        css = _overrides_for(request).get("css", "")
+        if not css.strip():
+            return EMPTY
+        safe = css.replace("</", "<\\/")
+        return Markup(f'<style id="user-custom-css">\n{safe}\n</style>')
+    except Exception as exc:  # noqa: BLE001 - a customization bug must never break page render
+        logger.warning("custom_css_tag failed: %s", exc)
+        return EMPTY
+
+
+def custom_js_tag(request: Request) -> Markup:
+    try:
+        if get_setting("customization_js_enabled", "1") != "1":
+            return EMPTY
+        code = _overrides_for(request).get("js", "")
+        if not code.strip():
+            return EMPTY
+        payload = json.dumps(code).replace("<", "\\u003c").replace(">", "\\u003e")
+        runner = (
+            f'<script type="application/json" id="user-custom-js-src">{payload}</script>'
+            '<script>(function(){try{'
+            'var src=document.getElementById("user-custom-js-src");'
+            'if(src){new Function(JSON.parse(src.textContent))();}'
+            '}catch(error){console.error("user custom js error",error);}})();</script>'
+        )
+        return Markup(runner)
+    except Exception as exc:  # noqa: BLE001 - a customization bug must never break page render
+        logger.warning("custom_js_tag failed: %s", exc)
+        return EMPTY

devplacepy/docs_examples.py

@@ -0,0 +1,81 @@
+from typing import Any, Union, get_args, get_origin
+
+from pydantic import BaseModel
+
+_PLACEHOLDERS = {
+    "uid": "UID",
+    "username": "username",
+    "slug": "slug",
+    "url": "/path",
+    "email": "user@example.com",
+    "title": "Title",
+    "content": "text",
+    "description": "text",
+    "bio": "text",
+    "language": "python",
+    "topic": "random",
+    "status": "published",
+    "next_cursor": "2026-01-01T00:00:00+00:00",
+    "created_at": "2026-01-01T00:00:00+00:00",
+    "updated_at": "2026-01-01T00:00:00+00:00",
+    "synced_at": "2026-01-01T00:00:00+00:00",
+    "time_ago": "2 hours ago",
+}
+
+
+def _placeholder(name: str) -> str:
+    for token, value in _PLACEHOLDERS.items():
+        if token in name:
+            return value
+    return "string"
+
+
+def _unwrap_optional(annotation):
+    if get_origin(annotation) is Union:
+        args = [arg for arg in get_args(annotation) if arg is not type(None)]
+        return args[0] if args else Any
+    return annotation
+
+
+def _value(name, annotation, stack):
+    annotation = _unwrap_optional(annotation)
+    origin = get_origin(annotation)
+    if origin in (list, set, tuple):
+        args = get_args(annotation)
+        inner = args[0] if args else str
+        if isinstance(inner, type) and issubclass(inner, BaseModel) and inner in stack:
+            return []
+        return [_value(name, inner, stack)]
+    if origin is dict or annotation is dict:
+        return {}
+    if annotation in (list, set, tuple):
+        return []
+    if isinstance(annotation, type) and issubclass(annotation, BaseModel):
+        return _from_model(annotation, stack)
+    if annotation is bool:
+        return False
+    if annotation is int:
+        return 0
+    if annotation is float:
+        return 0.0
+    if annotation is Any:
+        return None
+    return _placeholder(name)
+
+
+def _from_model(model, stack):
+    if model in stack:
+        return {}
+    stack = stack | {model}
+    example = {}
+    for name, info in model.model_fields.items():
+        example[name] = _value(name, info.annotation, stack)
+    return example
+
+
+def schema_example(model):
+    return _from_model(model, frozenset())
+
+
+def action_example(redirect, data=None):
+    return {"ok": True, "redirect": redirect, "data": data}

devplacepy/docs_export.py

@@ -0,0 +1,159 @@
+import base64
+import json
+import re
+from functools import lru_cache
+from pathlib import Path
+
+from devplacepy.config import STATIC_DIR
+from devplacepy import docs_api
+
+
+@lru_cache(maxsize=None)
+def _vendor(name: str) -> str:
+    return (Path(STATIC_DIR) / "vendor" / name).read_text(encoding="utf-8")
+
+
+def _prose_markdown(slug: str, ctx: dict) -> str:
+    from devplacepy.templating import templates
+    try:
+        rendered = templates.env.get_template(f"docs/{slug}.html").render(**ctx)
+    except Exception:
+        return ""
+    rendered = re.sub(r"^\s*<div[^>]*>", "", rendered.strip())
+    rendered = re.sub(r"</div>\s*$", "", rendered)
+    return rendered.strip()
+
+
+def _params_table(params: list) -> str:
+    if not params:
+        return ""
+    rows = ["| Name | In | Type | Required | Description |",
+            "|------|----|------|----------|-------------|"]
+    for p in params:
+        desc = (p.get("description", "") or "").replace("|", "\\|")
+        if p["type"] == "enum" and p.get("options"):
+            allowed = ", ".join(str(option) for option in p["options"]).replace("|", "\\|")
+            desc = f"{desc} Allowed: {allowed}.".strip()
+        rows.append(f"| `{p['name']}` | {p['location']} | {p['type']} | "
+                    f"{'yes' if p['required'] else 'no'} | {desc} |")
+    return "\n".join(rows)
+
+
+def _endpoint_markdown(ep: dict) -> str:
+    lines = [f"### `{ep['method']} {ep['path']}` - {ep['title']}", "", ep["summary"], "",
+             f"*Minimal role:* {ep.get('min_role', ep['auth'])}"]
+    if ep.get("params"):
+        lines += ["", "**Parameters**", "", _params_table(ep["params"])]
+    for note in ep.get("notes", []) or []:
+        lines += ["", f"> {note}"]
+    if ep.get("sample_response") is not None:
+        lines += ["", "**Sample response**", "", "```json",
+                  json.dumps(ep["sample_response"], indent=2), "```"]
+    return "\n".join(lines)
+
+
+def _group_markdown(group: dict) -> str:
+    parts = [(group.get("intro", "") or "").strip(), ""]
+    for ep in group.get("endpoints", []) or []:
+        parts.append(_endpoint_markdown(ep))
+        parts.append("")
+    return "\n".join(parts).strip()
+
+
+def build_markdown(is_admin: bool, base: str, username: str = "", api_key: str = "") -> str:
+    from devplacepy.routers.docs import DOCS_PAGES
+    from devplacepy.services.manager import service_manager
+
+    user_ctx = {"role": "Admin"} if is_admin else None
+    replacements = {
+        "{{ base }}": base,
+        "{{ username }}": username or "YOUR_USERNAME",
+        "{{ api_key }}": api_key or "YOUR_API_KEY",
+    }
+    pages = [p for p in DOCS_PAGES if (not p.get("admin") or is_admin) and p["kind"] != "live"]
+    slugs = {p["slug"] for p in pages}
+
+    out = [f"# DevPlace Documentation", "",
+           f"Complete developer documentation for `{base}`.", "", "## Contents", ""]
+    for page in pages:
+        out.append(f"- [{page['title']}](#doc-{page['slug']})")
+    out += ["", "---", ""]
+
+    for page in pages:
+        if page["kind"] == "prose":
+            body = _prose_markdown(page["slug"], {"base": base, "user": user_ctx})
+        elif page.get("dynamic"):
+            group = docs_api.build_services_group(service_manager.describe_all(), base)
+            body = _group_markdown(docs_api._substitute(group, replacements))
+        else:
+            group = docs_api.render_group(page["slug"], base, username, api_key) or {}
+            body = _group_markdown(group)
+        out += [f'<a id="doc-{page["slug"]}"></a>', body, "", "---", ""]
+
+    text = "\n".join(out).strip() + "\n"
+    return re.sub(
+        r"\(/docs/([a-z0-9-]+)\.html(?:#[^)]*)?\)",
+        lambda m: f"(#doc-{m.group(1)})" if m.group(1) in slugs else m.group(0),
+        text,
+    )
+
+
+_LAYOUT_CSS = """
+:root { color-scheme: light dark; }
+* { box-sizing: border-box; }
+body { margin: 0; font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
+  line-height: 1.65; color: #1f2328; background: #fff; }
+.doc-header { position: sticky; top: 0; z-index: 5; background: #0d1117; color: #fff;
+  padding: 0.9rem 1.5rem; font-weight: 700; letter-spacing: 0.02em; box-shadow: 0 1px 0 rgba(0,0,0,0.15); }
+.doc-header span { opacity: 0.6; font-weight: 400; }
+main.doc { max-width: 860px; margin: 0 auto; padding: 2rem 1.5rem 4rem; }
+main.doc h1 { font-size: 1.9rem; margin: 2.4rem 0 1rem; padding-bottom: 0.3rem; border-bottom: 2px solid #e1e4e8; }
+main.doc h1:first-child { margin-top: 0; }
+main.doc h2 { font-size: 1.4rem; margin: 2rem 0 0.8rem; }
+main.doc h3 { font-size: 1.08rem; margin: 1.6rem 0 0.5rem; }
+main.doc h3 code { background: #eef1f4; padding: 0.1em 0.4em; border-radius: 5px; font-size: 0.95em; }
+main.doc a { color: #0969da; text-decoration: none; }
+main.doc a:hover { text-decoration: underline; }
+main.doc code { font-family: "SF Mono", Monaco, Consolas, monospace; font-size: 0.9em; }
+main.doc :not(pre) > code { background: #eef1f4; padding: 0.12em 0.4em; border-radius: 5px; }
+main.doc pre { background: #282c34; border-radius: 8px; padding: 1rem; overflow-x: auto; }
+main.doc pre code { color: #abb2bf; font-size: 0.85rem; line-height: 1.5; }
+main.doc table { border-collapse: collapse; width: 100%; margin: 1rem 0; font-size: 0.92rem; }
+main.doc th, main.doc td { border: 1px solid #d0d7de; padding: 0.45rem 0.7rem; text-align: left; }
+main.doc th { background: #f6f8fa; }
+main.doc blockquote { margin: 0.8rem 0; padding: 0.2rem 1rem; border-left: 4px solid #d0d7de; color: #57606a; }
+main.doc hr { border: none; border-top: 1px solid #e1e4e8; margin: 2.5rem 0; }
+@media (prefers-color-scheme: dark) {
+  body { background: #0d1117; color: #c9d1d9; }
+  main.doc h1, main.doc h2 { border-color: #30363d; }
+  main.doc :not(pre) > code, main.doc h3 code { background: #21262d; }
+  main.doc th { background: #161b22; }
+  main.doc th, main.doc td, main.doc hr, main.doc blockquote { border-color: #30363d; }
+  main.doc a { color: #58a6ff; }
+}
+"""
+
+
+def build_html(markdown: str) -> str:
+    marked = _vendor("marked.umd.js")
+    hljs = _vendor("highlight.min.js")
+    hl_css = _vendor("atom-one-dark.min.css")
+    payload = base64.b64encode(markdown.encode("utf-8")).decode("ascii")
+    return (
+        "<!doctype html>\n<html lang=\"en\">\n<head>\n<meta charset=\"utf-8\">\n"
+        "<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n"
+        "<title>DevPlace Documentation</title>\n"
+        f"<style>{hl_css}</style>\n<style>{_LAYOUT_CSS}</style>\n"
+        f"<script>{marked}</script>\n<script>{hljs}</script>\n"
+        "</head>\n<body>\n"
+        "<header class=\"doc-header\">DevPlace <span>Documentation</span></header>\n"
+        "<main id=\"doc\" class=\"doc\"></main>\n"
+        f"<script id=\"docmd\" type=\"application/x-markdown-base64\">{payload}</script>\n"
+        "<script>\n(function(){\n"
+        "  var raw = atob(document.getElementById('docmd').textContent);\n"
+        "  var md = decodeURIComponent(escape(raw));\n"
+        "  marked.setOptions({ gfm: true, breaks: false });\n"
+        "  document.getElementById('doc').innerHTML = marked.parse(md);\n"
+        "  document.querySelectorAll('#doc pre code').forEach(function(el){ try { hljs.highlightElement(el); } catch (e) {} });\n"
+        "})();\n</script>\n</body>\n</html>\n"
+    )

devplacepy/docs_live.py

@@ -0,0 +1,147 @@
+from devplacepy.database import (
+    db,
+    get_table,
+    get_setting,
+    get_site_stats,
+    get_top_authors,
+    get_featured_news,
+    get_gist_languages,
+    get_daily_topic,
+    get_streaks,
+    get_user_stars,
+    get_user_rank,
+)
+from devplacepy.utils import format_date
+
+
+def _count(table: str, **criteria) -> int:
+    if table not in db.tables:
+        return 0
+    return get_table(table).count(**criteria)
+
+
+def _recent(table: str, user_uid: str, limit: int = 5) -> list:
+    if table not in db.tables:
+        return []
+    return list(get_table(table).find(user_uid=user_uid, order_by=["-created_at"], _limit=limit))
+
+
+def _user_facts(user: dict) -> dict:
+    uid = user["uid"]
+
+    posts = _recent("posts", uid)
+    gists = _recent("gists", uid)
+    projects = _recent("projects", uid)
+    badges = [b.get("badge_name", "") for b in get_table("badges").find(user_uid=uid)] if "badges" in db.tables else []
+    languages = sorted({
+        g.get("language") for g in (get_table("gists").find(user_uid=uid) if "gists" in db.tables else [])
+        if g.get("language")
+    })
+
+    return {
+        "username": user.get("username", ""),
+        "role": user.get("role", "Member"),
+        "member_since": format_date(user.get("created_at", "")) if user.get("created_at") else "",
+        "level": user.get("level", 1),
+        "xp": user.get("xp", 0),
+        "stars": get_user_stars(uid),
+        "rank": get_user_rank(uid),
+        "streak": get_streaks(uid),
+        "badges": badges,
+        "languages": languages,
+        "counts": {
+            "posts": _count("posts", user_uid=uid),
+            "gists": _count("gists", user_uid=uid),
+            "projects": _count("projects", user_uid=uid),
+            "comments": _count("comments", user_uid=uid),
+            "followers": _count("follows", following_uid=uid),
+            "following": _count("follows", follower_uid=uid),
+            "bookmarks": _count("bookmarks", user_uid=uid),
+            "unread_notifications": _count("notifications", user_uid=uid, read=False),
+            "unread_messages": _count("messages", receiver_uid=uid, read=False),
+        },
+        "recent_posts": [
+            {"title": (p.get("title") or (p.get("content") or "")[:60] or "Untitled"),
+             "url": f"/posts/{p.get('slug') or p['uid']}"}
+            for p in posts
+        ],
+        "recent_gists": [
+            {"title": g.get("title", "Untitled"), "language": g.get("language", ""),
+             "url": f"/gists/{g.get('slug') or g['uid']}"}
+            for g in gists
+        ],
+        "recent_projects": [
+            {"title": p.get("title", "Untitled"), "status": p.get("status", ""),
+             "url": f"/projects/{p.get('slug') or p['uid']}"}
+            for p in projects
+        ],
+    }
+
+
+def build_live_facts(user: dict | None) -> dict:
+    stats = get_site_stats()
+    facts = {
+        "site": {
+            "name": get_setting("site_name", "DevPlace"),
+            "tagline": get_setting("site_tagline", ""),
+            "total_members": stats.get("total_members", 0),
+            "posts_today": stats.get("posts_today", 0),
+            "total_projects": stats.get("total_projects", 0),
+            "total_gists": stats.get("total_gists", 0),
+        },
+        "top_authors": [
+            {"username": a.get("username", ""), "stars": a.get("stars", 0)}
+            for a in get_top_authors(10)
+        ],
+        "languages": sorted(get_gist_languages()),
+        "news": get_featured_news(5),
+        "topic": get_daily_topic(),
+        "user": _user_facts(user) if user else None,
+    }
+    return facts
+
+
+def live_text(facts: dict) -> str:
+    parts: list[str] = []
+    site = facts["site"]
+    parts += [
+        site["name"], site["tagline"],
+        "members", str(site["total_members"]),
+        "posts today", str(site["posts_today"]),
+        "projects", str(site["total_projects"]),
+        "gists", str(site["total_gists"]),
+        "top authors leaderboard",
+    ]
+    parts += [f"{a['username']} {a['stars']} stars" for a in facts["top_authors"]]
+    parts += ["trending languages"] + facts["languages"]
+    parts += ["developer news"] + [n.get("title", "") for n in facts["news"]]
+    parts += [facts["topic"].get("title", "")]
+
+    user = facts.get("user")
+    if user:
+        counts = user["counts"]
+        parts += [
+            "your dashboard account stats",
+            user["username"], user["role"], "member since", user["member_since"],
+            "level", str(user["level"]), "xp", str(user["xp"]),
+            "stars", str(user["stars"]), "rank", str(user["rank"]),
+            "current streak", str(user["streak"].get("current", 0)),
+            "longest streak", str(user["streak"].get("longest", 0)),
+            "posts", str(counts["posts"]),
+            "gists", str(counts["gists"]),
+            "projects", str(counts["projects"]),
+            "comments", str(counts["comments"]),
+            "followers", str(counts["followers"]),
+            "following", str(counts["following"]),
+            "bookmarks saved", str(counts["bookmarks"]),
+            "unread notifications", str(counts["unread_notifications"]),
+            "unread messages", str(counts["unread_messages"]),
+            "badges",
+        ]
+        parts += user["badges"]
+        parts += ["languages you use"] + user["languages"]
+        parts += ["your recent posts"] + [p["title"] for p in user["recent_posts"]]
+        parts += ["your gists"] + [g["title"] for g in user["recent_gists"]]
+        parts += ["your projects"] + [p["title"] for p in user["recent_projects"]]
+
+    return " ".join(str(part) for part in parts if part)

devplacepy/docs_prose.py

@@ -0,0 +1,28 @@
+# retoor <retoor@molodetz.nl>
+
+import html
+import re
+
+import mistune
+
+_markdown = mistune.create_markdown(
+    escape=False,
+    hard_wrap=True,
+    plugins=["table", "strikethrough", "url"],
+)
+
+_RENDER_BLOCK = re.compile(
+    r'<div class="docs-content" data-render>(.*?)</div>', re.DOTALL
+)
+
+
+def _convert(match: re.Match) -> str:
+    source = html.unescape(match.group(1)).strip()
+    return f'<div class="docs-content">{_markdown(source)}</div>'
+
+
+def render_prose(slug: str, context: dict) -> str:
+    from devplacepy.templating import templates
+
+    rendered = templates.env.get_template(f"docs/{slug}.html").render(**context)
+    return _RENDER_BLOCK.sub(_convert, rendered, count=1)

devplacepy/docs_search.py

@@ -0,0 +1,188 @@
+import html
+import math
+import re
+from pathlib import Path
+
+from devplacepy.config import TEMPLATES_DIR
+from devplacepy import docs_api
+
+K1 = 1.5
+B = 0.75
+_TOKEN = re.compile(r"[a-z0-9]+")
+_static_docs = None
+
+
+def _tokenize(text: str) -> list:
+    return [t for t in _TOKEN.findall(text.lower()) if len(t) > 1]
+
+
+def _demarkdown(text: str) -> str:
+    text = re.sub(r"(?m)^[ \t]*```.*$", " ", text)
+    text = re.sub(r"!\[([^\]]*)\]\([^)]*\)", r"\1", text)
+    text = re.sub(r"\[([^\]]+)\]\([^)]*\)", r"\1", text)
+    text = re.sub(r"(?m)^[ \t]{0,3}#{1,6}[ \t]*", "", text)
+    text = re.sub(r"(?m)^[ \t]{0,3}>[ \t]?", "", text)
+    text = re.sub(r"(?m)^[ \t]*(?:[-*+]|\d+\.)[ \t]+", "", text)
+    text = re.sub(r"(?m)^[ \t]*\|?(?:[ \t]*:?-{2,}:?[ \t]*\|)+[ \t]*:?-{0,}:?[ \t]*$", " ", text)
+    text = text.replace("|", " ")
+    text = re.sub(r"[`*~]", "", text)
+    return text
+
+
+def _strip(raw: str) -> str:
+    raw = re.sub(r"(?is)<script\b.*?</script>", " ", raw)
+    raw = re.sub(r"(?is)<style\b.*?</style>", " ", raw)
+    raw = re.sub(r"{#.*?#}", " ", raw, flags=re.S)
+    raw = re.sub(r"{%.*?%}", " ", raw, flags=re.S)
+    raw = re.sub(r"{{.*?}}", " ", raw, flags=re.S)
+    raw = html.unescape(raw)
+    raw = re.sub(r"<[^>]+>", " ", raw)
+    raw = _demarkdown(raw)
+    return re.sub(r"\s+", " ", raw).strip()
+
+
+def _prose_text(slug: str) -> str:
+    path = Path(TEMPLATES_DIR) / "docs" / f"{slug}.html"
+    try:
+        return _strip(path.read_text(encoding="utf-8"))
+    except OSError:
+        return ""
+
+
+def _group_text(group: dict) -> str:
+    parts = [group.get("intro", "")]
+    for ep in group.get("endpoints", []) or []:
+        parts += [ep.get("method", ""), ep.get("path", ""), ep.get("title", ""), ep.get("summary", "")]
+        parts += [str(note) for note in ep.get("notes", []) or []]
+        for param in ep.get("params", []) or []:
+            parts += [param.get("name", ""), param.get("description", "")]
+    return _strip("\n".join(str(p) for p in parts))
+
+
+def _collect_static_docs() -> list:
+    global _static_docs
+    if _static_docs is not None:
+        return _static_docs
+    from devplacepy.routers.docs import DOCS_PAGES
+    from devplacepy.services.manager import service_manager
+
+    docs = []
+    for page in DOCS_PAGES:
+        if page["kind"] == "live":
+            continue
+        slug, title = page["slug"], page["title"]
+        if page["kind"] == "prose":
+            body = _prose_text(slug)
+        elif page.get("dynamic"):
+            body = _group_text(docs_api.build_services_group(service_manager.describe_all(), ""))
+        else:
+            body = _group_text(docs_api.get_group(slug) or {})
+        docs.append({
+            "slug": slug,
+            "title": title,
+            "admin": page.get("admin", False),
+            "text": f"{title}\n{body}",
+        })
+    _static_docs = docs
+    return docs
+
+
+def _live_docs(user) -> list:
+    from devplacepy import docs_live
+
+    facts = docs_live.build_live_facts(user)
+    return [{
+        "slug": "dashboard",
+        "title": "Your dashboard",
+        "admin": False,
+        "text": "Your dashboard live data stats\n" + docs_live.live_text(facts),
+    }]
+
+
+def _build(docs: list) -> dict:
+    postings: dict = {}
+    doc_len: list = []
+    doc_text: list = []
+    df: dict = {}
+    for i, doc in enumerate(docs):
+        tokens = _tokenize(doc["text"])
+        doc_len.append(len(tokens) or 1)
+        doc_text.append(doc["text"])
+        freqs: dict = {}
+        for token in tokens:
+            freqs[token] = freqs.get(token, 0) + 1
+        for token, count in freqs.items():
+            postings.setdefault(token, []).append((i, count))
+            df[token] = df.get(token, 0) + 1
+    n = len(docs) or 1
+    avgdl = (sum(doc_len) / n) if doc_len else 1.0
+    idf = {t: math.log(1 + (n - d + 0.5) / (d + 0.5)) for t, d in df.items()}
+    return {
+        "docs": docs,
+        "postings": postings,
+        "doc_len": doc_len,
+        "doc_text": doc_text,
+        "avgdl": avgdl or 1.0,
+        "idf": idf,
+    }
+
+
+def get_index(user=None) -> dict:
+    return _build(list(_collect_static_docs()) + _live_docs(user))
+
+
+def reset_index() -> None:
+    global _static_docs
+    _static_docs = None
+
+
+def _snippet(text: str, terms: list, width: int = 280) -> str:
+    low = text.lower()
+    pos = -1
+    for term in terms:
+        found = low.find(term)
+        if found != -1 and (pos == -1 or found < pos):
+            pos = found
+    if pos == -1:
+        pos = 0
+    start = max(0, pos - width // 3)
+    end = min(len(text), start + width)
+    fragment = html.escape(text[start:end])
+    for term in sorted(set(terms), key=len, reverse=True):
+        fragment = re.sub(r"(?i)(" + re.escape(html.escape(term)) + r")", r"<mark>\1</mark>", fragment)
+    prefix = "… " if start > 0 else ""
+    suffix = " …" if end < len(text) else ""
+    return prefix + fragment + suffix
+
+
+def search(query: str, user=None, is_admin: bool = False, limit: int = 20) -> list:
+    terms = _tokenize(query or "")
+    if not terms:
+        return []
+    idx = get_index(user)
+    scores: dict = {}
+    for term in set(terms):
+        plist = idx["postings"].get(term)
+        if not plist:
+            continue
+        idf_t = idx["idf"][term]
+        for i, tf in plist:
+            dl = idx["doc_len"][i]
+            denom = tf + K1 * (1 - B + B * dl / idx["avgdl"])
+            scores[i] = scores.get(i, 0.0) + idf_t * (tf * (K1 + 1)) / denom
+    ranked = sorted(scores.items(), key=lambda kv: kv[1], reverse=True)
+    results = []
+    for i, score in ranked:
+        doc = idx["docs"][i]
+        if doc["admin"] and not is_admin:
+            continue
+        results.append({
+            "slug": doc["slug"],
+            "title": doc["title"],
+            "score": round(score, 3),
+            "url": f"/docs/{doc['slug']}.html",
+            "snippet": _snippet(idx["doc_text"][i], terms),
+        })
+        if len(results) >= limit:
+            break
+    return results

devplacepy/main.py

@@ -1,19 +1,31 @@
-import asyncio
+import fcntl
 import logging
 import os
 import time
 from collections import defaultdict
+from contextlib import contextmanager
+from pathlib import Path
 from fastapi import FastAPI, Request
 from fastapi.responses import HTMLResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
-from devplacepy.config import STATIC_DIR, PORT
-from devplacepy.database import init_db, get_table, db, get_users_by_uids, get_comment_counts_by_post_uids, get_vote_counts
+from fastapi.exceptions import RequestValidationError
+from devplacepy.config import STATIC_DIR, PORT, SERVICE_LOCK_FILE, INIT_LOCK_FILE
+from devplacepy.database import init_db, get_table, db, refresh_snapshot, get_users_by_uids, get_comment_counts_by_post_uids, get_vote_counts, get_news_images_by_uids, get_setting, get_int_setting
 from devplacepy.templating import templates
+from devplacepy.responses import wants_json, json_error
+from devplacepy.schemas import ValidationErrorOut
+from fastapi.responses import JSONResponse
 from devplacepy.utils import get_current_user, time_ago
-from devplacepy.seo import base_seo_context, site_url, website_schema, breadcrumb_schema, combine
-from devplacepy.routers import auth, feed, posts, comments, projects, profile, messages, notifications, votes, avatar, follow, admin, seo, bugs, news, gists, services as services_router, uploads
+from devplacepy.seo import base_seo_context, site_url, website_schema
+from devplacepy.routers import auth, feed, posts, comments, projects, project_files, profile, messages, notifications, votes, avatar, follow, admin, seo, bugs, news, gists, services as services_router, uploads, push, leaderboard, reactions, bookmarks, polls, docs, openai_gateway, devii, zips, forks, containers, containers_admin, proxy
 from devplacepy.services.manager import service_manager
 from devplacepy.services.news import NewsService
+from devplacepy.services.bot import BotsService
+from devplacepy.services.openai_gateway import GatewayService
+from devplacepy.services.devii import DeviiService
+from devplacepy.services.jobs.zip_service import ZipService
+from devplacepy.services.jobs.fork_service import ForkService
+from devplacepy.services.containers.service import ContainerService
 
 logging.basicConfig(
     level=logging.INFO,
@@ -22,42 +34,164 @@ logging.basicConfig(
 logger = logging.getLogger(__name__)
 
 _rate_limit_store = defaultdict(list)
-RATE_LIMIT = 60
+RATE_LIMIT = int(os.environ.get("DEVPLACE_RATE_LIMIT", "60"))
 RATE_WINDOW = 60
+WEB_WORKERS = max(1, int(os.environ.get("DEVPLACE_WEB_WORKERS", "1")))
 
-app = FastAPI(title="DevPlace")
+
+def _worker_rate_limit(limit: int) -> int:
+    return max(1, -(-limit // WEB_WORKERS))
+
+
+_service_lock_handle = None
+
+
+@contextmanager
+def init_lock():
+    handle = open(INIT_LOCK_FILE, "w")
+    try:
+        fcntl.flock(handle, fcntl.LOCK_EX)
+        yield
+    finally:
+        fcntl.flock(handle, fcntl.LOCK_UN)
+        handle.close()
+
+
+def acquire_service_lock() -> bool:
+    global _service_lock_handle
+    handle = open(SERVICE_LOCK_FILE, "w")
+    try:
+        fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except OSError:
+        handle.close()
+        return False
+    _service_lock_handle = handle
+    return True
+
+INLINE_MEDIA_EXTENSIONS = {
+    ".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".tiff",
+    ".mp4", ".webm", ".ogv", ".mov", ".m4v", ".mp3",
+}
+
+
+class UploadStaticFiles(StaticFiles):
+    async def get_response(self, path, scope):
+        response = await super().get_response(path, scope)
+        disposition = "inline" if Path(path).suffix.lower() in INLINE_MEDIA_EXTENSIONS else "attachment"
+        response.headers["Content-Disposition"] = disposition
+        return response
+
+
+app = FastAPI(title="DevPlace", docs_url="/swagger", redoc_url="/redoc", openapi_url="/openapi.json")
+app.mount("/static/uploads", UploadStaticFiles(directory=str(STATIC_DIR / "uploads"), check_dir=False), name="uploads")
 app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
 
 
 @app.exception_handler(404)
 async def not_found(request: Request, exc):
-    seo_ctx = base_seo_context(request, title="Not Found — DevPlace", description="The page you requested does not exist.", robots="noindex")
-    return templates.TemplateResponse("error.html", {**seo_ctx, "request": request, "error_code": 404, "error_message": "Page not found"}, status_code=404)
+    if wants_json(request):
+        return json_error(404, "Not found")
+    seo_ctx = base_seo_context(request, title="Not Found - DevPlace", description="The page you requested does not exist.", robots="noindex")
+    return templates.TemplateResponse(request, "error.html", {**seo_ctx, "request": request, "error_code": 404, "error_message": "Page not found"}, status_code=404)
 
 
 @app.exception_handler(500)
 async def server_error(request: Request, exc):
-    seo_ctx = base_seo_context(request, title="Server Error — DevPlace", description="Something went wrong.", robots="noindex")
-    return templates.TemplateResponse("error.html", {**seo_ctx, "request": request, "error_code": 500, "error_message": "Internal server error"}, status_code=500)
+    logger.exception("500 error on %s %s", request.method, request.url.path)
+    if wants_json(request):
+        return json_error(500, "Internal server error")
+    seo_ctx = base_seo_context(request, title="Server Error - DevPlace", description="Something went wrong.", robots="noindex")
+    return templates.TemplateResponse(request, "error.html", {**seo_ctx, "request": request, "error_code": 500, "error_message": "Internal server error"}, status_code=500)
+
+
+_AUTH_FORM_PAGES = {
+    "/auth/signup": ("signup.html", "Join DevPlace"),
+    "/auth/login": ("login.html", "Sign In"),
+    "/auth/forgot-password": ("forgot_password.html", "Reset Password"),
+}
+
+_FRIENDLY_ERRORS = {
+    ("username", "too_short"): "Username must be between 3 and 32 characters",
+    ("username", "too_long"): "Username must be between 3 and 32 characters",
+    ("password", "too_short"): "Password must be at least 6 characters",
+}
+
+
+def _friendly_error(err):
+    field = err["loc"][-1] if err.get("loc") else ""
+    key = (field, err.get("type", "").replace("string_", ""))
+    if key in _FRIENDLY_ERRORS:
+        return _FRIENDLY_ERRORS[key]
+    msg = err.get("msg", "Invalid input")
+    prefix = "Value error, "
+    return msg[len(prefix):] if msg.startswith(prefix) else msg
+
+
+@app.exception_handler(RequestValidationError)
+async def on_validation_error(request: Request, exc: RequestValidationError):
+    errors = [_friendly_error(e) for e in exc.errors()]
+    if wants_json(request):
+        fields: dict = {}
+        for raw, friendly in zip(exc.errors(), errors):
+            name = raw["loc"][-1] if raw.get("loc") else "_"
+            fields.setdefault(str(name), []).append(friendly)
+        return JSONResponse(ValidationErrorOut(fields=fields, messages=errors).model_dump(mode="json"), status_code=422)
+    path = request.url.path
+    page = _AUTH_FORM_PAGES.get(path)
+    if page is None and path.startswith("/auth/reset-password/"):
+        page = ("reset_password.html", "Set New Password")
+    if page:
+        template_name, title = page
+        context = {**base_seo_context(request, title=title, robots="noindex,nofollow"), "request": request, "errors": errors}
+        try:
+            form = await request.form()
+            context.update({k: v for k, v in form.items() if isinstance(v, str)})
+        except Exception:
+            pass
+        if "token" in request.path_params:
+            context["token"] = request.path_params["token"]
+        return templates.TemplateResponse(request, template_name, context, status_code=400)
+    referer = request.headers.get("referer") or "/feed"
+    return RedirectResponse(url=referer, status_code=303)
 
 app.include_router(auth.router, prefix="/auth")
 app.include_router(feed.router, prefix="/feed")
 app.include_router(posts.router, prefix="/posts")
 app.include_router(comments.router, prefix="/comments")
 app.include_router(projects.router, prefix="/projects")
+app.include_router(project_files.router, prefix="/projects")
 app.include_router(profile.router, prefix="/profile")
 app.include_router(messages.router, prefix="/messages")
 app.include_router(notifications.router, prefix="/notifications")
 app.include_router(votes.router, prefix="/votes")
+app.include_router(reactions.router, prefix="/reactions")
+app.include_router(bookmarks.router, prefix="/bookmarks")
+app.include_router(polls.router, prefix="/polls")
 app.include_router(avatar.router, prefix="/avatar")
 app.include_router(follow.router, prefix="/follow")
+app.include_router(leaderboard.router, prefix="/leaderboard")
 app.include_router(admin.router, prefix="/admin")
 app.include_router(seo.router)
+app.include_router(push.router)
+app.include_router(docs.router)
+app.include_router(openai_gateway.router, prefix="/openai")
+app.include_router(devii.router, prefix="/devii")
 app.include_router(bugs.router, prefix="/bugs")
 app.include_router(gists.router, prefix="/gists")
 app.include_router(news.router, prefix="/news")
 app.include_router(services_router.router, prefix="/admin/services")
 app.include_router(uploads.router, prefix="/uploads")
+app.include_router(zips.router, prefix="/zips")
+app.include_router(forks.router, prefix="/forks")
+app.include_router(containers.router, prefix="/projects")
+app.include_router(containers_admin.router, prefix="/admin/containers")
+app.include_router(proxy.router, prefix="/p")
+
+
+@app.middleware("http")
+async def refresh_db_snapshot(request: Request, call_next):
+    refresh_snapshot()
+    return await call_next(request)
 
 
 @app.middleware("http")
@@ -71,31 +205,70 @@ async def add_security_headers(request: Request, call_next):
 
 @app.middleware("http")
 async def rate_limit_middleware(request: Request, call_next):
-    if request.method in ("POST", "PUT", "DELETE", "PATCH"):
-        ip = request.client.host if request.client else "unknown"
+    if request.method in ("POST", "PUT", "DELETE", "PATCH") and not request.url.path.startswith("/openai"):
+        limit = _worker_rate_limit(max(1, get_int_setting("rate_limit_per_minute", RATE_LIMIT)))
+        window = max(1, get_int_setting("rate_limit_window_seconds", RATE_WINDOW))
+        ip = request.headers.get("X-Real-IP") or (request.client.host if request.client else "unknown")
         now = time.time()
-        window_start = now - RATE_WINDOW
+        window_start = now - window
         _rate_limit_store[ip] = [t for t in _rate_limit_store[ip] if t > window_start]
-        if len(_rate_limit_store[ip]) >= RATE_LIMIT:
-            return HTMLResponse("Rate limit exceeded. Try again later.", status_code=429)
+        if len(_rate_limit_store[ip]) >= limit:
+            retry_after = {"Retry-After": str(window)}
+            if wants_json(request):
+                response = json_error(429, "Rate limit exceeded. Try again later.")
+                response.headers["Retry-After"] = str(window)
+                return response
+            return HTMLResponse("Rate limit exceeded. Try again later.", status_code=429, headers=retry_after)
         _rate_limit_store[ip].append(now)
     return await call_next(request)
 
 
+_MAINTENANCE_ALLOWED_PREFIXES = ("/static", "/avatar", "/auth", "/admin", "/openai")
+
+
+@app.middleware("http")
+async def maintenance_middleware(request: Request, call_next):
+    if get_setting("maintenance_mode", "0") != "1":
+        return await call_next(request)
+    if request.url.path.startswith(_MAINTENANCE_ALLOWED_PREFIXES):
+        return await call_next(request)
+    user = get_current_user(request)
+    if user and user.get("role") == "Admin":
+        return await call_next(request)
+    message = get_setting("maintenance_message", "DevPlace is undergoing scheduled maintenance. Please check back shortly.")
+    if wants_json(request):
+        return json_error(503, message)
+    seo_ctx = base_seo_context(request, title="Maintenance - DevPlace", description=message, robots="noindex")
+    return templates.TemplateResponse(request, "error.html", {**seo_ctx, "request": request, "error_code": 503, "error_message": message}, status_code=503)
+
+
 @app.on_event("startup")
 async def startup():
-    init_db()
+    with init_lock():
+        init_db()
+        from devplacepy.push import ensure_certificates
+        ensure_certificates()
+    service_manager.register(NewsService())
+    service_manager.register(BotsService())
+    service_manager.register(GatewayService())
+    service_manager.register(DeviiService())
+    service_manager.register(ZipService())
+    service_manager.register(ForkService())
+    service_manager.register(ContainerService())
     if not os.environ.get("DEVPLACE_DISABLE_SERVICES"):
-        news_service = NewsService()
-        service_manager.register(news_service)
-        asyncio.create_task(service_manager.start_all())
+        if acquire_service_lock():
+            service_manager.set_lock_owner(True)
+            service_manager.supervise()
+            logger.info(f"Background services started in worker pid {os.getpid()}")
+        else:
+            logger.info(f"Worker pid {os.getpid()} declined service lock; another worker owns background services")
     logger.info(f"DevPlace started on port {PORT}")
 
 
 @app.on_event("shutdown")
 async def shutdown():
     logger.info("Shutting down services...")
-    await service_manager.stop_all()
+    await service_manager.shutdown_all()
 
 
 @app.get("/")
@@ -107,14 +280,9 @@ async def landing(request: Request):
     landing_articles = []
     if "news" in db.tables:
         news_table = get_table("news")
-        raw = list(news_table.find(order_by=["-synced_at"], _limit=6))
-        images_table = get_table("news_images") if "news_images" in db.tables else None
+        raw = list(news_table.find(show_on_landing=1, order_by=["-synced_at"], _limit=6))
+        images_by_news = get_news_images_by_uids([a["uid"] for a in raw])
         for a in raw:
-            image_url = ""
-            if images_table:
-                img = images_table.find_one(news_uid=a["uid"])
-                if img:
-                    image_url = img["url"]
             landing_articles.append({
                 "uid": a["uid"],
                 "slug": a.get("slug", ""),
@@ -125,7 +293,7 @@ async def landing(request: Request):
                 "grade": a.get("grade", 0),
                 "synced_at": a.get("synced_at", "") or "",
                 "time_ago": time_ago(a["synced_at"]),
-                "image_url": image_url,
+                "image_url": images_by_news.get(a["uid"], ""),
             })
 
     landing_posts = []
@@ -151,12 +319,12 @@ async def landing(request: Request):
     base = site_url(request)
     seo_ctx = base_seo_context(
         request,
-        title="DevPlace — The Developer Social Network",
+        title="DevPlace - The Developer Social Network",
         description="Track industry shifts. Discover bold releases. Share what you're building in an open, uncensored environment.",
         breadcrumbs=[],
         schemas=[website_schema(base)],
     )
-    return templates.TemplateResponse("landing.html", {
+    return templates.TemplateResponse(request, "landing.html", {
         **seo_ctx, "request": request,
         "landing_articles": landing_articles,
         "landing_posts": landing_posts,

devplacepy/models.py

@@ -1,49 +1,322 @@
-from pydantic import BaseModel, Field
-from typing import Optional
-from datetime import date
+from datetime import datetime
+from typing import Literal, Optional
+from pydantic import BaseModel, Field, field_validator, model_validator
+from devplacepy.constants import TOPICS, REACTION_EMOJI
 
 
-class SignupRequest(BaseModel):
-    username: str = Field(min_length=3, max_length=32, pattern=r"^[a-zA-Z0-9_-]+$")
-    email: str = Field(min_length=5, max_length=255)
+def normalize_european_date(value):
+    if not value:
+        return ""
+    text = str(value).strip()
+    if not text:
+        return ""
+    for fmt in ("%d/%m/%Y", "%Y-%m-%d"):
+        try:
+            return datetime.strptime(text, fmt).strftime("%Y-%m-%d")
+        except ValueError:
+            continue
+    raise ValueError("Date must be in DD/MM/YYYY format")
+
+
+def normalize_poll_options(value):
+    if value is None:
+        return []
+    if isinstance(value, str):
+        value = [value]
+    if not isinstance(value, list):
+        return value
+    if len(value) == 1 and isinstance(value[0], str):
+        single = value[0]
+        separator = "\n" if "\n" in single else ("," if "," in single else "")
+        if separator:
+            return [part.strip() for part in single.split(separator) if part.strip()]
+    return value
+
+
+class SignupForm(BaseModel):
+    username: str = Field(min_length=3, max_length=32)
+    email: str = Field(min_length=1, max_length=255)
     password: str = Field(min_length=6, max_length=128)
-    confirm_password: str = Field(min_length=6, max_length=128)
+    confirm_password: str = Field(min_length=1, max_length=128)
+
+    @field_validator("username")
+    @classmethod
+    def username_chars(cls, value):
+        if not value.isascii() or not all(c.isalnum() or c in ("-", "_") for c in value):
+            raise ValueError("Username can only contain letters, numbers, hyphens, and underscores")
+        return value
+
+    @field_validator("email")
+    @classmethod
+    def email_has_at(cls, value):
+        if "@" not in value:
+            raise ValueError("Valid email is required")
+        return value
+
+    @model_validator(mode="after")
+    def passwords_match(self):
+        if self.password != self.confirm_password:
+            raise ValueError("Passwords do not match")
+        return self
 
 
-class LoginRequest(BaseModel):
-    email: str = Field(min_length=5, max_length=255)
+class LoginForm(BaseModel):
+    email: str = Field(min_length=1, max_length=255)
+    password: str = Field(min_length=1, max_length=128)
+    remember_me: str = ""
+    next: str = ""
+
+
+class ForgotPasswordForm(BaseModel):
+    email: str = Field(min_length=1, max_length=255)
+
+    @field_validator("email")
+    @classmethod
+    def email_has_at(cls, value):
+        if "@" not in value:
+            raise ValueError("Valid email is required")
+        return value
+
+
+class ResetPasswordForm(BaseModel):
     password: str = Field(min_length=6, max_length=128)
-    remember_me: bool = False
+    confirm_password: str = Field(min_length=1, max_length=128)
+
+    @model_validator(mode="after")
+    def passwords_match(self):
+        if self.password != self.confirm_password:
+            raise ValueError("Passwords do not match")
+        return self
 
 
-class PostCreate(BaseModel):
-    content: str = Field(min_length=1, max_length=2000)
-    title: Optional[str] = Field(default=None, max_length=500)
-    topic: str = Field(pattern=r"^(devlog|showcase|question|rant|fun|random)$")
-    project_uid: Optional[str] = None
+class PostForm(BaseModel):
+    content: str = Field(min_length=10, max_length=2000)
+    title: str = Field(default="", max_length=500)
+    topic: str = "random"
+    project_uid: str = ""
+    attachment_uids: list[str] = []
+    poll_question: str = Field(default="", max_length=200)
+    poll_options: list[str] = []
+
+    @field_validator("topic")
+    @classmethod
+    def valid_topic(cls, value):
+        return value if value in TOPICS else "random"
+
+    @field_validator("poll_options", mode="before")
+    @classmethod
+    def split_poll_options(cls, value):
+        return normalize_poll_options(value)
 
 
-class CommentCreate(BaseModel):
-    content: str = Field(min_length=1, max_length=1000)
-    parent_uid: Optional[str] = None
+class PostEditForm(BaseModel):
+    content: str = Field(min_length=10, max_length=2000)
+    title: str = Field(default="", max_length=500)
+    topic: str = "random"
+    poll_question: str = Field(default="", max_length=200)
+    poll_options: list[str] = []
+
+    @field_validator("topic")
+    @classmethod
+    def valid_topic(cls, value):
+        return value if value in TOPICS else "random"
+
+    @field_validator("poll_options", mode="before")
+    @classmethod
+    def split_poll_options(cls, value):
+        return normalize_poll_options(value)
 
 
-class ProjectCreate(BaseModel):
+class CommentForm(BaseModel):
+    content: str = Field(min_length=3, max_length=1000)
+    target_uid: str = ""
+    post_uid: str = ""
+    target_type: Literal["post", "project", "news", "bug", "gist"] = "post"
+    parent_uid: str = ""
+    attachment_uids: list[str] = []
+
+    @model_validator(mode="after")
+    def require_target(self):
+        if not (self.target_uid or self.post_uid):
+            raise ValueError("A target is required")
+        return self
+
+
+class ProjectForm(BaseModel):
     title: str = Field(min_length=1, max_length=200)
     description: str = Field(min_length=1, max_length=5000)
-    release_date: Optional[date] = None
-    demo_date: Optional[date] = None
-    project_type: str = Field(pattern=r"^(game|game_asset|software|mobile_app|website)$")
+    release_date: str = ""
+    demo_date: str = ""
+    project_type: Literal["game", "game_asset", "software", "mobile_app", "website"] = "software"
     platforms: str = Field(default="", max_length=500)
+    status: str = Field(default="In Development", max_length=100)
+    is_private: bool = False
+    attachment_uids: list[str] = []
+
+    @field_validator("release_date", "demo_date", mode="before")
+    @classmethod
+    def normalize_dates(cls, value):
+        return normalize_european_date(value)
 
 
-class MessageCreate(BaseModel):
+class ProjectFlagForm(BaseModel):
+    value: bool = False
+
+
+class ForkForm(BaseModel):
+    title: str = Field(min_length=1, max_length=200)
+
+
+class ProjectFileWriteForm(BaseModel):
+    path: str = Field(min_length=1, max_length=1024)
+    content: str = Field(default="", max_length=400000)
+
+
+class ProjectFileMkdirForm(BaseModel):
+    path: str = Field(min_length=1, max_length=1024)
+
+
+class ProjectFileMoveForm(BaseModel):
+    from_path: str = Field(min_length=1, max_length=1024)
+    to_path: str = Field(min_length=1, max_length=1024)
+
+
+class ProjectFileDeleteForm(BaseModel):
+    path: str = Field(min_length=1, max_length=1024)
+
+
+class ProjectFileReplaceLinesForm(BaseModel):
+    path: str = Field(min_length=1, max_length=1024)
+    start: int = Field(ge=1)
+    end: int = Field(ge=0)
+    content: str = Field(default="", max_length=400000)
+
+
+class ProjectFileInsertLinesForm(BaseModel):
+    path: str = Field(min_length=1, max_length=1024)
+    at: int = Field(ge=1)
+    content: str = Field(default="", max_length=400000)
+
+
+class ProjectFileDeleteLinesForm(BaseModel):
+    path: str = Field(min_length=1, max_length=1024)
+    start: int = Field(ge=1)
+    end: int = Field(ge=1)
+
+
+class ProjectFileAppendForm(BaseModel):
+    path: str = Field(min_length=1, max_length=1024)
+    content: str = Field(default="", max_length=400000)
+
+
+class ContainerInstanceForm(BaseModel):
+    name: str = Field(min_length=1, max_length=64)
+    boot_command: str = Field(default="", max_length=500)
+    env: str = Field(default="", max_length=10000)
+    cpu_limit: str = Field(default="", max_length=16)
+    mem_limit: str = Field(default="", max_length=16)
+    ports: str = Field(default="", max_length=500)
+    volumes: str = Field(default="", max_length=2000)
+    restart_policy: str = Field(default="never", max_length=20)
+    autostart: bool = True
+    ingress_slug: str = Field(default="", max_length=64)
+    ingress_port: Optional[int] = Field(default=None, ge=1, le=65535)
+
+
+class ContainerExecForm(BaseModel):
+    command: str = Field(min_length=1, max_length=2000)
+
+
+class ContainerScheduleForm(BaseModel):
+    action: str = Field(max_length=10)
+    kind: str = Field(max_length=10)
+    cron: str = Field(default="", max_length=120)
+    run_at: str = Field(default="", max_length=40)
+    delay_seconds: Optional[int] = Field(default=None, ge=1)
+    every_seconds: Optional[int] = Field(default=None, ge=1)
+    max_runs: Optional[int] = Field(default=None, ge=1)
+
+
+class MessageForm(BaseModel):
     content: str = Field(min_length=1, max_length=2000)
-    receiver_uid: str
+    receiver_uid: str = Field(min_length=1)
+    attachment_uids: list[str] = []
 
 
-class ProfileUpdate(BaseModel):
-    bio: Optional[str] = Field(default=None, max_length=500)
-    location: Optional[str] = Field(default=None, max_length=200)
-    git_link: Optional[str] = Field(default=None, max_length=500)
-    website: Optional[str] = Field(default=None, max_length=500)
+class ProfileForm(BaseModel):
+    bio: str = Field(default="", max_length=500)
+    location: str = Field(default="", max_length=200)
+    git_link: str = Field(default="", max_length=500)
+    website: str = Field(default="", max_length=500)
+
+
+class GistForm(BaseModel):
+    title: str = Field(min_length=1, max_length=200)
+    description: str = Field(default="", max_length=5000)
+    source_code: str = Field(min_length=1, max_length=400000)
+    language: str = Field(default="plaintext", max_length=50)
+    attachment_uids: list[str] = []
+
+
+class GistEditForm(BaseModel):
+    title: str = Field(min_length=1, max_length=200)
+    description: str = Field(default="", max_length=5000)
+    source_code: str = Field(min_length=1, max_length=400000)
+    language: str = Field(default="plaintext", max_length=50)
+
+
+class BugForm(BaseModel):
+    title: str = Field(min_length=1, max_length=200)
+    description: str = Field(min_length=1, max_length=5000)
+    attachment_uids: list[str] = []
+
+
+class VoteForm(BaseModel):
+    value: int
+
+    @field_validator("value")
+    @classmethod
+    def valid_value(cls, value):
+        if value not in (1, -1):
+            raise ValueError("value must be 1 or -1")
+        return value
+
+
+class ReactionForm(BaseModel):
+    emoji: str = Field(min_length=1, max_length=16)
+
+    @field_validator("emoji")
+    @classmethod
+    def valid_emoji(cls, value):
+        if value not in REACTION_EMOJI:
+            raise ValueError("Invalid reaction")
+        return value
+
+
+class PollVoteForm(BaseModel):
+    option_uid: str = Field(min_length=1)
+
+
+class AdminRoleForm(BaseModel):
+    role: Literal["member", "admin"]
+
+
+class AdminPasswordForm(BaseModel):
+    password: str = Field(min_length=6, max_length=128)
+
+
+class AdminSettingsForm(BaseModel):
+    site_name: str = Field(default="", max_length=200)
+    site_description: str = Field(default="", max_length=500)
+    site_tagline: str = Field(default="", max_length=500)
+    site_url: str = Field(default="", max_length=300)
+    max_upload_size_mb: str = Field(default="", max_length=10)
+    allowed_file_types: str = Field(default="", max_length=1000)
+    max_attachments_per_resource: str = Field(default="", max_length=10)
+    rate_limit_per_minute: str = Field(default="", max_length=10)
+    rate_limit_window_seconds: str = Field(default="", max_length=10)
+    session_max_age_days: str = Field(default="", max_length=10)
+    session_remember_days: str = Field(default="", max_length=10)
+    registration_open: str = Field(default="", max_length=1)
+    maintenance_mode: str = Field(default="", max_length=1)
+    maintenance_message: str = Field(default="", max_length=300)

devplacepy/project_files.py

@@ -0,0 +1,565 @@
+# retoor <retoor@molodetz.nl>
+
+import logging
+import os
+import shutil
+from datetime import datetime, timezone
+from pathlib import Path
+
+from devplacepy.database import get_table, db, get_int_setting
+from devplacepy.config import STATIC_DIR
+from devplacepy.utils import generate_uid
+from devplacepy.attachments import _directory_for, _detect_mime
+
+logger = logging.getLogger(__name__)
+
+PROJECT_FILES_DIR = STATIC_DIR / "uploads" / "project_files"
+MAX_TEXT_CHARS = 400_000
+MAX_FILES_PER_PROJECT = 5000
+MAX_PATH_LENGTH = 1024
+MAX_SEGMENT_LENGTH = 255
+MAX_DEPTH = 32
+
+TEXT_EXTENSIONS = {
+    ".txt", ".md", ".markdown", ".rst", ".log", ".csv", ".tsv",
+    ".py", ".pyi", ".ipynb", ".js", ".mjs", ".cjs", ".jsx", ".ts", ".tsx",
+    ".json", ".jsonc", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".conf", ".env",
+    ".html", ".htm", ".xml", ".svg", ".css", ".scss", ".sass", ".less",
+    ".c", ".h", ".cpp", ".cc", ".hpp", ".hh", ".cs", ".java", ".kt", ".kts",
+    ".go", ".rs", ".rb", ".php", ".pl", ".pm", ".lua", ".r", ".dart", ".swift",
+    ".scala", ".clj", ".ex", ".exs", ".erl", ".hs", ".ml", ".vue", ".svelte",
+    ".sh", ".bash", ".zsh", ".fish", ".ps1", ".bat", ".cmd",
+    ".sql", ".graphql", ".gql", ".proto", ".tf", ".hcl", ".dockerfile",
+    ".gitignore", ".gitattributes", ".editorconfig", ".properties",
+}
+TEXT_FILENAMES = {
+    "dockerfile", "makefile", "license", "readme", "changelog", "authors",
+    "procfile", "rakefile", "gemfile", ".gitignore", ".gitattributes",
+    ".env", ".editorconfig", ".dockerignore", ".npmrc", ".prettierrc", ".babelrc",
+}
+
+
+class ProjectFileError(ValueError):
+    pass
+
+
+def is_readonly(project_uid: str) -> bool:
+    if "projects" not in db.tables:
+        return False
+    project = get_table("projects").find_one(uid=project_uid)
+    return bool(project and project.get("read_only"))
+
+
+def _guard_writable(project_uid: str) -> None:
+    if is_readonly(project_uid):
+        raise ProjectFileError("project is read-only; files cannot be modified")
+
+
+def normalize_path(raw) -> str:
+    if raw is None:
+        raise ProjectFileError("path is required")
+    text = str(raw).strip().replace("\\", "/")
+    if "\x00" in text:
+        raise ProjectFileError("path contains a null byte")
+    parts = []
+    for segment in text.split("/"):
+        segment = segment.strip()
+        if segment in ("", "."):
+            continue
+        if segment == "..":
+            raise ProjectFileError("path may not contain '..'")
+        if len(segment) > MAX_SEGMENT_LENGTH:
+            raise ProjectFileError("path segment is too long")
+        if any(ord(ch) < 32 for ch in segment):
+            raise ProjectFileError("path contains control characters")
+        parts.append(segment)
+    if not parts:
+        raise ProjectFileError("path is empty")
+    if len(parts) > MAX_DEPTH:
+        raise ProjectFileError("path is too deep")
+    normalized = "/".join(parts)
+    if len(normalized) > MAX_PATH_LENGTH:
+        raise ProjectFileError("path is too long")
+    return normalized
+
+
+def _name_of(path: str) -> str:
+    return path.rsplit("/", 1)[-1]
+
+
+def _parent_of(path: str) -> str:
+    return path.rsplit("/", 1)[0] if "/" in path else ""
+
+
+def is_text_name(name: str) -> bool:
+    lower = name.lower()
+    if lower in TEXT_FILENAMES:
+        return True
+    return Path(name).suffix.lower() in TEXT_EXTENSIONS
+
+
+def _table():
+    return get_table("project_files")
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _file_url(directory: str, stored_name: str) -> str:
+    return f"/static/uploads/project_files/{directory}/{stored_name}"
+
+
+def get_node(project_uid: str, path: str):
+    return _table().find_one(project_uid=project_uid, path=path)
+
+
+def _count(project_uid: str) -> int:
+    if "project_files" not in db.tables:
+        return 0
+    return _table().count(project_uid=project_uid)
+
+
+def _guard_capacity(project_uid: str) -> None:
+    if _count(project_uid) >= MAX_FILES_PER_PROJECT:
+        raise ProjectFileError(f"project reached the {MAX_FILES_PER_PROJECT}-node limit")
+
+
+def _insert_dir(project_uid: str, user: dict, path: str) -> None:
+    _table().insert({
+        "uid": generate_uid(),
+        "project_uid": project_uid,
+        "user_uid": user["uid"],
+        "path": path,
+        "name": _name_of(path),
+        "parent_path": _parent_of(path),
+        "type": "dir",
+        "content": None,
+        "is_binary": 0,
+        "stored_name": None,
+        "directory": None,
+        "mime_type": None,
+        "size": 0,
+        "created_at": _now(),
+        "updated_at": _now(),
+    })
+
+
+def ensure_dirs(project_uid: str, user: dict, dir_path: str) -> None:
+    if not dir_path:
+        return
+    current = ""
+    for segment in dir_path.split("/"):
+        current = f"{current}/{segment}" if current else segment
+        existing = get_node(project_uid, current)
+        if existing is None:
+            _guard_capacity(project_uid)
+            _insert_dir(project_uid, user, current)
+        elif existing["type"] != "dir":
+            raise ProjectFileError(f"'{current}' exists and is a file")
+
+
+def make_dir(project_uid: str, user: dict, raw_path: str) -> dict:
+    _guard_writable(project_uid)
+    path = normalize_path(raw_path)
+    existing = get_node(project_uid, path)
+    if existing is not None:
+        if existing["type"] != "dir":
+            raise ProjectFileError(f"'{path}' exists and is a file")
+        return existing
+    ensure_dirs(project_uid, user, path)
+    return get_node(project_uid, path)
+
+
+def write_text_file(project_uid: str, user: dict, raw_path: str, content: str) -> dict:
+    _guard_writable(project_uid)
+    path = normalize_path(raw_path)
+    content = content or ""
+    if len(content) > MAX_TEXT_CHARS:
+        raise ProjectFileError(f"file content exceeds the {MAX_TEXT_CHARS}-character limit")
+    ensure_dirs(project_uid, user, _parent_of(path))
+    existing = get_node(project_uid, path)
+    if existing is not None and existing["type"] == "dir":
+        raise ProjectFileError(f"'{path}' is a directory")
+    if existing is not None:
+        if existing.get("is_binary"):
+            _unlink_blob(existing)
+        _table().update({
+            "uid": existing["uid"],
+            "content": content,
+            "is_binary": 0,
+            "stored_name": None,
+            "directory": None,
+            "mime_type": "text/plain",
+            "size": len(content.encode("utf-8")),
+            "updated_at": _now(),
+        }, ["uid"])
+        return get_node(project_uid, path)
+    _guard_capacity(project_uid)
+    _table().insert({
+        "uid": generate_uid(),
+        "project_uid": project_uid,
+        "user_uid": user["uid"],
+        "path": path,
+        "name": _name_of(path),
+        "parent_path": _parent_of(path),
+        "type": "file",
+        "content": content,
+        "is_binary": 0,
+        "stored_name": None,
+        "directory": None,
+        "mime_type": "text/plain",
+        "size": len(content.encode("utf-8")),
+        "created_at": _now(),
+        "updated_at": _now(),
+    })
+    return get_node(project_uid, path)
+
+
+def _store_blob(data: bytes, filename: str) -> tuple[str, str, str]:
+    uid = generate_uid()
+    ext = Path(filename).suffix.lower()
+    stored_name = f"{uid}{ext}"
+    directory = _directory_for(uid)
+    file_dir = PROJECT_FILES_DIR / directory
+    file_dir.mkdir(parents=True, exist_ok=True)
+    (file_dir / stored_name).write_bytes(data)
+    return stored_name, directory, _detect_mime(data, filename)
+
+
+def _unlink_blob(node: dict) -> None:
+    stored_name = node.get("stored_name")
+    directory = node.get("directory")
+    if not (stored_name and directory):
+        return
+    try:
+        (PROJECT_FILES_DIR / directory / stored_name).unlink(missing_ok=True)
+    except OSError as exc:
+        logger.warning("Failed to delete project blob %s/%s: %s", directory, stored_name, exc)
+
+
+def store_upload(project_uid: str, user: dict, dir_path: str, filename: str, data: bytes) -> dict:
+    _guard_writable(project_uid)
+    max_bytes = get_int_setting("max_upload_size_mb", 10) * 1024 * 1024
+    if len(data) > max_bytes:
+        raise ProjectFileError(f"file exceeds the {get_int_setting('max_upload_size_mb', 10)}MB limit")
+    base = _name_of(normalize_path(filename))
+    path = normalize_path(f"{dir_path}/{base}" if dir_path else base)
+
+    if is_text_name(base):
+        try:
+            text = data.decode("utf-8")
+            if len(text) <= MAX_TEXT_CHARS:
+                return write_text_file(project_uid, user, path, text)
+        except UnicodeDecodeError:
+            pass
+
+    ensure_dirs(project_uid, user, _parent_of(path))
+    existing = get_node(project_uid, path)
+    if existing is not None and existing["type"] == "dir":
+        raise ProjectFileError(f"'{path}' is a directory")
+    stored_name, directory, mime = _store_blob(data, base)
+    if existing is not None:
+        if existing.get("is_binary"):
+            _unlink_blob(existing)
+        _table().update({
+            "uid": existing["uid"], "content": None, "is_binary": 1,
+            "stored_name": stored_name, "directory": directory, "mime_type": mime,
+            "size": len(data), "updated_at": _now(),
+        }, ["uid"])
+        return get_node(project_uid, path)
+    _guard_capacity(project_uid)
+    _table().insert({
+        "uid": generate_uid(),
+        "project_uid": project_uid,
+        "user_uid": user["uid"],
+        "path": path,
+        "name": base,
+        "parent_path": _parent_of(path),
+        "type": "file",
+        "content": None,
+        "is_binary": 1,
+        "stored_name": stored_name,
+        "directory": directory,
+        "mime_type": mime,
+        "size": len(data),
+        "created_at": _now(),
+        "updated_at": _now(),
+    })
+    return get_node(project_uid, path)
+
+
+def _descendants(project_uid: str, path: str) -> list:
+    prefix = f"{path}/"
+    return [
+        row for row in _table().find(project_uid=project_uid)
+        if row["path"] == path or row["path"].startswith(prefix)
+    ]
+
+
+def move_node(project_uid: str, user: dict, raw_from: str, raw_to: str) -> dict:
+    _guard_writable(project_uid)
+    src_path = normalize_path(raw_from)
+    dst_path = normalize_path(raw_to)
+    if src_path == dst_path:
+        raise ProjectFileError("source and target are the same")
+    if dst_path == src_path or dst_path.startswith(f"{src_path}/"):
+        raise ProjectFileError("cannot move a node into itself")
+    src = get_node(project_uid, src_path)
+    if src is None:
+        raise ProjectFileError(f"'{src_path}' does not exist")
+    if get_node(project_uid, dst_path) is not None:
+        raise ProjectFileError(f"'{dst_path}' already exists")
+    ensure_dirs(project_uid, user, _parent_of(dst_path))
+
+    if src["type"] == "file":
+        _table().update({
+            "uid": src["uid"], "path": dst_path, "name": _name_of(dst_path),
+            "parent_path": _parent_of(dst_path), "updated_at": _now(),
+        }, ["uid"])
+        return get_node(project_uid, dst_path)
+
+    for row in _descendants(project_uid, src_path):
+        new_path = dst_path + row["path"][len(src_path):]
+        _table().update({
+            "uid": row["uid"], "path": new_path, "name": _name_of(new_path),
+            "parent_path": _parent_of(new_path), "updated_at": _now(),
+        }, ["uid"])
+    return get_node(project_uid, dst_path)
+
+
+def delete_node(project_uid: str, raw_path: str) -> None:
+    _guard_writable(project_uid)
+    path = normalize_path(raw_path)
+    node = get_node(project_uid, path)
+    if node is None:
+        raise ProjectFileError(f"'{path}' does not exist")
+    for row in _descendants(project_uid, path):
+        if row.get("is_binary"):
+            _unlink_blob(row)
+        _table().delete(uid=row["uid"])
+
+
+def node_to_dict(row: dict) -> dict:
+    is_binary = bool(row.get("is_binary"))
+    url = None
+    if is_binary and row.get("stored_name") and row.get("directory"):
+        url = _file_url(row["directory"], row["stored_name"])
+    return {
+        "uid": row["uid"],
+        "path": row["path"],
+        "name": row["name"],
+        "type": row["type"],
+        "is_binary": is_binary,
+        "mime_type": row.get("mime_type"),
+        "size": row.get("size", 0),
+        "url": url,
+        "updated_at": row.get("updated_at"),
+    }
+
+
+def list_files(project_uid: str) -> list:
+    if "project_files" not in db.tables:
+        return []
+    rows = sorted(_table().find(project_uid=project_uid), key=lambda r: r["path"])
+    return [node_to_dict(row) for row in rows]
+
+
+def read_file(project_uid: str, raw_path: str) -> dict:
+    path = normalize_path(raw_path)
+    node = get_node(project_uid, path)
+    if node is None:
+        raise ProjectFileError(f"'{path}' does not exist")
+    payload = node_to_dict(node)
+    payload["content"] = node.get("content") if node["type"] == "file" and not node.get("is_binary") else None
+    return payload
+
+
+IMPORT_SKIP_NAMES = {
+    ".git", "node_modules", "__pycache__", ".venv", "venv",
+    ".mypy_cache", ".pytest_cache", ".DS_Store", ".idea", ".cache",
+}
+IMPORT_MAX_FILE_BYTES = 25 * 1024 * 1024
+
+
+def import_from_dir(project_uid: str, src_dir, user: dict, *, skip_names=None) -> int:
+    _guard_writable(project_uid)
+    src = Path(src_dir).resolve()
+    if not src.is_dir():
+        raise ProjectFileError("source directory does not exist")
+    skip = set(skip_names) if skip_names is not None else set(IMPORT_SKIP_NAMES)
+    imported = 0
+    for root, dirs, files in os.walk(src):
+        dirs[:] = [d for d in sorted(dirs) if d not in skip]
+        for name in sorted(files):
+            if name in skip:
+                continue
+            full = Path(root) / name
+            if full.is_symlink() or not full.is_file():
+                continue
+            try:
+                if full.stat().st_size > IMPORT_MAX_FILE_BYTES:
+                    continue
+                relative = full.relative_to(src).as_posix()
+                path = normalize_path(relative)
+                data = full.read_bytes()
+            except (OSError, ProjectFileError):
+                continue
+            try:
+                store_upload(project_uid, user, _parent_of(path), _name_of(path), data)
+                imported += 1
+            except ProjectFileError:
+                continue
+    return imported
+
+
+def export_to_dir(project_uid: str, subpath: str, dest_dir) -> int:
+    dest = Path(dest_dir).resolve()
+    dest.mkdir(parents=True, exist_ok=True)
+    if subpath:
+        base = normalize_path(subpath)
+        rows = _descendants(project_uid, base)
+        strip = _parent_of(base)
+    else:
+        rows = list(_table().find(project_uid=project_uid))
+        strip = ""
+    written = 0
+    for row in sorted(rows, key=lambda r: r["path"]):
+        relative = row["path"][len(strip):].lstrip("/") if strip else row["path"]
+        target = (dest / relative).resolve()
+        if target != dest and not target.is_relative_to(dest):
+            raise ProjectFileError("path escapes export directory")
+        if row["type"] == "dir":
+            target.mkdir(parents=True, exist_ok=True)
+            continue
+        target.parent.mkdir(parents=True, exist_ok=True)
+        if target.is_symlink() or target.is_file():
+            target.unlink()
+        if row.get("is_binary") and row.get("stored_name") and row.get("directory"):
+            shutil.copyfile(PROJECT_FILES_DIR / row["directory"] / row["stored_name"], target)
+        else:
+            target.write_text(row.get("content") or "", encoding="utf-8")
+        written += 1
+    return written
+
+
+def _load_text_node(project_uid: str, raw_path: str) -> tuple[dict, str]:
+    path = normalize_path(raw_path)
+    node = get_node(project_uid, path)
+    if node is None:
+        raise ProjectFileError(f"'{path}' does not exist")
+    if node["type"] != "file":
+        raise ProjectFileError(f"'{path}' is a directory, not a file")
+    if node.get("is_binary"):
+        raise ProjectFileError(f"'{path}' is a binary file and has no editable lines")
+    return node, node.get("content") or ""
+
+
+def _split_lines(content: str) -> tuple[list, bool]:
+    if content == "":
+        return [], False
+    trailing = content.endswith("\n")
+    body = content[:-1] if trailing else content
+    return body.split("\n"), trailing
+
+
+def _join_lines(lines: list, trailing: bool) -> str:
+    if not lines:
+        return ""
+    return "\n".join(lines) + ("\n" if trailing else "")
+
+
+def _replacement_lines(content: str) -> list:
+    return _split_lines(content)[0] if content else []
+
+
+def _save_text(project_uid: str, node: dict, new_content: str) -> dict:
+    if len(new_content) > MAX_TEXT_CHARS:
+        raise ProjectFileError(f"file content exceeds the {MAX_TEXT_CHARS}-character limit")
+    _table().update({
+        "uid": node["uid"],
+        "content": new_content,
+        "is_binary": 0,
+        "stored_name": None,
+        "directory": None,
+        "mime_type": "text/plain",
+        "size": len(new_content.encode("utf-8")),
+        "updated_at": _now(),
+    }, ["uid"])
+    return get_node(project_uid, node["path"])
+
+
+def read_lines(project_uid: str, raw_path: str, start: int = 1, end=None) -> dict:
+    node, content = _load_text_node(project_uid, raw_path)
+    lines, _ = _split_lines(content)
+    total = len(lines)
+    start = max(1, int(start))
+    end = total if end is None else int(end)
+    if end < 0:
+        end = total
+    end = min(end, total)
+    selected = lines[start - 1:end] if start <= total and end >= start else []
+    return {
+        "path": node["path"],
+        "start": start,
+        "end": end if selected else start - 1,
+        "total_lines": total,
+        "lines": selected,
+        "content": "\n".join(selected),
+    }
+
+
+def replace_lines(project_uid: str, raw_path: str, start: int, end: int, content: str) -> dict:
+    _guard_writable(project_uid)
+    node, original = _load_text_node(project_uid, raw_path)
+    lines, trailing = _split_lines(original)
+    total = len(lines)
+    start = max(1, int(start))
+    end = int(end)
+    if start > total:
+        raise ProjectFileError(f"start line {start} is past the end of the file ({total} lines)")
+    end = min(max(end, start - 1), total)
+    new_lines = lines[:start - 1] + _replacement_lines(content) + lines[end:]
+    return _save_text(project_uid, node, _join_lines(new_lines, trailing if new_lines else False))
+
+
+def insert_lines(project_uid: str, raw_path: str, at: int, content: str) -> dict:
+    _guard_writable(project_uid)
+    node, original = _load_text_node(project_uid, raw_path)
+    lines, trailing = _split_lines(original)
+    total = len(lines)
+    at = max(1, int(at))
+    if at > total + 1:
+        at = total + 1
+    new_lines = lines[:at - 1] + _replacement_lines(content) + lines[at - 1:]
+    return _save_text(project_uid, node, _join_lines(new_lines, trailing if total else False))
+
+
+def delete_lines(project_uid: str, raw_path: str, start: int, end: int) -> dict:
+    _guard_writable(project_uid)
+    node, original = _load_text_node(project_uid, raw_path)
+    lines, trailing = _split_lines(original)
+    total = len(lines)
+    start = max(1, int(start))
+    end = min(int(end), total)
+    if start > total or end < start:
+        raise ProjectFileError(f"no lines in range {start}-{end} (file has {total} lines)")
+    new_lines = lines[:start - 1] + lines[end:]
+    return _save_text(project_uid, node, _join_lines(new_lines, trailing if new_lines else False))
+
+
+def append_lines(project_uid: str, raw_path: str, content: str) -> dict:
+    _guard_writable(project_uid)
+    node, original = _load_text_node(project_uid, raw_path)
+    lines, trailing = _split_lines(original)
+    new_lines = lines + _replacement_lines(content)
+    return _save_text(project_uid, node, _join_lines(new_lines, trailing if lines else False))
+
+
+def delete_all_project_files(project_uid: str) -> None:
+    if "project_files" not in db.tables:
+        return
+    for row in _table().find(project_uid=project_uid):
+        if row.get("is_binary"):
+            _unlink_blob(row)
+    _table().delete(project_uid=project_uid)

devplacepy/push.py

@@ -0,0 +1,286 @@
+# retoor <retoor@molodetz.nl>
+
+import base64
+import fcntl
+import json
+import logging
+import os
+import random
+import time
+from datetime import datetime, timezone
+from typing import Any
+from urllib.parse import urlparse
+
+import httpx
+import jwt
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric import ec
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+from cryptography.hazmat.primitives.hashes import SHA256
+from cryptography.hazmat.primitives.kdf.hkdf import HKDF
+
+from devplacepy.config import (
+    SECONDS_PER_DAY,
+    VAPID_PRIVATE_KEY_FILE,
+    VAPID_PRIVATE_KEY_PKCS8_FILE,
+    VAPID_PUBLIC_KEY_FILE,
+    VAPID_SUB,
+)
+from devplacepy.database import get_table
+from devplacepy.utils import generate_uid
+
+logger = logging.getLogger(__name__)
+
+JWT_LIFETIME_SECONDS = 60 * 60
+PUSH_TTL_SECONDS = str(SECONDS_PER_DAY)
+DEAD_SUBSCRIPTION_STATUSES = (404, 410)
+ACCEPTED_STATUSES = (200, 201)
+
+
+def generate_private_key() -> None:
+    if not VAPID_PRIVATE_KEY_FILE.exists():
+        private_key = ec.generate_private_key(ec.SECP256R1(), default_backend())
+        pem = private_key.private_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PrivateFormat.TraditionalOpenSSL,
+            encryption_algorithm=serialization.NoEncryption(),
+        )
+        VAPID_PRIVATE_KEY_FILE.write_bytes(pem)
+        logger.info("Generated VAPID private key at %s", VAPID_PRIVATE_KEY_FILE)
+
+
+def generate_pkcs8_private_key() -> None:
+    if not VAPID_PRIVATE_KEY_PKCS8_FILE.exists():
+        private_key = serialization.load_pem_private_key(
+            VAPID_PRIVATE_KEY_FILE.read_bytes(), password=None, backend=default_backend()
+        )
+        pem = private_key.private_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PrivateFormat.PKCS8,
+            encryption_algorithm=serialization.NoEncryption(),
+        )
+        VAPID_PRIVATE_KEY_PKCS8_FILE.write_bytes(pem)
+        logger.info("Generated VAPID PKCS8 private key at %s", VAPID_PRIVATE_KEY_PKCS8_FILE)
+
+
+def generate_public_key() -> None:
+    if not VAPID_PUBLIC_KEY_FILE.exists():
+        private_key = serialization.load_pem_private_key(
+            VAPID_PRIVATE_KEY_FILE.read_bytes(), password=None, backend=default_backend()
+        )
+        pem = private_key.public_key().public_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PublicFormat.SubjectPublicKeyInfo,
+        )
+        VAPID_PUBLIC_KEY_FILE.write_bytes(pem)
+        logger.info("Generated VAPID public key at %s", VAPID_PUBLIC_KEY_FILE)
+
+
+def ensure_certificates() -> None:
+    if VAPID_PRIVATE_KEY_FILE.exists() and VAPID_PRIVATE_KEY_PKCS8_FILE.exists() and VAPID_PUBLIC_KEY_FILE.exists():
+        return
+    lock_path = VAPID_PRIVATE_KEY_FILE.parent / ".vapid.lock"
+    with open(lock_path, "w") as lock:
+        fcntl.flock(lock, fcntl.LOCK_EX)
+        try:
+            generate_private_key()
+            generate_pkcs8_private_key()
+            generate_public_key()
+        finally:
+            fcntl.flock(lock, fcntl.LOCK_UN)
+
+
+def hkdf(input_key: bytes, salt: bytes, info: bytes, length: int) -> bytes:
+    return HKDF(
+        algorithm=SHA256(),
+        length=length,
+        salt=salt,
+        info=info,
+        backend=default_backend(),
+    ).derive(input_key)
+
+
+def browser_base64(data: bytes) -> str:
+    return base64.urlsafe_b64encode(data).decode("utf-8").rstrip("=")
+
+
+_keys: dict[str, Any] = {}
+
+
+def _load_keys() -> dict[str, Any]:
+    if _keys:
+        return _keys
+    ensure_certificates()
+    private_key = serialization.load_pem_private_key(
+        VAPID_PRIVATE_KEY_FILE.read_bytes(), password=None, backend=default_backend()
+    )
+    public_key = serialization.load_pem_public_key(
+        VAPID_PUBLIC_KEY_FILE.read_bytes(), backend=default_backend()
+    )
+    uncompressed_point = public_key.public_bytes(
+        encoding=serialization.Encoding.X962,
+        format=serialization.PublicFormat.UncompressedPoint,
+    )
+    _keys["private_key_pem"] = private_key.private_bytes(
+        encoding=serialization.Encoding.PEM,
+        format=serialization.PrivateFormat.TraditionalOpenSSL,
+        encryption_algorithm=serialization.NoEncryption(),
+    )
+    _keys["public_key_point"] = uncompressed_point
+    _keys["public_key_base64"] = browser_base64(uncompressed_point)
+    logger.debug("Loaded VAPID key material into cache")
+    return _keys
+
+
+def public_key_standard_b64() -> str:
+    point = _load_keys()["public_key_point"]
+    return base64.b64encode(point).decode("utf-8").rstrip("=")
+
+
+def create_notification_authorization(push_url: str) -> str:
+    target = urlparse(push_url)
+    audience = f"{target.scheme}://{target.netloc}"
+    issued_at = int(time.time())
+    return jwt.encode(
+        {
+            "sub": VAPID_SUB,
+            "aud": audience,
+            "exp": issued_at + JWT_LIFETIME_SECONDS,
+            "nbf": issued_at,
+            "iat": issued_at,
+            "jti": generate_uid(),
+        },
+        _load_keys()["private_key_pem"],
+        algorithm="ES256",
+    )
+
+
+def create_notification_info_with_payload(
+    endpoint: str, auth: str, p256dh: str, payload: str
+) -> dict[str, Any]:
+    message_private_key = ec.generate_private_key(ec.SECP256R1(), default_backend())
+    message_public_key_bytes = message_private_key.public_key().public_bytes(
+        encoding=serialization.Encoding.X962,
+        format=serialization.PublicFormat.UncompressedPoint,
+    )
+
+    salt = os.urandom(16)
+
+    user_key_bytes = base64.urlsafe_b64decode(p256dh + "==")
+    shared_secret = message_private_key.exchange(
+        ec.ECDH(),
+        ec.EllipticCurvePublicKey.from_encoded_point(ec.SECP256R1(), user_key_bytes),
+    )
+
+    encryption_key = hkdf(
+        shared_secret,
+        base64.urlsafe_b64decode(auth + "=="),
+        b"Content-Encoding: auth\x00",
+        32,
+    )
+
+    context = (
+        b"P-256\x00"
+        + len(user_key_bytes).to_bytes(2, "big")
+        + user_key_bytes
+        + len(message_public_key_bytes).to_bytes(2, "big")
+        + message_public_key_bytes
+    )
+
+    nonce = hkdf(encryption_key, salt, b"Content-Encoding: nonce\x00" + context, 12)
+    content_encryption_key = hkdf(
+        encryption_key, salt, b"Content-Encoding: aesgcm\x00" + context, 16
+    )
+
+    padding_length = random.randint(0, 16)
+    padding = padding_length.to_bytes(2, "big") + b"\x00" * padding_length
+    data = AESGCM(content_encryption_key).encrypt(
+        nonce, padding + payload.encode("utf-8"), None
+    )
+
+    return {
+        "headers": {
+            "Authorization": f"WebPush {create_notification_authorization(endpoint)}",
+            "Crypto-Key": f"dh={browser_base64(message_public_key_bytes)}; p256ecdsa={_load_keys()['public_key_base64']}",
+            "Encryption": f"salt={browser_base64(salt)}",
+            "Content-Encoding": "aesgcm",
+            "Content-Length": str(len(data)),
+            "Content-Type": "application/octet-stream",
+        },
+        "data": data,
+    }
+
+
+def _mark_subscription_dead(subscription_id: int) -> None:
+    get_table("push_registration").update(
+        {"id": subscription_id, "deleted_at": datetime.now(timezone.utc).isoformat()},
+        ["id"],
+    )
+    logger.info("Soft-deleted dead push subscription id=%s", subscription_id)
+
+
+async def notify_user(user_uid: str, payload: dict[str, Any]) -> None:
+    registrations = list(
+        get_table("push_registration").find(user_uid=user_uid, deleted_at=None)
+    )
+    if not registrations:
+        logger.debug("No active push subscriptions for user %s", user_uid)
+        return
+
+    body = json.dumps(payload)
+    async with httpx.AsyncClient(timeout=10.0) as client:
+        for subscription in registrations:
+            endpoint = subscription["endpoint"]
+            try:
+                notification_info = create_notification_info_with_payload(
+                    endpoint,
+                    subscription["key_auth"],
+                    subscription["key_p256dh"],
+                    body,
+                )
+                headers = {**notification_info["headers"], "TTL": PUSH_TTL_SECONDS}
+                response = await client.post(
+                    endpoint, headers=headers, content=notification_info["data"]
+                )
+            except (httpx.HTTPError, ValueError) as exc:
+                logger.warning("Push error for %s via %s: %s", user_uid, endpoint, exc)
+                continue
+
+            if response.status_code in ACCEPTED_STATUSES:
+                logger.debug("Push delivered to %s via %s", user_uid, endpoint)
+            elif response.status_code in DEAD_SUBSCRIPTION_STATUSES:
+                _mark_subscription_dead(subscription["id"])
+            else:
+                logger.warning(
+                    "Push rejected (%s) for %s via %s", response.status_code, user_uid, endpoint
+                )
+
+
+async def register(
+    user_uid: str, endpoint: str, key_auth: str, key_p256dh: str
+) -> tuple[dict[str, Any], bool]:
+    table = get_table("push_registration")
+    existing = table.find_one(
+        user_uid=user_uid,
+        endpoint=endpoint,
+        key_auth=key_auth,
+        key_p256dh=key_p256dh,
+        deleted_at=None,
+    )
+    if existing:
+        logger.debug("Push subscription already registered for user %s", user_uid)
+        return existing, False
+
+    record = {
+        "uid": generate_uid(),
+        "user_uid": user_uid,
+        "endpoint": endpoint,
+        "key_auth": key_auth,
+        "key_p256dh": key_p256dh,
+        "created_at": datetime.now(timezone.utc).isoformat(),
+        "deleted_at": None,
+    }
+    table.insert(record)
+    logger.info("Registered push subscription for user %s", user_uid)
+    return record, True

devplacepy/responses.py

@@ -0,0 +1,42 @@
+import logging
+
+from fastapi import Request
+from fastapi.responses import JSONResponse, RedirectResponse
+
+from devplacepy.schemas import ActionResult
+
+logger = logging.getLogger(__name__)
+
+
+def wants_json(request: Request) -> bool:
+    # NOTE: `X-Requested-With: fetch` is intentionally NOT a trigger here. The browser
+    # frontend sends it on form/engagement fetches and relies on the legacy HTML/redirect
+    # behaviour (the four engagement endpoints handle that header themselves). JSON is
+    # requested via the Accept or Content-Type header so existing flows are unchanged.
+    content_type = request.headers.get("content-type", "")
+    if content_type.startswith("application/json"):
+        return True
+    accept = request.headers.get("accept", "")
+    return "application/json" in accept and "text/html" not in accept
+
+
+def respond(request: Request, template: str, context: dict, *, model):
+    if wants_json(request):
+        try:
+            payload = model.model_validate(context).model_dump(mode="json")
+        except Exception as exc:
+            logger.warning("JSON serialization failed for %s: %s", template, exc)
+            return json_error(500, "Could not serialize response")
+        return JSONResponse(payload)
+    from devplacepy.templating import templates
+    return templates.TemplateResponse(request, template, context)
+
+
+def action_result(request: Request, redirect_url: str, *, data=None, status_code: int = 302):
+    if wants_json(request):
+        return JSONResponse(ActionResult(ok=True, redirect=redirect_url, data=data).model_dump(mode="json"))
+    return RedirectResponse(url=redirect_url, status_code=status_code)
+
+
+def json_error(status_code: int, message: str, **extra) -> JSONResponse:
+    return JSONResponse({"error": {"status": status_code, "message": message, **extra}}, status_code=status_code)

devplacepy/routers/admin.py

@@ -1,11 +1,17 @@
 import logging
-from datetime import datetime
-from fastapi import APIRouter, Request
-from fastapi.responses import HTMLResponse, RedirectResponse
-from devplacepy.database import get_table, db, build_pagination
+from typing import Annotated
+from fastapi import APIRouter, Request, Form
+from devplacepy.models import AdminRoleForm, AdminPasswordForm, AdminSettingsForm
+from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse
+from devplacepy.database import get_table, build_pagination, get_post_counts_by_user_uids, get_news_images_by_uids, clear_settings_cache, get_platform_analytics
 from devplacepy.templating import templates
-from devplacepy.utils import require_admin, hash_password, generate_uid, time_ago
+from devplacepy.utils import require_admin, hash_password, generate_uid, time_ago, clear_user_cache, get_current_user, is_admin
 from devplacepy.seo import base_seo_context, site_url, website_schema
+from devplacepy.responses import respond, action_result
+from devplacepy.schemas import AdminUsersOut, AdminNewsOut, AdminSettingsOut, GatewayUsageOut
+from devplacepy.services.manager import service_manager
+from devplacepy.services.openai_gateway.analytics import build_analytics, build_user_usage
+from devplacepy.services.openai_gateway.usage import pricing_from_cfg
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
@@ -14,24 +20,81 @@ router = APIRouter()
 @router.get("", response_class=HTMLResponse)
 async def admin_index(request: Request):
     require_admin(request)
-    return RedirectResponse(url="/admin/users", status_code=302)
+    return action_result(request, "/admin/users")
+
+
+@router.get("/analytics")
+async def admin_analytics(request: Request, top_n: int = 10):
+    user = get_current_user(request)
+    if not user:
+        return JSONResponse({"error": "Authentication required"}, status_code=401)
+    if not is_admin(user):
+        return JSONResponse({"error": "Admin access required"}, status_code=403)
+    return JSONResponse(get_platform_analytics(top_n))
+
+
+@router.get("/ai-usage", response_class=HTMLResponse)
+async def admin_ai_usage(request: Request):
+    admin = require_admin(request)
+    base = site_url(request)
+    seo_ctx = base_seo_context(
+        request,
+        title="AI usage - Admin",
+        description="AI gateway token usage, cost, latency, and reliability metrics.",
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {"name": "Admin", "url": "/admin"},
+            {"name": "AI usage", "url": "/admin/ai-usage"},
+        ],
+        schemas=[website_schema(base)],
+    )
+    return respond(request, "ai_usage.html", {
+        **seo_ctx,
+        "request": request,
+        "user": admin,
+        "admin_section": "ai-usage",
+    }, model=GatewayUsageOut)
+
+
+@router.get("/ai-usage/data")
+async def admin_ai_usage_data(request: Request, hours: int = 48, top_n: int = 10):
+    user = get_current_user(request)
+    if not user:
+        return JSONResponse({"error": "Authentication required"}, status_code=401)
+    if not is_admin(user):
+        return JSONResponse({"error": "Admin access required"}, status_code=403)
+    service = service_manager.get_service("openai")
+    pricing = pricing_from_cfg(service.get_config()) if service is not None else None
+    return JSONResponse(build_analytics(hours, top_n=top_n, pricing=pricing))
+
+
+@router.get("/users/{uid}/ai-usage")
+async def admin_user_ai_usage(request: Request, uid: str, hours: int = 24):
+    user = get_current_user(request)
+    if not user:
+        return JSONResponse({"error": "Authentication required"}, status_code=401)
+    if not is_admin(user):
+        return JSONResponse({"error": "Admin access required"}, status_code=403)
+    service = service_manager.get_service("openai")
+    pricing = pricing_from_cfg(service.get_config()) if service is not None else None
+    return JSONResponse(build_user_usage(uid, hours=hours, pricing=pricing))
 
 
 @router.get("/users", response_class=HTMLResponse)
 async def admin_users(request: Request, page: int = 1):
     admin = require_admin(request)
     users_table = get_table("users")
-    total = len(list(users_table.all()))
+    total = users_table.count()
     pagination = build_pagination(page, total)
     offset = (pagination["page"] - 1) * pagination["per_page"]
     page_users = list(users_table.find(order_by=["-created_at"], _limit=pagination["per_page"], _offset=offset))
+    post_counts = get_post_counts_by_user_uids([u["uid"] for u in page_users])
     for u in page_users:
-        posts_count = len(list(get_table("posts").find(user_uid=u["uid"])))
-        u["posts_count"] = posts_count
+        u["posts_count"] = post_counts.get(u["uid"], 0)
     base = site_url(request)
     seo_ctx = base_seo_context(
         request,
-        title="Users — Admin",
+        title="Users - Admin",
         description="Manage DevPlace users.",
         breadcrumbs=[
             {"name": "Home", "url": "/feed"},
@@ -40,56 +103,78 @@ async def admin_users(request: Request, page: int = 1):
         ],
         schemas=[website_schema(base)],
     )
-    return templates.TemplateResponse("admin_users.html", {
+    return respond(request, "admin_users.html", {
         **seo_ctx,
         "request": request,
         "user": admin,
         "users": page_users,
         "pagination": pagination,
         "admin_section": "users",
-    })
+    }, model=AdminUsersOut)
 
 
 @router.post("/users/{uid}/role")
-async def admin_user_role(request: Request, uid: str):
+async def admin_user_role(request: Request, uid: str, data: Annotated[AdminRoleForm, Form()]):
     admin = require_admin(request)
-    form = await request.form()
-    role = form.get("role", "").strip().capitalize()
-    if role not in ("Member", "Admin"):
-        return RedirectResponse(url="/admin/users", status_code=302)
+    role = data.role.capitalize()
     if uid == admin["uid"]:
-        return RedirectResponse(url="/admin/users", status_code=302)
+        return action_result(request, "/admin/users")
     users = get_table("users")
     users.update({"uid": uid, "role": role}, ["uid"])
+    clear_user_cache(uid)
     logger.info(f"Admin {admin['username']} set user {uid} role to {role}")
-    return RedirectResponse(url="/admin/users", status_code=302)
+    return action_result(request, "/admin/users")
 
 
 @router.post("/users/{uid}/password")
-async def admin_user_password(request: Request, uid: str):
+async def admin_user_password(request: Request, uid: str, data: Annotated[AdminPasswordForm, Form()]):
     admin = require_admin(request)
-    form = await request.form()
-    password = form.get("password", "")
-    if len(password) < 6:
-        return RedirectResponse(url="/admin/users", status_code=302)
     users = get_table("users")
-    users.update({"uid": uid, "password_hash": hash_password(password)}, ["uid"])
+    users.update({"uid": uid, "password_hash": hash_password(data.password)}, ["uid"])
     logger.info(f"Admin {admin['username']} changed password for user {uid}")
-    return RedirectResponse(url="/admin/users", status_code=302)
+    return action_result(request, "/admin/users")
 
 
 @router.post("/users/{uid}/toggle")
 async def admin_user_toggle(request: Request, uid: str):
     admin = require_admin(request)
     if uid == admin["uid"]:
-        return RedirectResponse(url="/admin/users", status_code=302)
+        return action_result(request, "/admin/users")
     users = get_table("users")
     user = users.find_one(uid=uid)
     if user:
         new_state = not user.get("is_active", True)
         users.update({"uid": uid, "is_active": new_state}, ["uid"])
+        clear_user_cache(uid)
         logger.info(f"Admin {admin['username']} {'disabled' if not new_state else 'enabled'} user {uid}")
-    return RedirectResponse(url="/admin/users", status_code=302)
+    return action_result(request, "/admin/users")
+
+
+@router.post("/users/{uid}/reset-ai-quota")
+async def admin_user_reset_ai_quota(request: Request, uid: str):
+    admin = require_admin(request)
+    devii = service_manager.get_service("devii")
+    removed = devii.reset_quota("user", uid) if devii is not None else 0
+    logger.info(f"Admin {admin['username']} reset AI quota for user {uid} ({removed} ledger rows)")
+    return action_result(request, "/admin/users")
+
+
+@router.post("/ai-quota/reset-guests")
+async def admin_reset_guest_ai_quota(request: Request):
+    admin = require_admin(request)
+    devii = service_manager.get_service("devii")
+    removed = devii.reset_guest_quotas() if devii is not None else 0
+    logger.info(f"Admin {admin['username']} reset all guest AI quotas ({removed} ledger rows)")
+    return action_result(request, "/admin/ai-usage")
+
+
+@router.post("/ai-quota/reset-all")
+async def admin_reset_all_ai_quota(request: Request):
+    admin = require_admin(request)
+    devii = service_manager.get_service("devii")
+    removed = devii.reset_all_quotas() if devii is not None else 0
+    logger.info(f"Admin {admin['username']} reset ALL AI quotas ({removed} ledger rows)")
+    return action_result(request, "/admin/ai-usage")
 
 
 @router.get("/settings", response_class=HTMLResponse)
@@ -100,7 +185,7 @@ async def admin_settings(request: Request):
     base = site_url(request)
     seo_ctx = base_seo_context(
         request,
-        title="Settings — Admin",
+        title="Settings - Admin",
         description="Manage DevPlace site settings.",
         breadcrumbs=[
             {"name": "Home", "url": "/feed"},
@@ -109,42 +194,39 @@ async def admin_settings(request: Request):
         ],
         schemas=[website_schema(base)],
     )
-    return templates.TemplateResponse("admin_settings.html", {
+    return respond(request, "admin_settings.html", {
         **seo_ctx,
         "request": request,
         "user": admin,
         "settings": all_settings,
         "admin_section": "settings",
-    })
+    }, model=AdminSettingsOut)
 
 
 @router.get("/news", response_class=HTMLResponse)
 async def admin_news(request: Request, page: int = 1):
     admin = require_admin(request)
     news_table = get_table("news")
-    total = len(list(news_table.all()))
+    total = news_table.count()
     pagination = build_pagination(page, total)
     offset = (pagination["page"] - 1) * pagination["per_page"]
     page_articles = list(news_table.find(order_by=["-synced_at"], _limit=pagination["per_page"], _offset=offset))
-    images_table = get_table("news_images")
+    images_by_news = get_news_images_by_uids([a["uid"] for a in page_articles])
 
     enriched = []
     for a in page_articles:
-        has_image = False
-        if "news_images" in db.tables:
-            has_image = images_table.find_one(news_uid=a["uid"]) is not None
         enriched.append({
             "article": a,
             "time_ago": time_ago(a["synced_at"]),
             "synced_at": a.get("synced_at", ""),
             "grade": a.get("grade", 0),
-            "has_image": has_image,
+            "has_image": a["uid"] in images_by_news,
         })
 
     base = site_url(request)
     seo_ctx = base_seo_context(
         request,
-        title="News — Admin",
+        title="News - Admin",
         description="Manage DevPlace news articles.",
         breadcrumbs=[
             {"name": "Home", "url": "/feed"},
@@ -153,14 +235,14 @@ async def admin_news(request: Request, page: int = 1):
         ],
         schemas=[website_schema(base)],
     )
-    return templates.TemplateResponse("admin_news.html", {
+    return respond(request, "admin_news.html", {
         **seo_ctx,
         "request": request,
         "user": admin,
         "articles": enriched,
         "pagination": pagination,
         "admin_section": "news",
-    })
+    }, model=AdminNewsOut)
 
 
 @router.post("/news/{uid}/toggle")
@@ -172,7 +254,7 @@ async def admin_news_toggle(request: Request, uid: str):
         current = article.get("featured", 0)
         news_table.update({"uid": uid, "featured": 0 if current else 1}, ["uid"])
         logger.info(f"Admin {admin['username']} toggled news {uid} featured={'off' if current else 'on'}")
-    return RedirectResponse(url="/admin/news", status_code=302)
+    return action_result(request, "/admin/news")
 
 
 @router.post("/news/{uid}/publish")
@@ -185,7 +267,7 @@ async def admin_news_publish(request: Request, uid: str):
         new_status = "draft" if current == "published" else "published"
         news_table.update({"uid": uid, "status": new_status}, ["uid"])
         logger.info(f"Admin {admin['username']} set news {uid} status={new_status}")
-    return RedirectResponse(url="/admin/news", status_code=302)
+    return action_result(request, "/admin/news")
 
 
 @router.post("/news/{uid}/landing")
@@ -197,7 +279,7 @@ async def admin_news_landing(request: Request, uid: str):
         current = article.get("show_on_landing", 0)
         news_table.update({"uid": uid, "show_on_landing": 0 if current else 1}, ["uid"])
         logger.info(f"Admin {admin['username']} toggled news {uid} landing={'on' if not current else 'off'}")
-    return RedirectResponse(url="/admin/news", status_code=302)
+    return action_result(request, "/admin/news")
 
 
 @router.post("/news/{uid}/delete")
@@ -211,19 +293,21 @@ async def admin_news_delete(request: Request, uid: str):
             images_table.delete(id=img["id"])
         news_table.delete(id=article["id"])
         logger.info(f"Admin {admin['username']} deleted news {uid}")
-    return RedirectResponse(url="/admin/news", status_code=302)
+    return action_result(request, "/admin/news")
 
 
 @router.post("/settings")
-async def admin_settings_save(request: Request):
+async def admin_settings_save(request: Request, data: Annotated[AdminSettingsForm, Form()]):
     admin = require_admin(request)
-    form = await request.form()
     settings = get_table("site_settings")
-    for key, value in form.multi_items():
+    for key, value in data.model_dump().items():
         existing = settings.find_one(key=key)
         if existing:
+            if value == "":
+                continue
             settings.update({"id": existing["id"], "key": key, "value": value}, ["id"])
         else:
             settings.insert({"uid": generate_uid(), "key": key, "value": value})
+    clear_settings_cache()
     logger.info(f"Admin {admin['username']} updated settings")
-    return RedirectResponse(url="/admin/settings", status_code=302)
+    return action_result(request, "/admin/settings")

devplacepy/routers/auth.py

@@ -1,13 +1,18 @@
 import hashlib
 import secrets
 import logging
-from datetime import datetime, timedelta
-from fastapi import APIRouter, Request
-from fastapi.responses import RedirectResponse, HTMLResponse
-from devplacepy.database import get_table
+from typing import Annotated
+from datetime import datetime, timedelta, timezone
+from fastapi import APIRouter, Request, Form
+from fastapi.responses import RedirectResponse, HTMLResponse, JSONResponse
+from devplacepy.database import get_table, get_setting, get_int_setting
 from devplacepy.templating import templates
-from devplacepy.utils import hash_password, verify_password, create_session, generate_uid, get_current_user
+from devplacepy.utils import hash_password, verify_password, create_session, generate_uid, get_current_user, award_badge, clear_session_cache, safe_next
 from devplacepy.seo import base_seo_context
+from devplacepy.models import SignupForm, LoginForm, ForgotPasswordForm, ResetPasswordForm
+from devplacepy.config import SECONDS_PER_DAY
+from devplacepy.responses import respond, action_result, wants_json, json_error
+from devplacepy.schemas import AuthPageOut
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
@@ -17,50 +22,47 @@ router = APIRouter()
 async def signup_page(request: Request):
     user = get_current_user(request)
     if user:
-        return RedirectResponse(url="/feed", status_code=302)
+        return action_result(request, "/feed")
     seo_ctx = base_seo_context(
         request,
         title="Join DevPlace",
         description="Create your DevPlace account and start connecting with developers.",
         robots="noindex,nofollow",
     )
-    return templates.TemplateResponse("signup.html", {**seo_ctx, "request": request})
+    registration_closed = get_setting("registration_open", "1") != "1"
+    return respond(request, "signup.html", {**seo_ctx, "request": request, "page": "signup", "registration_closed": registration_closed}, model=AuthPageOut)
 
 
 @router.get("/login", response_class=HTMLResponse)
-async def login_page(request: Request):
+async def login_page(request: Request, next: str | None = None):
     user = get_current_user(request)
     if user:
-        return RedirectResponse(url="/feed", status_code=302)
+        return action_result(request, safe_next(next))
     seo_ctx = base_seo_context(
         request,
         title="Sign In",
         description="Sign in to DevPlace to connect with developers.",
         robots="noindex,nofollow",
     )
-    return templates.TemplateResponse("login.html", {**seo_ctx, "request": request})
+    return respond(request, "login.html", {**seo_ctx, "request": request, "page": "login", "next_url": safe_next(next, "")}, model=AuthPageOut)
 
 
 @router.post("/signup")
-async def signup(request: Request):
-    form = await request.form()
-    username = form.get("username", "").strip()
-    email = form.get("email", "").strip().lower()
-    password = form.get("password", "")
-    confirm_password = form.get("confirm_password", "")
+async def signup(request: Request, data: Annotated[SignupForm, Form()]):
+    username = data.username
+    email = data.email.strip().lower()
+    password = data.password
+
+    if get_setting("registration_open", "1") != "1":
+        if wants_json(request):
+            return json_error(403, "Registration is closed")
+        seo_ctx = base_seo_context(request, title="Join DevPlace", robots="noindex,nofollow")
+        return templates.TemplateResponse(
+            request, "signup.html",
+            {**seo_ctx, "request": request, "registration_closed": True},
+        )
 
     errors = []
-    if len(username) < 3 or len(username) > 32:
-        errors.append("Username must be between 3 and 32 characters")
-    if not username.isascii() or not all(c.isalnum() or c in ("-", "_") for c in username):
-        errors.append("Username can only contain letters, numbers, hyphens, and underscores")
-    if not email or "@" not in email or len(email) > 255:
-        errors.append("Valid email is required")
-    if len(password) < 6:
-        errors.append("Password must be at least 6 characters")
-    if password != confirm_password:
-        errors.append("Passwords do not match")
-
     users = get_table("users")
     if users.find_one(username=username):
         errors.append("Username already taken")
@@ -68,18 +70,21 @@ async def signup(request: Request):
         errors.append("Email already registered")
 
     if errors:
+        if wants_json(request):
+            return json_error(400, "; ".join(errors), errors=errors)
         seo_ctx = base_seo_context(request, title="Join DevPlace", robots="noindex,nofollow")
         return templates.TemplateResponse(
-            "signup.html",
+            request, "signup.html",
             {**seo_ctx, "request": request, "errors": errors, "username": username, "email": email},
         )
 
     uid = generate_uid()
-    is_first = len(list(users.all())) == 0
+    is_first = users.count() == 0
     users.insert({
         "uid": uid,
         "username": username,
         "email": email,
+        "api_key": generate_uid(),
         "password_hash": hash_password(password),
         "bio": "",
         "location": "",
@@ -90,51 +95,45 @@ async def signup(request: Request):
         "level": 1,
         "xp": 0,
         "stars": 0,
-        "created_at": datetime.utcnow().isoformat(),
+        "created_at": datetime.now(timezone.utc).isoformat(),
     })
 
-    badges = get_table("badges")
-    badges.insert({
-        "uid": generate_uid(),
-        "user_uid": uid,
-        "badge_name": "Member",
-        "created_at": datetime.utcnow().isoformat(),
-    })
+    award_badge(uid, "Member")
 
-    token = create_session(uid)
-    response = RedirectResponse(url="/feed", status_code=302)
-    response.set_cookie(key="session", value=token, max_age=86400 * 7, httponly=True, samesite="lax")
+    max_age = max(1, get_int_setting("session_max_age_days", 7)) * SECONDS_PER_DAY
+    token = create_session(uid, max_age)
+    response = action_result(request, "/feed", data={"username": username})
+    response.set_cookie(key="session", value=token, max_age=max_age, httponly=True, samesite="lax")
     logger.info(f"User {username} signed up")
     return response
 
 
 @router.post("/login")
-async def login(request: Request):
-    form = await request.form()
-    email = form.get("email", "").strip().lower()
-    password = form.get("password", "")
-    remember_me = form.get("remember_me") == "on"
+async def login(request: Request, data: Annotated[LoginForm, Form()]):
+    email = data.email.strip().lower()
+    password = data.password
+    remember_me = data.remember_me == "on"
 
     errors = []
-    if not email or not password:
-        errors.append("Email and password are required")
-
     users = get_table("users")
-    user = users.find_one(email=email) if not errors else None
+    user = users.find_one(email=email)
 
     if not user or not verify_password(password, user["password_hash"]):
         errors.append("Invalid email or password")
 
     if errors:
+        if wants_json(request):
+            return json_error(401, "; ".join(errors), errors=errors)
         seo_ctx = base_seo_context(request, title="Sign In", robots="noindex,nofollow")
         return templates.TemplateResponse(
-            "login.html",
-            {**seo_ctx, "request": request, "errors": errors, "email": email},
+            request, "login.html",
+            {**seo_ctx, "request": request, "errors": errors, "email": email, "next_url": safe_next(data.next, "")},
         )
 
-    token = create_session(user["uid"])
-    max_age = 86400 * 30 if remember_me else 86400 * 7
-    response = RedirectResponse(url="/feed", status_code=302)
+    days = get_int_setting("session_remember_days", 30) if remember_me else get_int_setting("session_max_age_days", 7)
+    max_age = max(1, days) * SECONDS_PER_DAY
+    token = create_session(user["uid"], max_age)
+    response = action_result(request, safe_next(data.next), data={"username": user["username"]})
     response.set_cookie(key="session", value=token, max_age=max_age, httponly=True, samesite="lax")
     logger.info(f"User {user['username']} logged in")
     return response
@@ -147,22 +146,13 @@ async def forgot_password_page(request: Request):
         title="Reset Password",
         robots="noindex,nofollow",
     )
-    return templates.TemplateResponse("forgot_password.html", {**seo_ctx, "request": request})
+    return respond(request, "forgot_password.html", {**seo_ctx, "request": request, "page": "forgot-password"}, model=AuthPageOut)
 
 
 @router.post("/forgot-password")
-async def forgot_password(request: Request):
-    form = await request.form()
-    email = form.get("email", "").strip().lower()
-    errors = []
-    if not email or "@" not in email:
-        errors.append("Valid email is required")
-
+async def forgot_password(request: Request, data: Annotated[ForgotPasswordForm, Form()]):
+    email = data.email.strip().lower()
     seo_ctx = base_seo_context(request, title="Reset Password", robots="noindex,nofollow")
-    if errors:
-        return templates.TemplateResponse("forgot_password.html", {
-            **seo_ctx, "request": request, "errors": errors,
-        })
 
     users = get_table("users")
     user = users.find_one(email=email)
@@ -174,13 +164,15 @@ async def forgot_password(request: Request):
             "uid": generate_uid(),
             "user_uid": user["uid"],
             "token": token_hash,
-            "expires_at": (datetime.utcnow() + timedelta(hours=1)).isoformat(),
+            "expires_at": (datetime.now(timezone.utc) + timedelta(hours=1)).isoformat(),
             "used": False,
-            "created_at": datetime.utcnow().isoformat(),
+            "created_at": datetime.now(timezone.utc).isoformat(),
         })
         logger.info(f"Password reset requested for {email}")
 
-    return templates.TemplateResponse("forgot_password.html", {
+    if wants_json(request):
+        return JSONResponse({"ok": True, "sent": True})
+    return templates.TemplateResponse(request, "forgot_password.html", {
         **seo_ctx, "request": request, "sent": True,
     })
 
@@ -192,44 +184,40 @@ async def reset_password_page(request: Request, token: str):
         title="Set New Password",
         robots="noindex,nofollow",
     )
-    return templates.TemplateResponse("reset_password.html", {
-        **seo_ctx, "request": request, "token": token,
-    })
+    return respond(request, "reset_password.html", {
+        **seo_ctx, "request": request, "page": "reset-password", "token": token,
+    }, model=AuthPageOut)
 
 
 @router.post("/reset-password/{token}")
-async def reset_password(request: Request, token: str):
-    form = await request.form()
-    password = form.get("password", "")
-    confirm = form.get("confirm_password", "")
-
+async def reset_password(request: Request, token: str, data: Annotated[ResetPasswordForm, Form()]):
+    password = data.password
     errors = []
-    if len(password) < 6:
-        errors.append("Password must be at least 6 characters")
-    if password != confirm:
-        errors.append("Passwords do not match")
-
-    if errors:
-        seo_ctx = base_seo_context(request, title="Set New Password", robots="noindex,nofollow")
-        return templates.TemplateResponse("reset_password.html", {
-            **seo_ctx, "request": request, "token": token, "errors": errors,
-        })
 
     resets = get_table("password_resets")
     users = get_table("users")
     token_hash = hashlib.sha256(token.encode()).hexdigest()
     matched = resets.find_one(token=token_hash, used=False)
 
-    if not matched or datetime.fromisoformat(matched["expires_at"]) < datetime.utcnow():
+    expired = False
+    if matched:
+        expires = datetime.fromisoformat(matched["expires_at"])
+        if expires.tzinfo is None:
+            expires = expires.replace(tzinfo=timezone.utc)
+        expired = expires < datetime.now(timezone.utc)
+
+    if not matched or expired:
         errors.append("Invalid or expired reset token")
-        return templates.TemplateResponse("reset_password.html", {
+        if wants_json(request):
+            return json_error(400, errors[0], errors=errors)
+        return templates.TemplateResponse(request, "reset_password.html", {
             "request": request, "token": token, "errors": errors,
         })
 
     users.update({"uid": matched["user_uid"], "password_hash": hash_password(password)}, ["uid"])
     resets.update({"id": matched["id"], "used": True}, ["id"])
     logger.info(f"Password reset completed for user {matched['user_uid']}")
-    return RedirectResponse(url="/auth/login", status_code=302)
+    return action_result(request, "/auth/login")
 
 
 @router.get("/logout")
@@ -240,6 +228,7 @@ async def logout(request: Request):
         session = sessions.find_one(session_token=token)
         if session:
             sessions.delete(id=session["id"])
-    response = RedirectResponse(url="/", status_code=302)
+        clear_session_cache(token)
+    response = action_result(request, "/")
     response.delete_cookie("session")
     return response

devplacepy/routers/avatar.py

@@ -1,20 +1,29 @@
+import hashlib
 import logging
 from fastapi import APIRouter, Request
 from fastapi.responses import Response
 from devplacepy.avatar import generate_avatar_svg
+from devplacepy.cache import TTLCache
+from devplacepy.config import SECONDS_PER_DAY
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
-_cache = {}
+_cache = TTLCache(ttl=SECONDS_PER_DAY, max_size=4096)
+_CACHE_CONTROL = f"public, max-age={SECONDS_PER_DAY}, immutable"
 
 
 @router.get("/{style}/{seed}")
 async def avatar_proxy(request: Request, style: str, seed: str, size: int = 128):
     cache_key = f"{seed}:{size}"
-    if cache_key in _cache:
-        return Response(content=_cache[cache_key], media_type="image/svg+xml")
+    etag = '"' + hashlib.md5(cache_key.encode("utf-8")).hexdigest() + '"'
+    headers = {"ETag": etag, "Cache-Control": _CACHE_CONTROL}
 
-    svg = generate_avatar_svg(seed)
-    _cache[cache_key] = svg
-    return Response(content=svg, media_type="image/svg+xml")
+    if request.headers.get("if-none-match") == etag:
+        return Response(status_code=304, headers=headers)
+
+    svg = _cache.get(cache_key)
+    if svg is None:
+        svg = generate_avatar_svg(seed)
+        _cache.set(cache_key, svg)
+    return Response(content=svg, media_type="image/svg+xml", headers=headers)

devplacepy/routers/bookmarks.py

@@ -0,0 +1,89 @@
+# retoor <retoor@molodetz.nl>
+import logging
+from datetime import datetime, timezone
+from fastapi import APIRouter, Request
+from fastapi.responses import RedirectResponse, JSONResponse, HTMLResponse
+from devplacepy.database import get_table, db, paginate, resolve_object_url
+from devplacepy.templating import templates
+from devplacepy.utils import generate_uid, require_user, time_ago
+from devplacepy.seo import base_seo_context
+from devplacepy.responses import respond
+from devplacepy.schemas import SavedOut
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+BOOKMARKABLE: set[str] = {"post", "gist", "project", "news"}
+
+TABLE_BY_TYPE: dict[str, str] = {"post": "posts", "gist": "gists", "project": "projects", "news": "news"}
+
+LABEL_BY_TYPE: dict[str, str] = {"post": "Post", "gist": "Gist", "project": "Project", "news": "Article"}
+
+
+@router.get("/saved", response_class=HTMLResponse)
+async def saved_page(request: Request, before: str = None):
+    user = require_user(request)
+    bookmarks = get_table("bookmarks")
+    rows, next_cursor = paginate(bookmarks, before=before, user_uid=user["uid"])
+
+    uids_by_type: dict[str, list] = {}
+    for row in rows:
+        uids_by_type.setdefault(row["target_type"], []).append(row["target_uid"])
+
+    resolved: dict[tuple, dict] = {}
+    for target_type, uids in uids_by_type.items():
+        table_name = TABLE_BY_TYPE.get(target_type)
+        if not table_name or table_name not in db.tables:
+            continue
+        table = get_table(table_name)
+        for obj in table.find(table.table.columns.uid.in_(uids)):
+            resolved[(target_type, obj["uid"])] = obj
+
+    items = []
+    for row in rows:
+        obj = resolved.get((row["target_type"], row["target_uid"]))
+        if not obj:
+            continue
+        title = obj.get("title") or (obj.get("content", "") or "")[:80] or "Untitled"
+        items.append({
+            "target_type": row["target_type"],
+            "type_label": LABEL_BY_TYPE.get(row["target_type"], row["target_type"].title()),
+            "title": title,
+            "url": resolve_object_url(row["target_type"], row["target_uid"]),
+            "time_ago": time_ago(row["created_at"]),
+        })
+
+    seo_ctx = base_seo_context(request, title="Saved", description="Your saved content on DevPlace.", robots="noindex,nofollow")
+    return respond(request, "saved.html", {
+        **seo_ctx,
+        "request": request,
+        "user": user,
+        "items": items,
+        "next_cursor": next_cursor,
+    }, model=SavedOut)
+
+
+@router.post("/{target_type}/{target_uid}")
+async def toggle_bookmark(request: Request, target_type: str, target_uid: str):
+    user = require_user(request)
+    if target_type not in BOOKMARKABLE:
+        return JSONResponse({"error": "Invalid target"}, status_code=400)
+
+    bookmarks = get_table("bookmarks")
+    existing = bookmarks.find_one(user_uid=user["uid"], target_uid=target_uid, target_type=target_type)
+    if existing:
+        bookmarks.delete(id=existing["id"])
+        saved = False
+    else:
+        bookmarks.insert({
+            "uid": generate_uid(),
+            "user_uid": user["uid"],
+            "target_uid": target_uid,
+            "target_type": target_type,
+            "created_at": datetime.now(timezone.utc).isoformat(),
+        })
+        saved = True
+
+    if request.headers.get("x-requested-with") == "fetch":
+        return JSONResponse({"saved": saved})
+    return RedirectResponse(url=request.headers.get("Referer", "/feed"), status_code=302)

devplacepy/routers/bugs.py

@@ -1,11 +1,16 @@
 import logging
-from datetime import datetime
-from fastapi import APIRouter, Request
+from typing import Annotated
+from datetime import datetime, timezone
+from fastapi import APIRouter, Request, Form
 from fastapi.responses import HTMLResponse, RedirectResponse
-from devplacepy.database import get_table, load_comments, get_attachments_by_type
+from devplacepy.models import BugForm
+from devplacepy.database import get_table, load_comments
+from devplacepy.attachments import get_attachments_batch, link_attachments
 from devplacepy.templating import templates
 from devplacepy.utils import generate_uid, require_user, time_ago, get_current_user, create_mention_notifications
 from devplacepy.seo import base_seo_context
+from devplacepy.responses import respond, action_result
+from devplacepy.schemas import BugsOut
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
@@ -23,13 +28,13 @@ async def bugs_page(request: Request):
 
     bug_list = []
     bug_uids = [b["uid"] for b in all_bugs]
-    attachments_map = get_attachments_by_type("bug", bug_uids) if bug_uids else {}
+    attachments_map = get_attachments_batch("bug", bug_uids) if bug_uids else {}
     for b in all_bugs:
         bug_list.append({
             "bug": b,
             "author": users_map.get(b["user_uid"]),
             "time_ago": time_ago(b["created_at"]),
-            "comments": load_comments("bug", b["uid"]),
+            "comments": load_comments("bug", b["uid"], user),
             "attachments": attachments_map.get(b["uid"], []),
         })
 
@@ -42,23 +47,19 @@ async def bugs_page(request: Request):
             {"name": "Bug Reports", "url": "/bugs"},
         ],
     )
-    return templates.TemplateResponse("bugs.html", {
+    return respond(request, "bugs.html", {
         **seo_ctx,
         "request": request,
         "user": user,
         "bugs": bug_list,
-    })
+    }, model=BugsOut)
 
 
 @router.post("/create")
-async def create_bug(request: Request):
+async def create_bug(request: Request, data: Annotated[BugForm, Form()]):
     user = require_user(request)
-    form = await request.form()
-    title = form.get("title", "").strip()
-    description = form.get("description", "").strip()
-
-    if not title or not description:
-        return RedirectResponse(url="/bugs", status_code=302)
+    title = data.title.strip()
+    description = data.description.strip()
 
     bugs_table = get_table("bug_reports")
     bug_uid = generate_uid()
@@ -68,11 +69,12 @@ async def create_bug(request: Request):
         "title": title,
         "description": description,
         "status": "open",
-        "created_at": datetime.utcnow().isoformat(),
+        "created_at": datetime.now(timezone.utc).isoformat(),
     })
 
-    attachment_uids = form.getlist("attachment_uids") if hasattr(form, "getlist") else []
+    if data.attachment_uids:
+        link_attachments(data.attachment_uids, "bug", bug_uid)
 
     create_mention_notifications(description, user["uid"], f"/bugs?highlight={bug_uid}")
     logger.info(f"Bug report created by {user['username']}: {title}")
-    return RedirectResponse(url="/bugs", status_code=302)
+    return action_result(request, "/bugs", data={"uid": bug_uid})

devplacepy/routers/comments.py

@@ -1,51 +1,28 @@
 import logging
-from datetime import datetime
-from fastapi import APIRouter, Request
+from typing import Annotated
+from datetime import datetime, timezone
+from fastapi import APIRouter, Request, Form
 from fastapi.responses import RedirectResponse
-from devplacepy.database import get_table, delete_attachments
-from devplacepy.utils import generate_uid, require_user, create_mention_notifications
+from devplacepy.database import get_table, resolve_object_url, delete_engagement
+from devplacepy.attachments import link_attachments, delete_target_attachments
+from devplacepy.content import is_owner
+from devplacepy.utils import generate_uid, require_user, create_mention_notifications, create_notification, award_rewards, XP_COMMENT
+from devplacepy.models import CommentForm
+from devplacepy.responses import action_result, wants_json, json_error
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
 
-def resolve_target_redirect(target_type, target_uid):
-    if target_type == "post":
-        post = resolve_by_slug(get_table("posts"), target_uid)
-        return f"/posts/{post['slug'] or post['uid']}" if post else "/feed"
-    if target_type == "project":
-        project = resolve_by_slug(get_table("projects"), target_uid)
-        return f"/projects/{project['slug'] or project['uid']}" if project else "/projects"
-    if target_type == "news":
-        article = resolve_by_slug(get_table("news"), target_uid)
-        if article:
-            return f"/news/{article.get('slug', '') or article['uid']}"
-        return "/news"
-    if target_type == "bug":
-        return f"/bugs?highlight={target_uid}"
-    if target_type == "gist":
-        gist = resolve_by_slug(get_table("gists"), target_uid)
-        return f"/gists/{gist['slug'] or gist['uid']}" if gist else "/gists"
-    return "/bugs"
-
-
 @router.post("/create")
-async def create_comment(request: Request):
+async def create_comment(request: Request, data: Annotated[CommentForm, Form()]):
     user = require_user(request)
-    form = await request.form()
-    content = form.get("content", "").strip()
-    target_uid = form.get("target_uid") or form.get("post_uid", "")
-    target_type = form.get("target_type", "post")
-    parent_uid = form.get("parent_uid", "")
+    content = data.content.strip()
+    target_uid = data.target_uid or data.post_uid
+    target_type = data.target_type
+    parent_uid = data.parent_uid
 
-    redirect_url = resolve_target_redirect(target_type, target_uid)
-
-    if not content or not target_uid:
-        return RedirectResponse(url=redirect_url, status_code=302)
-    if len(content) < 3:
-        return RedirectResponse(url=redirect_url, status_code=302)
-    if len(content) > 1000:
-        return RedirectResponse(url=redirect_url, status_code=302)
+    redirect_url = resolve_object_url(target_type, target_uid)
 
     comment_uid = generate_uid()
     insert = {
@@ -55,62 +32,35 @@ async def create_comment(request: Request):
         "user_uid": user["uid"],
         "content": content,
         "parent_uid": parent_uid or None,
-        "created_at": datetime.utcnow().isoformat(),
+        "created_at": datetime.now(timezone.utc).isoformat(),
     }
     if target_type == "post":
         insert["post_uid"] = target_uid
     get_table("comments").insert(insert)
 
-    attachment_uids = form.getlist("attachment_uids") if hasattr(form, "getlist") else []
-    for auid in attachment_uids:
-        if auid.strip():
-            get_table("attachments").update({"uid": auid.strip(), "resource_uid": insert["uid"], "resource_type": "comment"}, ["uid"])
+    if data.attachment_uids:
+        link_attachments(data.attachment_uids, "comment", comment_uid)
 
-    badges = get_table("badges")
-    existing = badges.find_one(user_uid=user["uid"], badge_name="First Comment")
-    if not existing:
-        badges.insert({
-            "uid": generate_uid(),
-            "user_uid": user["uid"],
-            "badge_name": "First Comment",
-            "created_at": datetime.utcnow().isoformat(),
-        })
+    award_rewards(user["uid"], XP_COMMENT, "First Comment")
+
+    comment_url = f"{redirect_url}#comment-{comment_uid}"
 
     if target_type == "post":
         if parent_uid:
-            comments_table = get_table("comments")
-            parent = comments_table.find_one(uid=parent_uid)
+            parent = get_table("comments").find_one(uid=parent_uid)
             if parent and parent["user_uid"] != user["uid"]:
-                notifications = get_table("notifications")
-                notifications.insert({
-                    "uid": generate_uid(),
-                    "user_uid": parent["user_uid"],
-                    "type": "reply",
-                    "message": f"{user['username']} replied to your comment",
-                    "related_uid": user["uid"],
-                    "read": False,
-                    "created_at": datetime.utcnow().isoformat(),
-                })
+                create_notification(parent["user_uid"], "reply", f"{user['username']} replied to your comment", user["uid"], comment_url)
         else:
             posts = get_table("posts")
             post = posts.find_one(uid=target_uid)
             if not post:
                 post = posts.find_one(slug=target_uid)
             if post and post["user_uid"] != user["uid"]:
-                notifications = get_table("notifications")
-                notifications.insert({
-                    "uid": generate_uid(),
-                    "user_uid": post["user_uid"],
-                    "type": "comment",
-                    "message": f"{user['username']} commented on your post",
-                    "related_uid": user["uid"],
-                    "read": False,
-                    "created_at": datetime.utcnow().isoformat(),
-                })
+                create_notification(post["user_uid"], "comment", f"{user['username']} commented on your post", user["uid"], comment_url)
 
-    create_mention_notifications(content, user["uid"], redirect_url)
+    create_mention_notifications(content, user["uid"], comment_url)
     logger.info(f"Comment by {user['username']} on {target_type} {target_uid}")
-    return RedirectResponse(url=redirect_url, status_code=302)
+    return action_result(request, comment_url, data={"uid": comment_uid, "url": comment_url})
 
 
 @router.post("/delete/{comment_uid}")
@@ -118,15 +68,16 @@ async def delete_comment(request: Request, comment_uid: str):
     user = require_user(request)
     comments = get_table("comments")
     comment = comments.find_one(uid=comment_uid)
-    if not comment:
-        return RedirectResponse(url="/feed", status_code=302)
-    if comment["user_uid"] != user["uid"]:
+    if not is_owner(comment, user):
+        if wants_json(request):
+            return json_error(403, "Not allowed")
         return RedirectResponse(url="/feed", status_code=302)
     target_type = comment.get("target_type", "post")
     target_uid = comment.get("target_uid") or comment.get("post_uid", "")
-    delete_attachments("comment", comment_uid)
+    delete_target_attachments("comment", comment_uid)
     get_table("votes").delete(target_uid=comment_uid, target_type="comment")
+    delete_engagement("comment", [comment_uid])
     comments.delete(id=comment["id"])
     logger.info(f"Comment {comment_uid} deleted by {user['username']}")
-    redirect_url = resolve_target_redirect(target_type, target_uid)
-    return RedirectResponse(url=redirect_url, status_code=302)
+    redirect_url = resolve_object_url(target_type, target_uid)
+    return action_result(request, redirect_url)

devplacepy/routers/containers.py

@@ -0,0 +1,297 @@
+# retoor <retoor@molodetz.nl>
+
+import asyncio
+import fcntl
+import json
+import logging
+import os
+import pty
+import re
+import signal
+import struct
+import termios
+from typing import Annotated
+
+from fastapi import APIRouter, Form, Request, WebSocket, WebSocketDisconnect
+from fastapi.responses import HTMLResponse, JSONResponse
+
+from devplacepy.database import get_table, resolve_by_slug
+from devplacepy.models import (
+    ContainerExecForm,
+    ContainerInstanceForm,
+    ContainerScheduleForm,
+)
+from devplacepy.responses import action_result, json_error, respond
+from devplacepy.schemas import ContainersOut
+from devplacepy.seo import base_seo_context
+from devplacepy.utils import _user_from_session, is_admin, not_found, require_admin
+from devplacepy.services.manager import service_manager
+from devplacepy.services.containers import api, store
+from devplacepy.services.containers.api import ContainerError
+from devplacepy.services.containers.runtime import get_backend
+from devplacepy.services.devii.tasks.schedule import Schedule, from_iso
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+
+def _project(project_slug: str) -> dict:
+    project = resolve_by_slug(get_table("projects"), project_slug)
+    if not project:
+        raise not_found("Project not found")
+    return project
+
+
+def _slug(project: dict) -> str:
+    return project["slug"] or project["uid"]
+
+
+def _fail(exc: ContainerError):
+    return json_error(400, str(exc))
+
+
+def _instance(project: dict, uid: str) -> dict:
+    inst = store.get_instance(uid)
+    if not inst or inst["project_uid"] != project["uid"]:
+        raise not_found("Instance not found")
+    return inst
+
+
+@router.get("/{project_slug}/containers", response_class=HTMLResponse)
+async def containers_page(request: Request, project_slug: str):
+    user = require_admin(request)
+    project = _project(project_slug)
+    slug = _slug(project)
+    return respond(request, "containers.html", {
+        **base_seo_context(request, title=f"Containers - {project.get('title', 'Project')}",
+                           description="Container manager", robots="noindex,nofollow",
+                           breadcrumbs=[
+                               {"name": "Home", "url": "/feed"},
+                               {"name": project.get("title", "Project"), "url": f"/projects/{slug}"},
+                               {"name": "Containers", "url": f"/projects/{slug}/containers"},
+                           ]),
+        "request": request, "user": user, "project": project,
+        "instances": store.list_instances(project["uid"]),
+    }, model=ContainersOut)
+
+
+@router.get("/{project_slug}/containers/data")
+async def containers_data(request: Request, project_slug: str):
+    require_admin(request)
+    project = _project(project_slug)
+    return JSONResponse({
+        "instances": store.list_instances(project["uid"]),
+    })
+
+
+# ---------------- instances ----------------
+
+@router.post("/{project_slug}/containers/instances")
+async def create_instance(request: Request, project_slug: str, data: Annotated[ContainerInstanceForm, Form()]):
+    user = require_admin(request)
+    project = _project(project_slug)
+    try:
+        inst = await api.create_instance(
+            project, name=data.name, boot_command=data.boot_command, env=data.env,
+            cpu_limit=data.cpu_limit, mem_limit=data.mem_limit, ports=data.ports, volumes=data.volumes,
+            restart_policy=data.restart_policy, autostart=data.autostart,
+            ingress_slug=data.ingress_slug, ingress_port=data.ingress_port, actor=("user", user["uid"]))
+    except ContainerError as exc:
+        return _fail(exc)
+    return action_result(request, f"/projects/{_slug(project)}/containers", data={"instance": inst})
+
+
+@router.get("/{project_slug}/containers/instances/{uid}")
+async def instance_detail(request: Request, project_slug: str, uid: str):
+    require_admin(request)
+    project = _project(project_slug)
+    inst = _instance(project, uid)
+    return JSONResponse({
+        "instance": inst, "events": store.list_events(uid),
+        "schedules": store.list_schedules(uid), "stats": api.instance_stats(uid),
+        "runtime": api.instance_runtime(inst),
+    })
+
+
+_ACTIONS = {
+    "start": store.DESIRED_RUNNING, "stop": store.DESIRED_STOPPED,
+    "pause": store.DESIRED_PAUSED, "resume": store.DESIRED_RUNNING,
+}
+
+
+@router.post("/{project_slug}/containers/instances/{uid}/delete")
+async def delete_instance(request: Request, project_slug: str, uid: str):
+    user = require_admin(request)
+    project = _project(project_slug)
+    inst = _instance(project, uid)
+    api.mark_for_removal(inst, actor=("user", user["uid"]))
+    return action_result(request, f"/projects/{_slug(project)}/containers", data={"uid": uid})
+
+
+@router.post("/{project_slug}/containers/instances/{uid}/exec")
+async def instance_exec(request: Request, project_slug: str, uid: str, data: Annotated[ContainerExecForm, Form()]):
+    user = require_admin(request)
+    project = _project(project_slug)
+    inst = _instance(project, uid)
+    if not inst.get("container_id"):
+        return json_error(400, "instance is not running")
+    result = await get_backend().exec(inst["container_id"], ["/bin/sh", "-c", data.command])
+    store.record_event(inst, "exec", "user", user["uid"], {"command": data.command, "exit_code": result.exit_code})
+    return JSONResponse({"exit_code": result.exit_code, "output": result.output})
+
+
+@router.get("/{project_slug}/containers/instances/{uid}/logs")
+async def instance_logs(request: Request, project_slug: str, uid: str, tail: int = 200):
+    require_admin(request)
+    project = _project(project_slug)
+    inst = _instance(project, uid)
+    if not inst.get("container_id"):
+        return JSONResponse({"logs": ""})
+    lines: list = []
+
+    async def collect(line: str) -> None:
+        lines.append(line)
+
+    await get_backend().logs(inst["container_id"], follow=False, tail=max(1, min(tail, 2000)), on_log=collect)
+    return JSONResponse({"logs": "\n".join(lines)})
+
+
+@router.get("/{project_slug}/containers/instances/{uid}/metrics")
+async def instance_metrics(request: Request, project_slug: str, uid: str):
+    require_admin(request)
+    project = _project(project_slug)
+    inst = _instance(project, uid)
+    return JSONResponse({"metrics": store.recent_metrics(uid), "stats": api.instance_stats(uid)})
+
+
+@router.post("/{project_slug}/containers/instances/{uid}/sync")
+async def instance_sync(request: Request, project_slug: str, uid: str):
+    user = require_admin(request)
+    project = _project(project_slug)
+    inst = _instance(project, uid)
+    try:
+        count = await api.sync_workspace(inst, user)
+    except ContainerError as exc:
+        return _fail(exc)
+    return action_result(request, f"/projects/{_slug(project)}/containers", data={"imported": count})
+
+
+@router.post("/{project_slug}/containers/instances/{uid}/schedules")
+async def create_schedule(request: Request, project_slug: str, uid: str, data: Annotated[ContainerScheduleForm, Form()]):
+    require_admin(request)
+    project = _project(project_slug)
+    inst = _instance(project, uid)
+    try:
+        schedule = Schedule(
+            kind=data.kind, cron=data.cron or None,
+            run_at=from_iso(data.run_at) if data.run_at else None,
+            delay_seconds=data.delay_seconds, every_seconds=data.every_seconds, max_runs=data.max_runs)
+        sched = api.add_schedule(inst, data.action, schedule)
+    except (ContainerError, ValueError) as exc:
+        return json_error(400, str(exc))
+    return action_result(request, f"/projects/{_slug(project)}/containers", data={"schedule": sched})
+
+
+@router.post("/{project_slug}/containers/instances/{uid}/schedules/{sid}/delete")
+async def delete_schedule(request: Request, project_slug: str, uid: str, sid: str):
+    require_admin(request)
+    project = _project(project_slug)
+    _instance(project, uid)
+    store.delete_schedule(sid)
+    return action_result(request, f"/projects/{_slug(project)}/containers", data={"uid": sid})
+
+
+@router.post("/{project_slug}/containers/instances/{uid}/start")
+@router.post("/{project_slug}/containers/instances/{uid}/stop")
+@router.post("/{project_slug}/containers/instances/{uid}/pause")
+@router.post("/{project_slug}/containers/instances/{uid}/resume")
+@router.post("/{project_slug}/containers/instances/{uid}/restart")
+async def instance_action(request: Request, project_slug: str, uid: str):
+    user = require_admin(request)
+    project = _project(project_slug)
+    inst = _instance(project, uid)
+    actor = ("user", user["uid"])
+    action = request.url.path.rsplit("/", 1)[-1]
+    if action == "restart":
+        api.request_restart(inst, actor=actor)
+    else:
+        api.set_desired_state(inst, _ACTIONS[action], actor=actor)
+    return action_result(request, f"/projects/{_slug(project)}/containers", data={"uid": uid, "action": action})
+
+
+@router.websocket("/{project_slug}/containers/instances/{uid}/exec/ws")
+async def instance_exec_ws(websocket: WebSocket, project_slug: str, uid: str):
+    await websocket.accept()
+    user = _user_from_session(websocket)
+    if not user or not is_admin(user):
+        await websocket.close(code=1008)
+        return
+    if not service_manager.owns_lock():
+        await websocket.close(code=1013)
+        return
+    project = resolve_by_slug(get_table("projects"), project_slug)
+    inst = store.get_instance(uid)
+    if not project or not inst or inst["project_uid"] != project["uid"] or not inst.get("container_id"):
+        await websocket.close(code=1011)
+        return
+
+    master, slave = pty.openpty()
+    session = (websocket.query_params.get("session") or "").strip()
+    if session and re.match(r"^[A-Za-z0-9_-]{1,64}$", session):
+        shell_cmd = (f"if command -v tmux >/dev/null 2>&1; then exec tmux new-session -A -s {session}; "
+                     "elif command -v bash >/dev/null 2>&1; then exec bash; else exec sh; fi")
+    else:
+        shell_cmd = "if command -v bash >/dev/null 2>&1; then exec bash; else exec sh; fi"
+    proc = await asyncio.create_subprocess_exec(
+        "docker", "exec", "-it", inst["container_id"], "/bin/sh", "-c", shell_cmd,
+        stdin=slave, stdout=slave, stderr=slave, start_new_session=True)
+    os.close(slave)
+    loop = asyncio.get_event_loop()
+
+    async def pump_out():
+        try:
+            while True:
+                data = await loop.run_in_executor(None, os.read, master, 4096)
+                if not data:
+                    break
+                await websocket.send_text(data.decode("utf-8", "replace"))
+        except Exception:
+            pass
+
+    def set_winsize(rows: int, cols: int) -> None:
+        try:
+            fcntl.ioctl(master, termios.TIOCSWINSZ, struct.pack("HHHH", rows, cols, 0, 0))
+        except OSError:
+            pass
+        try:
+            proc.send_signal(signal.SIGWINCH)
+        except (ProcessLookupError, OSError, ValueError):
+            pass
+
+    reader = asyncio.create_task(pump_out())
+    try:
+        while True:
+            text = await websocket.receive_text()
+            if text[:1] == "{":
+                try:
+                    control = json.loads(text)
+                except ValueError:
+                    control = None
+                if isinstance(control, dict) and control.get("type") == "resize":
+                    rows = int(control.get("rows") or 0)
+                    cols = int(control.get("cols") or 0)
+                    if rows > 0 and cols > 0:
+                        await loop.run_in_executor(None, set_winsize, rows, cols)
+                    continue
+            await loop.run_in_executor(None, os.write, master, text.encode("utf-8"))
+    except WebSocketDisconnect:
+        pass
+    except Exception:
+        pass
+    finally:
+        reader.cancel()
+        try:
+            proc.kill()
+        except ProcessLookupError:
+            pass
+        os.close(master)

devplacepy/routers/containers_admin.py

@@ -0,0 +1,105 @@
+# retoor <retoor@molodetz.nl>
+
+import logging
+
+from fastapi import APIRouter, Request
+from fastapi.responses import HTMLResponse, JSONResponse
+
+from devplacepy.database import db, get_table
+from devplacepy.seo import base_seo_context, site_url, website_schema
+from devplacepy.services.containers import api, store
+from devplacepy.templating import templates
+from devplacepy.utils import not_found, require_admin
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+
+def _project_index(project_uids: set) -> dict:
+    if not project_uids or "projects" not in db.tables:
+        return {}
+    table = get_table("projects")
+    rows = table.find(table.table.columns.uid.in_(list(project_uids)))
+    return {row["uid"]: row for row in rows}
+
+
+def _decorate(instances: list) -> list:
+    index = _project_index({inst["project_uid"] for inst in instances})
+    decorated = []
+    for inst in instances:
+        project = index.get(inst["project_uid"], {})
+        row = dict(inst)
+        row["project_title"] = project.get("title", "")
+        row["project_slug"] = project.get("slug") or project.get("uid") or ""
+        decorated.append(row)
+    decorated.sort(key=lambda r: r.get("created_at", ""), reverse=True)
+    return decorated
+
+
+@router.get("", response_class=HTMLResponse)
+async def containers_index(request: Request):
+    admin = require_admin(request)
+    instances = _decorate(store.all_instances())
+    base = site_url(request)
+    seo_ctx = base_seo_context(
+        request,
+        title="Containers - Admin",
+        description="Manage container instances across all projects.",
+        robots="noindex,nofollow",
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {"name": "Admin", "url": "/admin"},
+            {"name": "Containers", "url": "/admin/containers"},
+        ],
+        schemas=[website_schema(base)],
+    )
+    return templates.TemplateResponse(request, "containers_admin.html", {
+        **seo_ctx,
+        "request": request,
+        "user": admin,
+        "instances": instances,
+        "admin_section": "containers",
+    })
+
+
+@router.get("/data")
+async def containers_index_data(request: Request):
+    require_admin(request)
+    return JSONResponse({"instances": _decorate(store.all_instances())})
+
+
+@router.get("/{uid}", response_class=HTMLResponse)
+async def container_instance_page(request: Request, uid: str):
+    admin = require_admin(request)
+    inst = store.get_instance(uid)
+    if not inst:
+        raise not_found("Instance not found")
+    project = get_table("projects").find_one(uid=inst["project_uid"]) or {}
+    project_slug = project.get("slug") or project.get("uid") or ""
+    base = site_url(request)
+    seo_ctx = base_seo_context(
+        request,
+        title=f"{inst['name']} - Containers",
+        description=f"Container instance {inst['name']}.",
+        robots="noindex,nofollow",
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {"name": "Admin", "url": "/admin"},
+            {"name": "Containers", "url": "/admin/containers"},
+            {"name": inst["name"], "url": f"/admin/containers/{inst['uid']}"},
+        ],
+        schemas=[website_schema(base)],
+    )
+    return templates.TemplateResponse(request, "containers_instance.html", {
+        **seo_ctx,
+        "request": request,
+        "user": admin,
+        "instance": inst,
+        "project": project,
+        "project_slug": project_slug,
+        "events": store.list_events(inst["uid"]),
+        "schedules": store.list_schedules(inst["uid"]),
+        "stats": api.instance_stats(inst["uid"]),
+        "runtime": api.instance_runtime(inst),
+        "admin_section": "containers",
+    })

devplacepy/routers/devii.py

@@ -0,0 +1,201 @@
+# retoor <retoor@molodetz.nl>
+
+import logging
+
+import httpx
+import uuid_utils
+from fastapi import APIRouter, Request, WebSocket, WebSocketDisconnect
+from fastapi.responses import JSONResponse, RedirectResponse
+
+from devplacepy.constants import DEVII_GUEST_COOKIE
+from devplacepy.database import get_int_setting
+from devplacepy.seo import site_url
+from devplacepy.services.manager import service_manager
+from devplacepy.templating import templates
+from devplacepy.utils import _user_from_api_key, _user_from_session, get_current_user, is_admin
+
+logger = logging.getLogger("devii.router")
+
+router = APIRouter()
+
+GUEST_COOKIE = DEVII_GUEST_COOKIE
+GUEST_COOKIE_MAX_AGE = 31536000
+
+
+def _service():
+    return service_manager.get_service("devii")
+
+
+def _bearer_key(headers) -> str:
+    key = headers.get("x-api-key", "")
+    if key:
+        return key.strip()
+    scheme, _, credentials = headers.get("authorization", "").partition(" ")
+    if scheme.lower() == "bearer":
+        return credentials.strip()
+    return ""
+
+
+def _resolve_ws_owner(websocket: WebSocket):
+    user = _user_from_session(websocket)
+    if not user:
+        key = _bearer_key(websocket.headers)
+        if key:
+            user = _user_from_api_key(key)
+    if user:
+        return "user", user["uid"], user.get("username", ""), user.get("api_key", ""), is_admin(user)
+    guest_id = websocket.cookies.get(GUEST_COOKIE) or uuid_utils.uuid7().hex
+    return "guest", guest_id, "guest", "", False
+
+
+def _owner_from_request(request: Request):
+    user = get_current_user(request)
+    if user:
+        return "user", user["uid"], user.get("username", "")
+    guest_id = request.cookies.get(GUEST_COOKIE) or ""
+    return "guest", guest_id, "guest"
+
+
+@router.get("/")
+async def devii_page(request: Request):
+    user = get_current_user(request)
+    response = templates.TemplateResponse(
+        request, "devii.html", {"request": request, "user": user, "page_title": "Devii", "meta_robots": "noindex,nofollow"}
+    )
+    if not user and not request.cookies.get(GUEST_COOKIE):
+        response.set_cookie(GUEST_COOKIE, uuid_utils.uuid7().hex, httponly=True, samesite="lax", max_age=GUEST_COOKIE_MAX_AGE)
+    return response
+
+
+@router.get("/session")
+async def devii_session(request: Request):
+    user = get_current_user(request)
+    svc = _service()
+    payload = {
+        "loggedIn": bool(user),
+        "username": user.get("username") if user else "guest",
+        "enabled": bool(svc and svc.is_enabled()),
+        "guestsEnabled": bool(svc and svc.guests_enabled()),
+        "baseUrl": site_url(request),
+    }
+    response = JSONResponse(payload)
+    if not user and not request.cookies.get(GUEST_COOKIE):
+        response.set_cookie(GUEST_COOKIE, uuid_utils.uuid7().hex, httponly=True, samesite="lax", max_age=GUEST_COOKIE_MAX_AGE)
+    return response
+
+
+def _safe_next(value: str) -> str:
+    if value.startswith("/") and not value.startswith("//"):
+        return value
+    return "/"
+
+
+@router.get("/adopt")
+async def devii_adopt(request: Request):
+    # The terminal's agent logged in; adopt the real session it minted into the browser so both
+    # share one session, then return to where the user was.
+    target = _safe_next(request.query_params.get("next", "/"))
+    response = RedirectResponse(target, status_code=303)
+    svc = _service()
+    if svc is None:
+        return response
+    owner_kind, owner_id, _ = _owner_from_request(request)
+    session = svc.hub().find(owner_kind, owner_id) if owner_id else None
+    token = session.take_pending_session() if session else None
+    if token:
+        max_age = max(1, get_int_setting("session_remember_days", 30)) * 86400
+        response.set_cookie("session", token, max_age=max_age, httponly=True, samesite="lax")
+    return response
+
+
+@router.get("/usage")
+async def devii_usage(request: Request):
+    svc = _service()
+    owner_kind, owner_id, _ = _owner_from_request(request)
+    if svc is None:
+        return JSONResponse({"used_pct": 0.0, "turns_today": 0})
+    spent = svc.spent_24h(owner_kind, owner_id) if owner_id else 0.0
+    limit = svc.daily_limit_for(owner_kind)
+    turns = svc.hub().ledger.turns_24h(owner_kind, owner_id) if owner_id else 0
+    used_pct = round(min(100.0, spent / limit * 100), 1) if limit > 0 else 0.0
+    payload = {
+        "used_pct": used_pct,
+        "turns_today": turns,
+        "owner_kind": owner_kind,
+    }
+    if is_admin(get_current_user(request)):
+        payload["spent_24h"] = round(spent, 6)
+        payload["limit"] = limit
+    return JSONResponse(payload)
+
+
+@router.post("/clippy/ai/chat")
+async def clippy_proxy(request: Request):
+    svc = _service()
+    if svc is None or not svc.is_enabled():
+        return JSONResponse({"error": "Devii is unavailable"}, status_code=503)
+    cfg = svc.effective_config()
+    body = await request.body()
+    headers = {"Content-Type": "application/json"}
+    if cfg.get("devii_ai_key"):
+        headers["Authorization"] = f"Bearer {cfg['devii_ai_key']}"
+    async with httpx.AsyncClient(timeout=45.0) as client:
+        upstream = await client.post(cfg["devii_ai_url"], content=body, headers=headers)
+    return JSONResponse(
+        content=upstream.json() if upstream.headers.get("content-type", "").startswith("application/json") else {"raw": upstream.text},
+        status_code=upstream.status_code,
+    )
+
+
+@router.websocket("/ws")
+async def devii_ws(websocket: WebSocket):
+    await websocket.accept()
+    svc = _service()
+    if svc is None or not svc.is_enabled():
+        await websocket.close(code=1013)
+        return
+    if not service_manager.owns_lock():
+        await websocket.close(code=4013)
+        return
+    owner_kind, owner_id, username, api_key, owner_is_admin = _resolve_ws_owner(websocket)
+    if owner_kind == "guest" and not svc.guests_enabled():
+        await websocket.send_json({"type": "error", "text": "Guest access to Devii is disabled."})
+        await websocket.close(code=1008)
+        return
+
+    session = svc.hub().get_or_create(
+        owner_kind, owner_id, username, api_key, svc.instance_base_url(), is_admin=owner_is_admin
+    )
+    session.attach(websocket)
+    try:
+        await session.send_bootstrap(websocket)
+        while True:
+            data = await websocket.receive_json()
+            kind = data.get("type")
+            if kind == "input":
+                text = str(data.get("text", "")).strip()
+                if not text:
+                    continue
+                limit = svc.daily_limit_for(owner_kind, owner_is_admin)
+                spent = svc.spent_24h(owner_kind, owner_id)
+                if limit > 0 and spent >= limit:
+                    await websocket.send_json({
+                        "type": "error",
+                        "text": "Daily AI quota reached (100%). Try again later.",
+                    })
+                    continue
+                session.spawn_turn(text)
+            elif kind == "reset":
+                await session.reset()
+            elif kind == "clear":
+                await session.reset_conversation()
+            elif kind == "visibility":
+                session.set_visibility(websocket, bool(data.get("visible", True)), bool(data.get("focused", False)))
+            elif kind in ("avatar_result", "client_result"):
+                session.resolve_query(str(data.get("id", "")), data.get("result"))
+    except WebSocketDisconnect:
+        pass
+    except Exception:  # noqa: BLE001 - never let the socket loop crash the worker
+        logger.exception("Devii websocket loop failed for %s/%s", owner_kind, owner_id)
+    finally:
+        session.detach(websocket)

devplacepy/routers/docs.py

@@ -0,0 +1,227 @@
+import logging
+from fastapi import APIRouter, Request
+from fastapi.responses import HTMLResponse, RedirectResponse, Response
+from devplacepy.templating import templates
+from devplacepy.utils import get_current_user, not_found, is_admin as user_is_admin
+from devplacepy.seo import base_seo_context, site_url, website_schema
+from devplacepy.docs_api import api_doc_pages, render_group, build_services_group
+from devplacepy.services.manager import service_manager
+from devplacepy import docs_search, docs_export, docs_live, docs_prose
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+SECTION_GENERAL = "General"
+SECTION_COMPONENTS = "Components"
+SECTION_STYLES = "Styles"
+SECTION_API = "API"
+SECTION_ADMIN = "Administration"
+SECTION_DEVII = "Devii internals"
+SECTION_SERVICES = "Services"
+SECTION_ARCH = "Architecture"
+SECTION_TESTING = "Testing"
+SECTION_PROD = "Production"
+
+DOCS_PAGES = [
+    # General - how to use the site and Devii (everyone)
+    {"slug": "index", "title": "Overview", "kind": "prose", "section": SECTION_GENERAL},
+    {"slug": "devii", "title": "Devii Assistant", "kind": "prose", "section": SECTION_GENERAL},
+    {"slug": "dashboard", "title": "Your dashboard", "kind": "live", "section": SECTION_GENERAL},
+    # Components - custom HTML web components with live examples (everyone)
+    {"slug": "components", "title": "Components overview", "kind": "prose", "section": SECTION_COMPONENTS},
+    {"slug": "component-dp-avatar", "title": "dp-avatar", "kind": "prose", "section": SECTION_COMPONENTS},
+    {"slug": "component-dp-code", "title": "dp-code", "kind": "prose", "section": SECTION_COMPONENTS},
+    {"slug": "component-dp-content", "title": "dp-content", "kind": "prose", "section": SECTION_COMPONENTS},
+    {"slug": "component-dp-upload", "title": "dp-upload", "kind": "prose", "section": SECTION_COMPONENTS},
+    {"slug": "component-dp-toast", "title": "dp-toast", "kind": "prose", "section": SECTION_COMPONENTS},
+    {"slug": "component-dp-dialog", "title": "dp-dialog", "kind": "prose", "section": SECTION_COMPONENTS},
+    {"slug": "component-dp-context-menu", "title": "dp-context-menu", "kind": "prose", "section": SECTION_COMPONENTS},
+    {"slug": "component-devii-terminal", "title": "devii-terminal", "kind": "prose", "section": SECTION_COMPONENTS},
+    {"slug": "component-devii-avatar", "title": "devii-avatar", "kind": "prose", "section": SECTION_COMPONENTS},
+    {"slug": "component-emoji-picker", "title": "emoji-picker", "kind": "prose", "section": SECTION_COMPONENTS},
+    # Styles - the design system: colors, layout, responsiveness, and hard structural rules (everyone)
+    {"slug": "styles", "title": "Design system overview", "kind": "prose", "section": SECTION_STYLES},
+    {"slug": "styles-colors", "title": "Colors", "kind": "prose", "section": SECTION_STYLES},
+    {"slug": "styles-layout", "title": "Layout", "kind": "prose", "section": SECTION_STYLES},
+    {"slug": "styles-responsiveness", "title": "Responsiveness", "kind": "prose", "section": SECTION_STYLES},
+    {"slug": "styles-consistency", "title": "Consistency rules", "kind": "prose", "section": SECTION_STYLES},
+    # API - developer reference (everyone); admin-only groups are routed to Administration below
+    {"slug": "authentication", "title": "Authentication", "kind": "prose", "section": SECTION_API},
+    *[{**page, "section": (SECTION_ADMIN if page.get("admin") else SECTION_API)} for page in api_doc_pages()],
+    # Devii internals - technical reference for the Devii assistant (admins only)
+    {"slug": "devii-internals", "title": "Devii internals", "kind": "prose", "admin": True, "section": SECTION_DEVII},
+    {"slug": "devii-architecture", "title": "Architecture and sessions", "kind": "prose", "admin": True, "section": SECTION_DEVII},
+    {"slug": "devii-tools", "title": "Tools and scopes", "kind": "prose", "admin": True, "section": SECTION_DEVII},
+    {"slug": "devii-data", "title": "Data and persistence", "kind": "prose", "admin": True, "section": SECTION_DEVII},
+    {"slug": "devii-security", "title": "Security and limits", "kind": "prose", "admin": True, "section": SECTION_DEVII},
+    {"slug": "devii-config", "title": "Configuration and CLI", "kind": "prose", "admin": True, "section": SECTION_DEVII},
+    # Services - the background service framework and every service in detail (admins only)
+    {"slug": "services-overview", "title": "Services overview", "kind": "prose", "admin": True, "section": SECTION_SERVICES},
+    {"slug": "services-framework", "title": "Service framework", "kind": "prose", "admin": True, "section": SECTION_SERVICES},
+    {"slug": "services-data", "title": "Reading service data", "kind": "prose", "admin": True, "section": SECTION_SERVICES},
+    {"slug": "services-gateway", "title": "GatewayService (AI)", "kind": "prose", "admin": True, "section": SECTION_SERVICES},
+    {"slug": "services-devii", "title": "DeviiService", "kind": "prose", "admin": True, "section": SECTION_SERVICES},
+    {"slug": "services-news", "title": "NewsService", "kind": "prose", "admin": True, "section": SECTION_SERVICES},
+    {"slug": "services-bots", "title": "BotsService", "kind": "prose", "admin": True, "section": SECTION_SERVICES},
+    {"slug": "services-zip", "title": "ZipService", "kind": "prose", "admin": True, "section": SECTION_SERVICES},
+    {"slug": "services-containers", "title": "Container Manager", "kind": "prose", "admin": True, "section": SECTION_SERVICES},
+    # Architecture - platform design, structure, and development process (admins only)
+    {"slug": "architecture", "title": "Architecture overview", "kind": "prose", "admin": True, "section": SECTION_ARCH},
+    {"slug": "architecture-backend", "title": "Backend and request pipeline", "kind": "prose", "admin": True, "section": SECTION_ARCH},
+    {"slug": "architecture-frontend", "title": "Frontend and components", "kind": "prose", "admin": True, "section": SECTION_ARCH},
+    {"slug": "architecture-styling", "title": "Styling and templates", "kind": "prose", "admin": True, "section": SECTION_ARCH},
+    {"slug": "architecture-conventions", "title": "Conventions and rules", "kind": "prose", "admin": True, "section": SECTION_ARCH},
+    {"slug": "architecture-workflow", "title": "Development workflow", "kind": "prose", "admin": True, "section": SECTION_ARCH},
+    {"slug": "architecture-jobs", "title": "Async job services", "kind": "prose", "admin": True, "section": SECTION_ARCH},
+    # Testing - test framework, load testing, and make targets (admins only)
+    {"slug": "testing", "title": "Testing overview", "kind": "prose", "admin": True, "section": SECTION_TESTING},
+    {"slug": "testing-framework", "title": "Test framework and rules", "kind": "prose", "admin": True, "section": SECTION_TESTING},
+    {"slug": "testing-locust", "title": "Load testing with Locust", "kind": "prose", "admin": True, "section": SECTION_TESTING},
+    {"slug": "testing-make", "title": "Running tests via make", "kind": "prose", "admin": True, "section": SECTION_TESTING},
+    {"slug": "testing-cicd", "title": "Coverage and CI/CD", "kind": "prose", "admin": True, "section": SECTION_TESTING},
+    # Production - deployment and operations reference (admins only)
+    {"slug": "production", "title": "Production overview", "kind": "prose", "admin": True, "section": SECTION_PROD},
+    {"slug": "production-deploy", "title": "Deploy and update", "kind": "prose", "admin": True, "section": SECTION_PROD},
+    {"slug": "production-nginx", "title": "nginx and networking", "kind": "prose", "admin": True, "section": SECTION_PROD},
+    {"slug": "production-concurrency", "title": "Multi-worker and concurrency", "kind": "prose", "admin": True, "section": SECTION_PROD},
+]
+_PAGES_BY_SLUG = {page["slug"]: page for page in DOCS_PAGES}
+
+
+@router.get("/docs")
+async def docs_root(request: Request):
+    return RedirectResponse(url="/docs/index.html", status_code=302)
+
+
+def _export_context(request: Request):
+    user = get_current_user(request)
+    is_admin = user_is_admin(user)
+    return (
+        is_admin,
+        site_url(request),
+        user.get("username") if user else "",
+        user.get("api_key") if user else "",
+    )
+
+
+@router.get("/docs/download.md")
+async def docs_download_md(request: Request):
+    is_admin, base, username, api_key = _export_context(request)
+    markdown = docs_export.build_markdown(is_admin, base, username, api_key)
+    return Response(
+        content=markdown,
+        media_type="text/markdown; charset=utf-8",
+        headers={"Content-Disposition": 'attachment; filename="devplace-docs.md"'},
+    )
+
+
+@router.get("/docs/download.html")
+async def docs_download_html(request: Request):
+    is_admin, base, username, api_key = _export_context(request)
+    markdown = docs_export.build_markdown(is_admin, base, username, api_key)
+    return HTMLResponse(
+        content=docs_export.build_html(markdown),
+        headers={"Content-Disposition": 'attachment; filename="devplace-docs.html"'},
+    )
+
+
+@router.get("/docs/{slug}.html", response_class=HTMLResponse)
+async def docs_page(request: Request, slug: str):
+    user = get_current_user(request)
+    is_admin = user_is_admin(user)
+    base = site_url(request)
+    visible_pages = [p for p in DOCS_PAGES if not p.get("admin") or is_admin]
+
+    if slug == "search":
+        query = request.query_params.get("q", "")
+        results = docs_search.search(query, user=user, is_admin=is_admin)
+        seo_ctx = base_seo_context(
+            request,
+            title="Search - Documentation",
+            description="Search the DevPlace developer documentation.",
+            robots="noindex",
+            breadcrumbs=[
+                {"name": "Home", "url": "/feed"},
+                {"name": "Docs", "url": "/docs/index.html"},
+                {"name": "Search", "url": "/docs/search.html"},
+            ],
+            schemas=[website_schema(base)],
+        )
+        return templates.TemplateResponse(request, "docs_base.html", {
+            **seo_ctx,
+            "request": request,
+            "user": user,
+            "pages": visible_pages,
+            "current": "search",
+            "kind": "search",
+            "page_title_doc": "Search",
+            "search_query": query,
+            "search_results": results,
+            "base": base,
+        })
+
+    page = _PAGES_BY_SLUG.get(slug)
+    if not page:
+        raise not_found("Documentation page not found")
+    if page.get("admin") and not is_admin:
+        raise not_found("Documentation page not found")
+    if page["kind"] == "live":
+        seo_ctx = base_seo_context(
+            request,
+            title=f"{page['title']} - Documentation",
+            description="Your live DevPlace stats, activity, and site data.",
+            robots="noindex",
+            breadcrumbs=[
+                {"name": "Home", "url": "/feed"},
+                {"name": "Docs", "url": "/docs/index.html"},
+                {"name": page["title"], "url": f"/docs/{slug}.html"},
+            ],
+            schemas=[website_schema(base)],
+        )
+        return templates.TemplateResponse(request, "docs_base.html", {
+            **seo_ctx,
+            "request": request,
+            "user": user,
+            "pages": visible_pages,
+            "current": slug,
+            "kind": "live",
+            "facts": docs_live.build_live_facts(user),
+            "page_title_doc": page["title"],
+            "base": base,
+        })
+    prose_html = None
+    group = None
+    if page["kind"] == "prose":
+        prose_html = docs_prose.render_prose(slug, {
+            "base": base,
+            "user": user,
+            "username": user.get("username") if user else "",
+            "api_key": user.get("api_key") if user else "",
+        })
+    elif page.get("dynamic"):
+        group = build_services_group(service_manager.describe_all(), base)
+    else:
+        group = render_group(slug, base, user.get("username") if user else "", user.get("api_key") if user else "")
+    seo_ctx = base_seo_context(
+        request,
+        title=f"{page['title']} - Documentation",
+        description="DevPlace developer documentation.",
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {"name": "Docs", "url": "/docs/index.html"},
+            {"name": page["title"], "url": f"/docs/{slug}.html"},
+        ],
+        schemas=[website_schema(base)],
+    )
+    return templates.TemplateResponse(request, "docs_base.html", {
+        **seo_ctx,
+        "request": request,
+        "user": user,
+        "pages": visible_pages,
+        "current": slug,
+        "kind": page["kind"],
+        "group": group,
+        "prose_html": prose_html,
+        "page_title_doc": page["title"],
+        "base": base,
+    })

devplacepy/routers/feed.py

@@ -1,25 +1,22 @@
 import logging
-from datetime import datetime
 from fastapi import APIRouter, Request
 from fastapi.responses import HTMLResponse
-from devplacepy.database import get_table, get_daily_topic, get_users_by_uids, get_comment_counts_by_post_uids
+from devplacepy.database import get_table, get_daily_topic, get_users_by_uids, get_comment_counts_by_post_uids, get_recent_comments_by_post_uids, get_site_stats, get_top_authors, get_reactions_by_targets, get_user_bookmarks, get_polls_by_post_uids, paginate
 from devplacepy.attachments import get_attachments_batch
+from devplacepy.content import enrich_items
 from devplacepy.templating import templates
-from devplacepy.utils import get_current_user, time_ago
-from devplacepy.seo import base_seo_context, site_url, website_schema, breadcrumb_schema, combine
+from devplacepy.utils import get_current_user
+from devplacepy.seo import list_page_seo, next_page_url
+from devplacepy.responses import respond
+from devplacepy.schemas import FeedOut
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
-PAGE_SIZE = 25
-
 
 def get_feed_posts(user, tab: str = "all", topic: str = None, before: str = None):
     posts_table = get_table("posts")
-
-    filters = {}
-    if topic:
-        filters["topic"] = topic
+    order = ["-stars", "-created_at"] if tab == "trending" else ["-created_at"]
 
     if tab == "following":
         if not user:
@@ -28,40 +25,18 @@ def get_feed_posts(user, tab: str = "all", topic: str = None, before: str = None
         following = [f["following_uid"] for f in follows.find(follower_uid=user["uid"])]
         if not following:
             return [], None
-        posts = list(posts_table.find(posts_table.table.columns.user_uid.in_(following), order_by=["-created_at"], _limit=PAGE_SIZE))
+        posts, next_cursor = paginate(posts_table, posts_table.table.columns.user_uid.in_(following), before=before, order=order)
     else:
-        order = ["-created_at"]
-        if tab == "trending":
-            order = ["-stars", "-created_at"]
-        if before:
-            posts = list(posts_table.find(**filters, order_by=order, _limit=PAGE_SIZE + 1))
-            posts = [p for p in posts if p["created_at"] < before][:PAGE_SIZE]
-        else:
-            posts = list(posts_table.find(**filters, order_by=order, _limit=PAGE_SIZE + 1))
-
-    has_more = len(posts) > PAGE_SIZE
-    posts = posts[:PAGE_SIZE]
-
-    next_cursor = None
-    if has_more and posts:
-        next_cursor = posts[-1]["created_at"]
+        filters = {"topic": topic} if topic else {}
+        posts, next_cursor = paginate(posts_table, before=before, order=order, **filters)
 
     if not posts:
         return [], next_cursor
 
-    uids = [p["user_uid"] for p in posts]
-    post_uids = [p["uid"] for p in posts]
-    authors = get_users_by_uids(uids)
-    counts = get_comment_counts_by_post_uids(post_uids)
+    authors = get_users_by_uids([p["user_uid"] for p in posts])
+    counts = get_comment_counts_by_post_uids([p["uid"] for p in posts])
 
-    result = []
-    for post in posts:
-        result.append({
-            "post": post,
-            "author": authors.get(post["user_uid"]),
-            "time_ago": time_ago(post["created_at"]),
-            "comment_count": counts.get(post["uid"], 0),
-        })
+    result = enrich_items(posts, "post", authors, {"comment_count": counts}, user=user)
     return result, next_cursor
 
 
@@ -69,41 +44,43 @@ def get_feed_posts(user, tab: str = "all", topic: str = None, before: str = None
 async def feed_page(request: Request, tab: str = "all", topic: str = None, before: str = None):
     user = get_current_user(request)
     posts, next_cursor = get_feed_posts(user, tab, topic, before)
-    users_table = get_table("users")
-    total_members = len(list(users_table.all()))
-    posts_table = get_table("posts")
-    today_start = datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0).isoformat()
-    posts_today = len(list(posts_table.find(created_at={">=": today_start})))
-    total_projects = len(list(get_table("projects").all()))
-    total_gists = len(list(get_table("gists").all()))
-    top_authors = list(users_table.find(stars={">": 0}, order_by=["-stars"], _limit=5))
+    stats = get_site_stats()
+    top_authors = get_top_authors(5)
     daily_topic = get_daily_topic()
 
     post_uids_list = [item["post"]["uid"] for item in posts]
     attachments_map = get_attachments_batch("post", post_uids_list)
+    recent_comments = get_recent_comments_by_post_uids(post_uids_list, 3, user)
+    reactions_map = get_reactions_by_targets("post", post_uids_list, user)
+    bookmark_set = get_user_bookmarks(user["uid"], "post", post_uids_list) if user else set()
+    polls_map = get_polls_by_post_uids(post_uids_list, user)
     for item in posts:
-        item["attachments"] = attachments_map.get(item["post"]["uid"], [])
+        uid = item["post"]["uid"]
+        item["attachments"] = attachments_map.get(uid, [])
+        item["recent_comments"] = recent_comments.get(uid, [])
+        item["reactions"] = reactions_map.get(uid, {"counts": {}, "mine": []})
+        item["bookmarked"] = uid in bookmark_set
+        item["poll"] = polls_map.get(uid)
 
-    base = site_url(request)
-    seo_ctx = base_seo_context(
+    seo_ctx = list_page_seo(
         request,
         title="Feed",
         description="Discover the latest developer discussions, projects, and community activity on DevPlace.",
         breadcrumbs=[{"name": "Home", "url": "/feed"}, {"name": "Feed", "url": "/feed"}],
-        schemas=[website_schema(base)],
+        next_url=next_page_url(request, next_cursor),
     )
-    return templates.TemplateResponse("feed.html", {
+    return respond(request, "feed.html", {
         **seo_ctx,
         "request": request,
         "user": user,
         "posts": posts,
         "current_tab": tab,
         "current_topic": topic,
-        "total_members": total_members,
-        "posts_today": posts_today,
-        "total_projects": total_projects,
-        "total_gists": total_gists,
+        "total_members": stats["total_members"],
+        "posts_today": stats["posts_today"],
+        "total_projects": stats["total_projects"],
+        "total_gists": stats["total_gists"],
         "top_authors": top_authors,
         "daily_topic": daily_topic,
         "next_cursor": next_cursor,
-    })
+    }, model=FeedOut)

devplacepy/routers/follow.py

@@ -1,9 +1,10 @@
 import logging
-from datetime import datetime
+from datetime import datetime, timezone
 from fastapi import APIRouter, Request
 from fastapi.responses import RedirectResponse
 from devplacepy.database import get_table
-from devplacepy.utils import generate_uid, require_user
+from devplacepy.utils import generate_uid, require_user, create_notification, award_rewards, XP_FOLLOW
+from devplacepy.responses import action_result
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
@@ -15,33 +16,25 @@ async def follow_user(request: Request, username: str):
     users = get_table("users")
     target = users.find_one(username=username)
     if not target or target["uid"] == user["uid"]:
-        return RedirectResponse(url=f"/profile/{username}", status_code=302)
+        return action_result(request, f"/profile/{username}")
 
     follows = get_table("follows")
     existing = follows.find_one(follower_uid=user["uid"], following_uid=target["uid"])
     if existing:
-        return RedirectResponse(url=f"/profile/{username}", status_code=302)
+        return action_result(request, f"/profile/{username}")
 
     follows.insert({
         "uid": generate_uid(),
         "follower_uid": user["uid"],
         "following_uid": target["uid"],
-        "created_at": datetime.utcnow().isoformat(),
+        "created_at": datetime.now(timezone.utc).isoformat(),
     })
 
-    notifications = get_table("notifications")
-    notifications.insert({
-        "uid": generate_uid(),
-        "user_uid": target["uid"],
-        "type": "follow",
-        "message": f"{user['username']} started following you",
-        "related_uid": user["uid"],
-        "read": False,
-        "created_at": datetime.utcnow().isoformat(),
-    })
+    create_notification(target["uid"], "follow", f"{user['username']} started following you", user["uid"], f"/profile/{user['username']}")
+    award_rewards(target["uid"], XP_FOLLOW)
 
     logger.info(f"{user['username']} followed {username}")
-    return RedirectResponse(url=f"/profile/{username}", status_code=302)
+    return action_result(request, f"/profile/{username}")
 
 
 @router.post("/unfollow/{username}")
@@ -50,7 +43,7 @@ async def unfollow_user(request: Request, username: str):
     users = get_table("users")
     target = users.find_one(username=username)
     if not target:
-        return RedirectResponse(url=f"/profile/{username}", status_code=302)
+        return action_result(request, f"/profile/{username}")
 
     follows = get_table("follows")
     existing = follows.find_one(follower_uid=user["uid"], following_uid=target["uid"])
@@ -58,4 +51,4 @@ async def unfollow_user(request: Request, username: str):
         follows.delete(id=existing["id"])
         logger.info(f"{user['username']} unfollowed {username}")
 
-    return RedirectResponse(url=f"/profile/{username}", status_code=302)
+    return action_result(request, f"/profile/{username}")

devplacepy/routers/forks.py

@@ -0,0 +1,39 @@
+# retoor <retoor@molodetz.nl>
+
+import logging
+
+from fastapi import APIRouter, Request
+from fastapi.responses import JSONResponse
+
+from devplacepy.services.jobs import queue
+from devplacepy.schemas import ForkJobOut
+from devplacepy.utils import not_found
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+
+def _status_payload(job: dict) -> dict:
+    result = job.get("result", {})
+    done = job.get("status") == queue.DONE
+    return {
+        "uid": job.get("uid", ""),
+        "kind": job.get("kind", ""),
+        "status": job.get("status", ""),
+        "preferred_name": job.get("preferred_name"),
+        "project_uid": result.get("project_uid") if done else None,
+        "project_url": result.get("project_url") if done else None,
+        "source_project_uid": result.get("source_project_uid") if done else None,
+        "error": job.get("error") or None,
+        "item_count": int(job.get("item_count") or 0),
+        "created_at": job.get("created_at"),
+        "completed_at": job.get("completed_at") or None,
+    }
+
+
+@router.get("/{uid}")
+async def fork_status(request: Request, uid: str):
+    job = queue.get_job(uid)
+    if not job or job.get("kind") != "fork":
+        raise not_found("Job not found")
+    return JSONResponse(ForkJobOut.model_validate(_status_payload(job)).model_dump(mode="json"))

devplacepy/routers/gists.py

@@ -1,11 +1,15 @@
 import logging
-from datetime import datetime
-from fastapi import APIRouter, Request, HTTPException
+from typing import Annotated
+from fastapi import APIRouter, Request, Form
+from devplacepy.models import GistForm, GistEditForm
 from fastapi.responses import HTMLResponse, RedirectResponse
-from devplacepy.database import get_table, load_comments, get_vote_counts, get_attachments, get_attachments_by_type, delete_attachments, resolve_by_slug
+from devplacepy.database import get_table, get_users_by_uids, get_gist_languages, paginate
+from devplacepy.content import load_detail, edit_content_item, delete_content_item, enrich_items, create_content_item, detail_context, canonical_redirect, first_image_url
 from devplacepy.templating import templates
-from devplacepy.utils import generate_uid, get_current_user, require_user, time_ago, make_combined_slug, create_mention_notifications
-from devplacepy.seo import base_seo_context, site_url, website_schema, breadcrumb_schema, combine
+from devplacepy.utils import get_current_user, require_user, not_found, XP_GIST
+from devplacepy.seo import base_seo_context, site_url, website_schema, software_source_code_schema, list_page_seo, next_page_url
+from devplacepy.responses import respond, action_result
+from devplacepy.schemas import GistsOut, GistDetailOut
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
@@ -21,7 +25,7 @@ LANGUAGES = [
 ]
 
 
-def get_gists_list(user_uid=None, language=None):
+def get_gists_list(user_uid=None, language=None, before=None, viewer=None):
     gists_table = get_table("gists")
     filters = {}
     if user_uid:
@@ -29,111 +33,110 @@ def get_gists_list(user_uid=None, language=None):
     if language:
         filters["language"] = language
 
-    all_gists = list(gists_table.find(**filters, order_by=["-created_at"]))
+    total = gists_table.count(**filters)
+    gists, next_cursor = paginate(gists_table, before=before, **filters)
 
-    if not all_gists:
-        return []
+    if not gists:
+        return [], next_cursor, total
 
-    from devplacepy.database import get_users_by_uids
-    uids = [g["user_uid"] for g in all_gists]
-    users_map = get_users_by_uids(uids)
+    users_map = get_users_by_uids([g["user_uid"] for g in gists])
+    return enrich_items(gists, "gist", users_map, user=viewer), next_cursor, total
 
-    result = []
-    for g in all_gists:
-        author = users_map.get(g["user_uid"])
-        result.append({
-            "gist": g,
-            "author": author,
-            "time_ago": time_ago(g["created_at"]),
-        })
-    return result
+
+@router.get("", response_class=HTMLResponse)
+async def gists_page(request: Request, language: str = None, user_uid: str = None, before: str = None):
+    user = get_current_user(request)
+    gists_data, next_cursor, total_count = get_gists_list(user_uid, language, before, viewer=user)
+    seo_ctx = list_page_seo(
+        request,
+        title="Gists",
+        description=f"Browse {total_count} code snippets on DevPlace.",
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {"name": "Gists", "url": "/gists"},
+        ],
+        next_url=next_page_url(request, next_cursor),
+    )
+    return respond(request, "gists.html", {
+        **seo_ctx,
+        "request": request,
+        "user": user,
+        "gists": gists_data,
+        "total_count": total_count,
+        "next_cursor": next_cursor,
+        "current_language": language,
+        "languages": LANGUAGES,
+        "gist_language_codes": get_gist_languages(),
+    }, model=GistsOut)
 
 
 @router.get("/{gist_slug}", response_class=HTMLResponse)
 async def gist_detail(request: Request, gist_slug: str):
     user = get_current_user(request)
-    gists = get_table("gists")
-    gist = resolve_by_slug(gists, gist_slug)
-    if not gist:
-        raise HTTPException(status_code=404, detail="Gist not found")
-
-    from devplacepy.database import get_users_by_uids
-    users_map = get_users_by_uids([gist["user_uid"]])
-    author = users_map.get(gist["user_uid"])
-
-    is_owner = user and user["uid"] == gist["user_uid"]
-
-    ups, downs = get_vote_counts([gist["uid"]])
-    star_count = ups.get(gist["uid"], 0) - downs.get(gist["uid"], 0)
-
-    comments = load_comments("gist", gist["uid"])
-
-    gist_attachments = get_attachments("gist", gist["uid"])
+    detail = load_detail("gists", "gist", gist_slug, user)
+    if not detail:
+        raise not_found("Gist not found")
+    gist = detail["item"]
+    redirect = canonical_redirect("gists", gist, gist_slug)
+    if redirect:
+        return redirect
 
     base = site_url(request)
     seo_ctx = base_seo_context(
         request,
         title=gist.get("title", "Gist"),
         description=gist.get("description", "")[:160],
+        og_type="article",
+        og_image=first_image_url(gist, detail["attachments"]),
         breadcrumbs=[
             {"name": "Home", "url": "/feed"},
             {"name": "Gists", "url": "/gists"},
             {"name": gist.get("title", "Gist"), "url": f"/gists/{gist['slug'] or gist['uid']}"},
         ],
-        schemas=[website_schema(base)],
+        schemas=[website_schema(base), software_source_code_schema(gist, base)],
     )
-    return templates.TemplateResponse("gist_detail.html", {
-        **seo_ctx,
-        "request": request,
-        "user": user,
-        "gist": gist,
-        "author": author,
-        "is_owner": is_owner,
-        "star_count": star_count,
-        "time_ago": time_ago(gist["created_at"]),
-        "comments": comments,
+    return respond(request, "gist_detail.html", detail_context(request, user, detail, "gist", seo_ctx, {
         "languages": LANGUAGES,
-        "attachments": gist_attachments,
-    })
+    }), model=GistDetailOut)
 
 
 @router.post("/create")
-async def create_gist(request: Request):
+async def create_gist(request: Request, data: Annotated[GistForm, Form()]):
     user = require_user(request)
-    form = await request.form()
-    title = form.get("title", "").strip()
-    description = form.get("description", "").strip()
-    source_code = form.get("source_code", "").strip()
-    language = form.get("language", "plaintext")
-
-    if not title:
-        return RedirectResponse(url="/gists", status_code=302)
-    if len(title) > 200:
-        return RedirectResponse(url="/gists", status_code=302)
-    if not source_code:
-        return RedirectResponse(url="/gists", status_code=302)
-    if len(source_code) > 50000:
-        return RedirectResponse(url="/gists", status_code=302)
-    if len(description) > 5000:
-        return RedirectResponse(url="/gists", status_code=302)
+    title = data.title.strip()
+    description = data.description.strip()
+    source_code = data.source_code.strip()
+    language = data.language
 
     valid_languages = {l[0] for l in LANGUAGES}
     if language not in valid_languages:
         language = "plaintext"
 
-    gists = get_table("gists")
-    uid = generate_uid()
-    gist_slug = make_combined_slug(title, uid)
-    gists.insert({
-        "uid": uid,
-        "user_uid": user["uid"],
+    uid, gist_slug = create_content_item("gists", "gist", user, {
         "title": title,
-        "slug": gist_slug,
         "description": description or None,
         "source_code": source_code,
         "language": language,
-        "stars": 0,
-        "created_at": datetime.utcnow().isoformat(),
-    })
+    }, title, XP_GIST, "First Gist", description or "", data.attachment_uids)
+    url = f"/gists/{gist_slug}"
+    return action_result(request, url, data={"uid": uid, "slug": gist_slug, "url": url})
 
-    attachment_uids = form.getlist("attachment_uids") if hasattr(form, "getlist") else []
+
+@router.post("/edit/{gist_slug}")
+async def edit_gist(request: Request, gist_slug: str, data: Annotated[GistEditForm, Form()]):
+    user = require_user(request)
+    language = data.language
+    if language not in {l[0] for l in LANGUAGES}:
+        language = "plaintext"
+    return edit_content_item(request, "gists", user, gist_slug, {
+        "title": data.title.strip(),
+        "description": data.description.strip() or None,
+        "source_code": data.source_code.strip(),
+        "language": language,
+    }, "/gists")
+
+
+@router.post("/delete/{gist_slug}")
+async def delete_gist(request: Request, gist_slug: str):
+    user = require_user(request)
+    return delete_content_item(request, "gists", "gist", user, gist_slug, "/gists")

devplacepy/routers/leaderboard.py

@@ -0,0 +1,46 @@
+import logging
+from fastapi import APIRouter, Request
+from fastapi.responses import HTMLResponse
+from devplacepy.database import get_leaderboard, get_user_rank, get_site_stats, get_top_authors, get_featured_news
+from devplacepy.templating import templates
+from devplacepy.utils import get_current_user
+from devplacepy.seo import list_page_seo
+from devplacepy.responses import respond
+from devplacepy.schemas import LeaderboardOut
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+TOP_LIMIT = 50
+
+
+@router.get("", response_class=HTMLResponse)
+async def leaderboard_page(request: Request):
+    entries = get_leaderboard(TOP_LIMIT, 0)
+
+    user = get_current_user(request)
+    user_rank = get_user_rank(user["uid"]) if user else None
+
+    stats = get_site_stats()
+    top_authors = get_top_authors(5)
+    featured_news = get_featured_news(5)
+
+    seo_ctx = list_page_seo(
+        request,
+        title="Leaderboard",
+        description="Top contributors on DevPlace ranked by the stars their posts, projects, and gists have earned.",
+        breadcrumbs=[{"name": "Home", "url": "/feed"}, {"name": "Leaderboard", "url": "/leaderboard"}],
+    )
+    return respond(request, "leaderboard.html", {
+        **seo_ctx,
+        "request": request,
+        "user": user,
+        "entries": entries,
+        "user_rank": user_rank,
+        "total_members": stats["total_members"],
+        "posts_today": stats["posts_today"],
+        "total_projects": stats["total_projects"],
+        "total_gists": stats["total_gists"],
+        "top_authors": top_authors,
+        "featured_news": featured_news,
+    }, model=LeaderboardOut)

devplacepy/routers/messages.py

@@ -1,11 +1,16 @@
 import logging
-from datetime import datetime
-from fastapi import APIRouter, Request
+from typing import Annotated
+from datetime import datetime, timezone
+from fastapi import APIRouter, Request, Form
+from devplacepy.models import MessageForm
 from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse
-from devplacepy.database import get_table, db, get_attachments_by_type
-from devplacepy.templating import templates
+from devplacepy.database import get_table, db
+from devplacepy.attachments import get_attachments_batch
+from devplacepy.templating import templates, clear_unread_cache, clear_messages_cache
 from devplacepy.utils import generate_uid, require_user, time_ago, create_mention_notifications
 from devplacepy.seo import base_seo_context
+from devplacepy.responses import respond, action_result
+from devplacepy.schemas import MessagesOut
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
@@ -65,9 +70,10 @@ def get_conversation_messages(user_uid: str, other_uid: str):
             msgs.append(m)
     msgs.sort(key=lambda m: m["created_at"])
 
-    for msg in msgs:
-        if msg["receiver_uid"] == user_uid and not msg["read"]:
-            messages_table.update({"id": msg["id"], "read": True}, ["id"])
+    if "messages" in db.tables:
+        with db:
+            db.query("UPDATE messages SET read = 1 WHERE receiver_uid = :me AND sender_uid = :other AND read = 0", me=user_uid, other=other_uid)
+        clear_messages_cache(user_uid)
 
     from devplacepy.database import get_users_by_uids
     user_ids = list({m["sender_uid"] for m in msgs} | {other_uid})
@@ -76,7 +82,7 @@ def get_conversation_messages(user_uid: str, other_uid: str):
 
     result = []
     msg_uids = [m["uid"] for m in msgs]
-    attachments_map = get_attachments_by_type("message", msg_uids) if msg_uids else {}
+    attachments_map = get_attachments_batch("message", msg_uids) if msg_uids else {}
     for m in msgs:
         result.append({
             "message": m,
@@ -115,7 +121,7 @@ async def messages_page(request: Request, with_uid: str = None, search: str = ""
             {"name": "Messages", "url": "/messages"},
         ],
     )
-    return templates.TemplateResponse("messages.html", {
+    return respond(request, "messages.html", {
         **seo_ctx,
         "request": request,
         "user": user,
@@ -124,7 +130,7 @@ async def messages_page(request: Request, with_uid: str = None, search: str = ""
         "other_user": other_user,
         "current_conversation": current_conversation,
         "search": search,
-    })
+    }, model=MessagesOut)
 
 
 @router.get("/search")
@@ -144,22 +150,44 @@ async def search_users(request: Request, q: str = ""):
 
 
 @router.post("/send")
-async def send_message(request: Request):
+async def send_message(request: Request, data: Annotated[MessageForm, Form()]):
     user = require_user(request)
-    form = await request.form()
-    content = form.get("content", "").strip()
-    receiver_uid = form.get("receiver_uid", "")
+    content = data.content.strip()
+    receiver_uid = data.receiver_uid
 
-    if content and receiver_uid:
-        messages_table = get_table("messages")
-        msg_uid = generate_uid()
-        messages_table.insert({
-            "uid": msg_uid,
-            "sender_uid": user["uid"],
-            "receiver_uid": receiver_uid,
-            "content": content,
+    if not get_table("users").find_one(uid=receiver_uid):
+        return action_result(request, "/messages")
+
+    messages_table = get_table("messages")
+    msg_uid = generate_uid()
+    messages_table.insert({
+        "uid": msg_uid,
+        "sender_uid": user["uid"],
+        "receiver_uid": receiver_uid,
+        "content": content,
+        "read": False,
+        "created_at": datetime.now(timezone.utc).isoformat(),
+    })
+
+    from devplacepy.attachments import link_attachments
+    link_attachments(data.attachment_uids, "message", msg_uid)
+
+    if user["uid"] != receiver_uid:
+        notifications = get_table("notifications")
+        notifications.insert({
+            "uid": generate_uid(),
+            "user_uid": receiver_uid,
+            "type": "message",
+            "message": f"{user['username']} sent you a message",
+            "related_uid": user["uid"],
+            "target_url": f"/messages?with_uid={user['uid']}",
             "read": False,
-            "created_at": datetime.utcnow().isoformat(),
+            "created_at": datetime.now(timezone.utc).isoformat(),
         })
+        clear_unread_cache(receiver_uid)
+        clear_messages_cache(receiver_uid)
 
-        attachment_uids = form.getlist("attachment_uids") if hasattr(form, "getlist") else []
+    create_mention_notifications(content, user["uid"], f"/messages?with_uid={receiver_uid}")
+
+    logger.info(f"Message {msg_uid} sent from {user['username']} to {receiver_uid}")
+    return action_result(request, f"/messages?with_uid={receiver_uid}", data={"uid": msg_uid})

devplacepy/routers/news.py

@@ -1,11 +1,14 @@
 import logging
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from fastapi import APIRouter, Request
 from fastapi.responses import HTMLResponse
-from devplacepy.database import get_table, db, load_comments, resolve_by_slug
+from devplacepy.database import get_table, db, load_comments, resolve_by_slug, get_news_images_by_uids, get_user_bookmarks, paginate
 from devplacepy.templating import templates
-from devplacepy.utils import get_current_user, time_ago
-from devplacepy.seo import base_seo_context, website_schema, site_url, discussion_forum_posting, combine
+from devplacepy.utils import get_current_user, time_ago, not_found
+from devplacepy.content import canonical_redirect
+from devplacepy.seo import base_seo_context, website_schema, site_url, news_article_schema, list_page_seo, next_page_url
+from devplacepy.responses import respond
+from devplacepy.schemas import NewsListOut, NewsDetailOut
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
@@ -14,26 +17,22 @@ NEWS_MAX_AGE_DAYS = 4
 
 
 @router.get("", response_class=HTMLResponse)
-async def news_page(request: Request):
+async def news_page(request: Request, before: str = None):
     user = get_current_user(request)
-    cutoff = (datetime.utcnow() - timedelta(days=NEWS_MAX_AGE_DAYS)).isoformat()
+    cutoff = (datetime.now(timezone.utc) - timedelta(days=NEWS_MAX_AGE_DAYS)).isoformat()
 
     news_table = get_table("news")
-    articles = list(news_table.find(
+    articles, next_cursor = paginate(
+        news_table,
+        before=before,
+        order=["-grade", "-synced_at"],
+        cursor_field="synced_at",
         status="published",
         synced_at={">=": cutoff},
-        order_by=["-grade", "-synced_at"],
-    ))
+    )
 
     article_uids = [a["uid"] for a in articles]
-
-    images_by_news = {}
-    if article_uids and "news_images" in db.tables:
-        images_table = get_table("news_images")
-        for uid in article_uids:
-            img = images_table.find_one(news_uid=uid, order_by=["uid"])
-            if img:
-                images_by_news[uid] = img["url"]
+    images_by_news = get_news_images_by_uids(article_uids)
 
     enriched = []
     for a in articles:
@@ -44,21 +43,21 @@ async def news_page(request: Request):
             "grade": a.get("grade", 0),
         })
 
-    base = site_url(request)
-    seo_ctx = base_seo_context(
+    seo_ctx = list_page_seo(
         request,
         title="Developer News",
         description="Curated developer news and industry signals. Stay ahead with hand-picked articles.",
         breadcrumbs=[{"name": "Home", "url": "/feed"}, {"name": "News", "url": "/news"}],
-        schemas=[website_schema(base)],
+        next_url=next_page_url(request, next_cursor),
     )
 
-    return templates.TemplateResponse("news.html", {
+    return respond(request, "news.html", {
         **seo_ctx,
         "request": request,
         "user": user,
         "articles": enriched,
-    })
+        "next_cursor": next_cursor,
+    }, model=NewsListOut)
 
 
 @router.get("/{news_slug}", response_class=HTMLResponse)
@@ -67,7 +66,10 @@ async def news_detail_page(request: Request, news_slug: str):
     news_table = get_table("news")
     article = resolve_by_slug(news_table, news_slug)
     if not article:
-        return HTMLResponse("News article not found", status_code=404)
+        raise not_found("News article not found")
+    redirect = canonical_redirect("news", article, news_slug)
+    if redirect:
+        return redirect
 
     image_url = ""
     if "news_images" in db.tables:
@@ -76,7 +78,8 @@ async def news_detail_page(request: Request, news_slug: str):
             image_url = img["url"]
 
     canonical_slug = article.get("slug", "") or article["uid"]
-    comments = load_comments("news", article["uid"])
+    comments = load_comments("news", article["uid"], user)
+    bookmarked = bool(user) and article["uid"] in get_user_bookmarks(user["uid"], "news", [article["uid"]])
 
     base = site_url(request)
     page_url = f"{base}/news/{canonical_slug}"
@@ -84,11 +87,13 @@ async def news_detail_page(request: Request, news_slug: str):
         request,
         title=article.get("title", "News Article"),
         description=(article.get("description", "") or "")[:200],
+        og_type="article",
+        og_image=image_url or None,
         breadcrumbs=[{"name": "Home", "url": "/feed"}, {"name": "News", "url": "/news"}, {"name": article.get("title", "")[:60], "url": page_url}],
-        schemas=[website_schema(base)],
+        schemas=[website_schema(base), news_article_schema(article, base, image_url)],
     )
 
-    return templates.TemplateResponse("news_detail.html", {
+    return respond(request, "news_detail.html", {
         **seo_ctx,
         "request": request,
         "user": user,
@@ -98,4 +103,5 @@ async def news_detail_page(request: Request, news_slug: str):
         "grade": article.get("grade", 0),
         "time_ago": time_ago(article["synced_at"]),
         "comments": comments,
-    })
+        "bookmarked": bookmarked,
+    }, model=NewsDetailOut)

devplacepy/routers/notifications.py

@@ -1,20 +1,29 @@
 import logging
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from fastapi import APIRouter, Request
-from fastapi.responses import HTMLResponse, RedirectResponse
-from devplacepy.database import get_table
-from devplacepy.templating import templates
-from devplacepy.utils import require_user, time_ago
+from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse
+from devplacepy.database import get_table, db
+from devplacepy.templating import (
+    templates,
+    clear_unread_cache,
+    jinja_unread_count,
+    jinja_unread_messages,
+)
+from devplacepy.utils import require_user, get_current_user, time_ago
 from devplacepy.seo import base_seo_context
+from devplacepy.responses import respond, action_result
+from devplacepy.schemas import NotificationsOut
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
+PAGE_SIZE = 25
+
 
 def _group_label(created_at: str) -> str:
     try:
         dt = datetime.fromisoformat(created_at)
-        now = datetime.utcnow()
+        now = datetime.now(timezone.utc)
         today = now.date()
         date = dt.date()
         if date == today:
@@ -29,15 +38,24 @@ def _group_label(created_at: str) -> str:
 
 
 @router.get("", response_class=HTMLResponse)
-async def notifications_page(request: Request):
+async def notifications_page(request: Request, before: str = None):
     user = require_user(request)
 
+    next_cursor = None
     try:
         notifications_table = get_table("notifications")
+        filters = {"user_uid": user["uid"]}
+        if before:
+            filters["created_at"] = {"<": before}
         raw_notifications = list(
-            notifications_table.find(user_uid=user["uid"], order_by=["-created_at"])
+            notifications_table.find(**filters, order_by=["-created_at"], _limit=PAGE_SIZE + 1)
         )
 
+        has_more = len(raw_notifications) > PAGE_SIZE
+        raw_notifications = raw_notifications[:PAGE_SIZE]
+        if has_more and raw_notifications:
+            next_cursor = raw_notifications[-1]["created_at"]
+
         enriched = []
         if raw_notifications:
             from devplacepy.database import get_users_by_uids
@@ -78,14 +96,39 @@ async def notifications_page(request: Request):
             {"name": "Notifications", "url": "/notifications"},
         ],
     )
-    return templates.TemplateResponse("notifications.html", {
+    return respond(request, "notifications.html", {
         **seo_ctx,
         "request": request,
         "user": user,
         "notification_groups": groups,
+        "next_cursor": next_cursor,
+    }, model=NotificationsOut)
+
+
+@router.get("/counts")
+async def unread_counts(request: Request):
+    user = get_current_user(request)
+    if not user:
+        return JSONResponse({"notifications": 0, "messages": 0})
+    return JSONResponse({
+        "notifications": jinja_unread_count(user["uid"]),
+        "messages": jinja_unread_messages(user["uid"]),
     })
 
 
+@router.get("/open/{notification_uid}")
+async def open_notification(request: Request, notification_uid: str):
+    user = require_user(request)
+    notifications_table = get_table("notifications")
+    n = notifications_table.find_one(uid=notification_uid)
+    if not n or n["user_uid"] != user["uid"]:
+        return action_result(request, "/notifications")
+    if not n["read"]:
+        notifications_table.update({"id": n["id"], "read": True}, ["id"])
+        clear_unread_cache(user["uid"])
+    return action_result(request, n.get("target_url") or "/notifications")
+
+
 @router.post("/mark-read/{notification_uid}")
 async def mark_read(request: Request, notification_uid: str):
     user = require_user(request)
@@ -93,13 +136,14 @@ async def mark_read(request: Request, notification_uid: str):
     n = notifications_table.find_one(uid=notification_uid)
     if n and n["user_uid"] == user["uid"]:
         notifications_table.update({"id": n["id"], "read": True}, ["id"])
-    return RedirectResponse(url="/notifications", status_code=302)
+        clear_unread_cache(user["uid"])
+    return action_result(request, "/notifications")
 
 
 @router.post("/mark-all-read")
 async def mark_all_read(request: Request):
     user = require_user(request)
-    notifications_table = get_table("notifications")
-    for n in notifications_table.find(user_uid=user["uid"], read=False):
-        notifications_table.update({"id": n["id"], "read": True}, ["id"])
-    return RedirectResponse(url="/notifications", status_code=302)
+    with db:
+        db.query("UPDATE notifications SET read = 1 WHERE user_uid = :u AND read = 0", u=user["uid"])
+    clear_unread_cache(user["uid"])
+    return action_result(request, "/notifications")

devplacepy/routers/openai_gateway.py

@@ -0,0 +1,23 @@
+import logging
+from fastapi import APIRouter, Request, HTTPException
+from devplacepy.services.manager import service_manager
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+
+def _service():
+    svc = service_manager.get_service("openai")
+    if svc is None:
+        raise HTTPException(status_code=503, detail="Gateway is not available")
+    return svc
+
+
+@router.post("/v1/chat/completions")
+async def chat_completions(request: Request):
+    return await _service().handle(request, "chat/completions")
+
+
+@router.api_route("/v1/{path:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"])
+async def passthrough(request: Request, path: str):
+    return await _service().handle(request, path)

devplacepy/routers/polls.py

@@ -0,0 +1,44 @@
+# retoor <retoor@molodetz.nl>
+import logging
+from typing import Annotated
+from datetime import datetime, timezone
+from fastapi import APIRouter, Request, Form
+from fastapi.responses import RedirectResponse, JSONResponse
+from devplacepy.database import get_table, get_poll_for_post
+from devplacepy.utils import generate_uid, require_user
+from devplacepy.models import PollVoteForm
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+
+@router.post("/{poll_uid}/vote")
+async def vote_poll(request: Request, poll_uid: str, data: Annotated[PollVoteForm, Form()]):
+    user = require_user(request)
+    poll = get_table("polls").find_one(uid=poll_uid)
+    if not poll:
+        return JSONResponse({"error": "Poll not found"}, status_code=404)
+    option = get_table("poll_options").find_one(uid=data.option_uid, poll_uid=poll_uid)
+    if not option:
+        return JSONResponse({"error": "Invalid option"}, status_code=400)
+
+    poll_votes = get_table("poll_votes")
+    existing = poll_votes.find_one(poll_uid=poll_uid, user_uid=user["uid"])
+    if existing:
+        if existing["option_uid"] == data.option_uid:
+            poll_votes.delete(id=existing["id"])
+        else:
+            poll_votes.update({"id": existing["id"], "option_uid": data.option_uid}, ["id"])
+    else:
+        poll_votes.insert({
+            "uid": generate_uid(),
+            "poll_uid": poll_uid,
+            "option_uid": data.option_uid,
+            "user_uid": user["uid"],
+            "created_at": datetime.now(timezone.utc).isoformat(),
+        })
+
+    result = get_poll_for_post(poll["post_uid"], user)
+    if request.headers.get("x-requested-with") == "fetch":
+        return JSONResponse(result)
+    return RedirectResponse(url=request.headers.get("Referer", "/feed"), status_code=302)

devplacepy/routers/posts.py

@@ -1,116 +1,86 @@
 import logging
-import aiofiles
-from datetime import datetime
-from pathlib import Path
-from fastapi import APIRouter, Request, HTTPException
+from typing import Annotated
+from datetime import datetime, timezone
+from fastapi import APIRouter, Request, Form
 from fastapi.responses import RedirectResponse, HTMLResponse
-from devplacepy.database import get_table, get_comment_counts_by_post_uids, load_comments, db
+from devplacepy.constants import TOPICS
+from devplacepy.database import db, get_table, resolve_by_slug
 from devplacepy.templating import templates
-from devplacepy.config import STATIC_DIR
-from devplacepy.utils import generate_uid, get_current_user, require_user, time_ago, make_combined_slug, create_mention_notifications
-from devplacepy.seo import base_seo_context, site_url, website_schema, breadcrumb_schema, discussion_forum_posting, combine, truncate
+from devplacepy.utils import get_current_user, require_user, time_ago, not_found, generate_uid, XP_POST
+from devplacepy.content import load_detail, edit_content_item, delete_content_item, create_content_item, detail_context, canonical_redirect, first_image_url
+from devplacepy.responses import respond, action_result
+from devplacepy.schemas import PostDetailOut
+from devplacepy.seo import base_seo_context, site_url, website_schema, discussion_forum_posting, truncate
+from devplacepy.attachments import save_inline_image
+from devplacepy.models import PostForm, PostEditForm
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
 
 @router.post("/create")
-async def create_post(request: Request):
+async def create_post(request: Request, data: Annotated[PostForm, Form()]):
     user = require_user(request)
-    form = await request.form()
-    content = form.get("content", "").strip()
-    title = form.get("title", "").strip()
-    topic = form.get("topic", "random")
-    project_uid = form.get("project_uid", "")
-
-    errors = []
-    if not content:
-        errors.append("Content is required")
-    if content and len(content) < 10:
-        errors.append("Content must be at least 10 characters")
-    if len(content) > 2000:
-        errors.append("Content too long (max 2000 characters)")
-    if topic not in TOPICS:
-        topic = "random"
-
-    if errors:
-        return RedirectResponse(url="/feed", status_code=302)
+    content = data.content.strip()
+    title = data.title.strip()
+    topic = data.topic
+    project_uid = data.project_uid
 
     image_filename = None
+    form = await request.form()
     image_file = form.get("image")
-    if image_file and hasattr(image_file, "filename") and image_file.filename:
-        try:
-            content_bytes = await image_file.read()
-            if len(content_bytes) > 5 * 1024 * 1024:
-                logger.warning(f"Image too large: {image_file.filename}")
-            else:
-                import imghdr
-                ext = Path(image_file.filename).suffix.lower()
-                allowed = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg"}
-                if ext not in allowed:
-                    logger.warning(f"Unsupported image type: {ext}")
-                else:
-                    upload_dir = STATIC_DIR / "uploads"
-                    upload_dir.mkdir(parents=True, exist_ok=True)
-                    image_filename = f"{generate_uid()}{ext}"
-                    file_path = upload_dir / image_filename
-                    async with aiofiles.open(str(file_path), "wb") as f:
-                        await f.write(content_bytes)
-                    logger.info(f"Image saved: {image_filename}")
-                    content += f"\n\n![](/static/uploads/{image_filename})"
-        except Exception as e:
-            logger.warning(f"Image upload failed: {e}")
+    if image_file is not None and hasattr(image_file, "filename") and image_file.filename:
+        image_filename = save_inline_image(await image_file.read(), image_file.filename)
+        if image_filename:
+            content += f"\n\n![](/static/uploads/{image_filename})"
 
-    posts = get_table("posts")
-    uid = generate_uid()
     slug_text = title if title else content[:50]
-    post_slug = make_combined_slug(slug_text, uid)
-    posts.insert({
-        "uid": uid,
-        "user_uid": user["uid"],
+    uid, post_slug = create_content_item("posts", "post", user, {
         "title": title or None,
-        "slug": post_slug,
         "content": content,
         "topic": topic,
         "project_uid": project_uid or None,
         "image": image_filename,
-        "stars": 0,
-        "created_at": datetime.utcnow().isoformat(),
+    }, slug_text, XP_POST, "First Post", content, data.attachment_uids)
+
+    create_poll(uid, user, data.poll_question, data.poll_options)
+    url = f"/posts/{post_slug}"
+    return action_result(request, url, data={"uid": uid, "slug": post_slug, "url": url})
+
+
+def create_poll(post_uid: str, user: dict, question: str, options: list[str]) -> None:
+    question = question.strip()
+    labels = [option.strip() for option in options if option.strip()][:6]
+    if not question or len(labels) < 2:
+        return
+    poll_uid = generate_uid()
+    get_table("polls").insert({
+        "uid": poll_uid,
+        "post_uid": post_uid,
+        "user_uid": user["uid"],
+        "question": question,
+        "created_at": datetime.now(timezone.utc).isoformat(),
     })
-
-    badges = get_table("badges")
-    existing = badges.find_one(user_uid=user["uid"], badge_name="First Post")
-    if not existing:
-        badges.insert({
-            "uid": generate_uid(),
-            "user_uid": user["uid"],
-            "badge_name": "First Post",
-            "created_at": datetime.utcnow().isoformat(),
-        })
-
-    attachment_uids = form.getlist("attachment_uids")
-    if attachment_uids:
-        link_attachments(attachment_uids, "post", uid)
-    if attachment_uids:
-        link_attachments(attachment_uids, "post", uid)
-
-    create_mention_notifications(content, user["uid"], f"/posts/{post_slug}")
-    logger.info(f"Post {uid} created by {user['username']}")
-    return RedirectResponse(url=f"/posts/{post_slug}", status_code=302)
+    get_table("poll_options").insert_many([{
+        "uid": generate_uid(),
+        "poll_uid": poll_uid,
+        "label": label,
+        "position": index,
+    } for index, label in enumerate(labels)])
 
 
 @router.get("/{post_slug}", response_class=HTMLResponse)
 async def view_post(request: Request, post_slug: str):
     user = get_current_user(request)
-    posts = get_table("posts")
-    post = resolve_by_slug(posts, post_slug)
-    if not post:
-        raise HTTPException(status_code=404, detail="Post not found")
-
-    users_table = get_table("users")
-    author = users_table.find_one(uid=post["user_uid"])
-
-    top_level = load_comments("post", post["uid"])
+    detail = load_detail("posts", "post", post_slug, user)
+    if not detail:
+        raise not_found("Post not found")
+    post = detail["item"]
+    redirect = canonical_redirect("posts", post, post_slug)
+    if redirect:
+        return redirect
+    author = detail["author"]
+    top_level = detail["comments"]
 
     def count_all(items):
         total = len(items)
@@ -118,7 +88,6 @@ async def view_post(request: Request, post_slug: str):
             total += count_all(item.get("children", []))
         return total
     comment_count = count_all(top_level)
-    star_count = post.get("stars", 0)
     base = site_url(request)
     seo_ctx = base_seo_context(
         request,
@@ -130,9 +99,10 @@ async def view_post(request: Request, post_slug: str):
             {"name": post.get("title") or "Post", "url": f"/posts/{post['slug'] or post['uid']}"},
         ],
         og_type="article",
+        og_image=first_image_url(post, detail["attachments"]),
         schemas=[
             website_schema(base),
-            discussion_forum_posting(post, author, comment_count, star_count, base),
+            discussion_forum_posting(post, author, comment_count, detail["star_count"], base),
         ],
     )
 
@@ -149,76 +119,29 @@ async def view_post(request: Request, post_slug: str):
                 "time_ago": time_ago(r["created_at"]),
             })
 
-    post_attachments = get_attachments("post", post["uid"])
-
-    return templates.TemplateResponse("post.html", {
-        **seo_ctx,
-        "request": request,
-        "user": user,
-        "post": post,
-        "author": author,
-        "comments": top_level,
-        "time_ago": time_ago(post["created_at"]),
+    return respond(request, "post.html", detail_context(request, user, detail, "post", seo_ctx, {
         "comment_count": comment_count,
         "related_posts": related_posts,
         "topics": list(TOPICS),
-        "attachments": post_attachments,
-    })
+    }), model=PostDetailOut)
 
 
 @router.post("/edit/{post_slug}")
-async def edit_post(request: Request, post_slug: str):
+async def edit_post(request: Request, post_slug: str, data: Annotated[PostEditForm, Form()]):
     user = require_user(request)
-    form = await request.form()
-    content = form.get("content", "").strip()
-    title = form.get("title", "").strip()
-    topic = form.get("topic", "random")
-
-    errors = []
-    if not content:
-        errors.append("Content is required")
-    if content and len(content) < 10:
-        errors.append("Content must be at least 10 characters")
-    if len(content) > 2000:
-        errors.append("Content too long (max 2000 characters)")
-    if topic not in TOPICS:
-        topic = "random"
-
-    posts = get_table("posts")
-    post = resolve_by_slug(posts, post_slug)
-    if not post or post["user_uid"] != user["uid"]:
-        return RedirectResponse(url="/feed", status_code=302)
-
-    if errors:
-        return RedirectResponse(url=f"/posts/{post['slug'] or post['uid']}", status_code=302)
-
-    posts.update({
-        "uid": post["uid"],
-        "content": content,
-        "title": title or None,
-        "topic": topic,
-    }, ["uid"])
-
-    logger.info(f"Post {post['uid']} edited by {user['username']}")
-    return RedirectResponse(url=f"/posts/{post['slug'] or post['uid']}", status_code=302)
+    result = edit_content_item(request, "posts", user, post_slug, {
+        "content": data.content.strip(),
+        "title": data.title.strip() or None,
+        "topic": data.topic,
+    }, "/feed")
+    if data.poll_question.strip():
+        post = resolve_by_slug(get_table("posts"), post_slug)
+        if post and post["user_uid"] == user["uid"] and not get_table("polls").find_one(post_uid=post["uid"]):
+            create_poll(post["uid"], user, data.poll_question, data.poll_options)
+    return result
 
 
 @router.post("/delete/{post_slug}")
 async def delete_post(request: Request, post_slug: str):
     user = require_user(request)
-    posts = get_table("posts")
-    post = resolve_by_slug(posts, post_slug)
-    if post and post["user_uid"] == user["uid"]:
-        delete_target_attachments("post", post["uid"])
-        get_table("comments").delete(post_uid=post["uid"])
-        get_table("votes").delete(target_uid=post["uid"])
-        image = post.get("image")
-        if image:
-            try:
-                (STATIC_DIR / "uploads" / image).unlink(missing_ok=True)
-            except Exception as e:
-                logger.warning(f"Failed to delete image {image}: {e}")
-        posts.delete(id=post["id"])
-        logger.info(f"Post {post['uid']} deleted by {user['username']}")
-    return RedirectResponse(url="/feed", status_code=302)
-from devplacepy.attachments import get_attachments, link_attachments, delete_target_attachments
+    return delete_content_item(request, "posts", "post", user, post_slug, "/feed", inline_image_field="image")

devplacepy/routers/profile.py

@@ -1,97 +1,44 @@
 import logging
-from fastapi import APIRouter, Request
+from typing import Annotated
+from fastapi import APIRouter, Request, Form
+from devplacepy.models import ProfileForm
 from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse
-from devplacepy.database import get_table, db
+from devplacepy.database import get_table, db, get_user_stars, get_user_rank, get_comment_counts_by_post_uids, get_reactions_by_targets, get_user_bookmarks, get_polls_by_post_uids, get_activity_heatmap, get_activity_months, get_streaks, get_follow_counts, get_follow_list, get_following_among
+from devplacepy.content import enrich_items
 from devplacepy.templating import templates
-from devplacepy.utils import get_current_user, require_user, time_ago
-from devplacepy.seo import base_seo_context, site_url, website_schema, breadcrumb_schema, profile_page_schema, combine
+from devplacepy.utils import get_current_user, require_user, require_user_api, time_ago, clear_user_cache, not_found, generate_uid
+from devplacepy.responses import respond, action_result
+from devplacepy.schemas import ProfileOut
+from devplacepy.avatar import avatar_url
+from devplacepy.seo import base_seo_context, site_url, website_schema, profile_page_schema
+from devplacepy.services.manager import service_manager
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
 
-@router.get("/{username}", response_class=HTMLResponse)
-async def profile_page(request: Request, username: str, tab: str = "posts"):
-    current_user = get_current_user(request)
-    users = get_table("users")
-    profile_user = users.find_one(username=username)
-    if not profile_user:
-        return RedirectResponse(url="/feed", status_code=302)
-
-    posts = []
-    if tab == "posts":
-        posts_table = get_table("posts")
-        raw_posts = list(posts_table.find(user_uid=profile_user["uid"], order_by=["-created_at"]))
-        if raw_posts:
-            from devplacepy.database import get_comment_counts_by_post_uids
-            counts = get_comment_counts_by_post_uids([p["uid"] for p in raw_posts])
-        else:
-            counts = {}
-        for p in raw_posts:
-            posts.append({
-                "post": p,
-                "time_ago": time_ago(p["created_at"]),
-                "comment_count": counts.get(p["uid"], 0),
-            })
-
-    badges = list(get_table("badges").find(user_uid=profile_user["uid"]))
-    projects = list(get_table("projects").find(user_uid=profile_user["uid"]))
-    gists = list(get_table("gists").find(user_uid=profile_user["uid"]))
-    posts_count = len(posts) or len(list(get_table("posts").find(user_uid=profile_user["uid"])))
-
-    activities = []
-    if tab == "activity":
-        posts_table = get_table("posts")
-        for p in posts_table.find(user_uid=profile_user["uid"], order_by=["-created_at"], _limit=10):
-            activities.append({"type": "post", "content": p.get("title") or p["content"][:80], "time_ago": time_ago(p["created_at"]), "created_at": p["created_at"], "uid": p["uid"]})
-        comments_table = get_table("comments")
-        for c in comments_table.find(user_uid=profile_user["uid"], order_by=["-created_at"], _limit=10):
-            activities.append({"type": "comment", "content": c["content"][:80], "time_ago": time_ago(c["created_at"]), "created_at": c["created_at"], "uid": c["post_uid"]})
-        activities.sort(key=lambda a: a["created_at"], reverse=True)
-
-    is_following = False
-    if current_user:
-        follows = get_table("follows")
-        is_following = bool(follows.find_one(follower_uid=current_user["uid"], following_uid=profile_user["uid"]))
-
-    base = site_url(request)
-    robots = "noindex,follow" if posts_count < 2 else "index,follow"
-    bio = profile_user.get("bio", "")
-    desc = bio or f"View {profile_user['username']}'s profile on DevPlace. {posts_count} posts."
-    seo_ctx = base_seo_context(
-        request,
-        title=f"{profile_user['username']} (@{profile_user['username']})",
-        description=desc,
-        robots=robots,
-        breadcrumbs=[
-            {"name": "Home", "url": "/feed"},
-            {"name": profile_user['username'], "url": f"/profile/{profile_user['username']}"},
-        ],
-        schemas=[
-            website_schema(base),
-            profile_page_schema(profile_user, posts_count, base),
-        ],
-    )
-
-    return templates.TemplateResponse("profile.html", {
-        **seo_ctx,
-        "request": request,
-        "user": current_user,
-        "profile_user": profile_user,
-        "posts": posts,
-        "badges": badges,
-        "projects": projects,
-        "gists": gists,
-        "current_tab": tab,
-        "posts_count": posts_count,
-        "is_following": is_following,
-        "activities": activities,
-    })
+def _ai_quota(user_uid: str, include_cost: bool = False, is_admin: bool = False) -> dict | None:
+    devii = service_manager.get_service("devii")
+    if devii is None:
+        return None
+    try:
+        limit = devii.daily_limit_for("user", is_admin)
+        spent = devii.spent_24h("user", user_uid)
+        turns = devii.hub().ledger.turns_24h("user", user_uid)
+    except Exception:
+        logger.exception("Failed to compute AI quota for %s", user_uid)
+        return None
+    used_pct = round(min(100.0, spent / limit * 100), 1) if limit > 0 else 0.0
+    quota = {"used_pct": used_pct, "turns": turns}
+    if include_cost:
+        quota["spent_usd"] = round(spent, 4)
+        quota["limit_usd"] = round(limit, 2)
+    return quota
 
 
 @router.get("/search")
 async def search_users(request: Request, q: str = ""):
-    require_user(request)
+    require_user_api(request)
     if not q or len(q) < 1:
         return JSONResponse({"results": []})
     if "users" in db.tables:
@@ -105,22 +52,184 @@ async def search_users(request: Request, q: str = ""):
     return JSONResponse({"results": results})
 
 
+def _follow_people(profile_uid: str, mode: str, current_user, page: int):
+    people, pagination = get_follow_list(profile_uid, mode, page)
+    mine = get_following_among(current_user["uid"], [p["uid"] for p in people]) if current_user else set()
+    for person in people:
+        person["is_following"] = person["uid"] in mine
+        person["is_self"] = bool(current_user and current_user["uid"] == person["uid"])
+    return people, pagination
+
+
+@router.get("/{username}/followers")
+async def profile_followers(request: Request, username: str, page: int = 1):
+    return _follow_list_json(request, username, "followers", page)
+
+
+@router.get("/{username}/following")
+async def profile_following(request: Request, username: str, page: int = 1):
+    return _follow_list_json(request, username, "following", page)
+
+
+def _follow_list_json(request: Request, username: str, mode: str, page: int):
+    profile_user = get_table("users").find_one(username=username)
+    if not profile_user:
+        raise not_found("Profile not found")
+    current_user = get_current_user(request)
+    people, pagination = _follow_people(profile_user["uid"], mode, current_user, page)
+    return JSONResponse({
+        "username": profile_user["username"],
+        "mode": mode,
+        "count": pagination["total"],
+        "page": pagination["page"],
+        "total_pages": pagination["total_pages"],
+        mode: [{"uid": p["uid"], "username": p["username"], "bio": p["bio"], "is_following": p["is_following"]} for p in people],
+    })
+
+
+@router.get("/{username}", response_class=HTMLResponse)
+async def profile_page(request: Request, username: str, tab: str = "posts", page: int = 1):
+    current_user = get_current_user(request)
+    users = get_table("users")
+    profile_user = users.find_one(username=username)
+    if not profile_user:
+        raise not_found("Profile not found")
+    profile_user["stars"] = get_user_stars(profile_user["uid"])
+    rank = get_user_rank(profile_user["uid"])
+    follow_counts = get_follow_counts(profile_user["uid"])
+
+    people = []
+    follow_pagination = None
+    if tab in ("followers", "following"):
+        people, follow_pagination = _follow_people(profile_user["uid"], tab, current_user, page)
+
+    posts = []
+    if tab == "posts":
+        posts_table = get_table("posts")
+        raw_posts = list(posts_table.find(user_uid=profile_user["uid"], order_by=["-created_at"]))
+        post_uids = [p["uid"] for p in raw_posts]
+        counts = get_comment_counts_by_post_uids(post_uids) if raw_posts else {}
+        authors = {profile_user["uid"]: profile_user}
+        posts = enrich_items(raw_posts, "post", authors, {"comment_count": counts}, user=current_user)
+        reactions_map = get_reactions_by_targets("post", post_uids, current_user)
+        bookmark_set = get_user_bookmarks(current_user["uid"], "post", post_uids) if current_user else set()
+        polls_map = get_polls_by_post_uids(post_uids, current_user)
+        for item in posts:
+            uid = item["post"]["uid"]
+            item["reactions"] = reactions_map.get(uid, {"counts": {}, "mine": []})
+            item["bookmarked"] = uid in bookmark_set
+            item["poll"] = polls_map.get(uid)
+
+    badges = list(get_table("badges").find(user_uid=profile_user["uid"]))
+    can_see_private = bool(current_user and (current_user["uid"] == profile_user["uid"] or current_user.get("role") == "Admin"))
+    projects = list(get_table("projects").find(user_uid=profile_user["uid"]))
+    if not can_see_private:
+        projects = [p for p in projects if not p.get("is_private")]
+    gists_raw = list(get_table("gists").find(user_uid=profile_user["uid"]))
+    gists = []
+    for g in gists_raw:
+        gists.append({"gist": g, "time_ago": time_ago(g["created_at"])})
+    posts_count = len(posts) or get_table("posts").count(user_uid=profile_user["uid"])
+
+    activities = []
+    if tab == "activity":
+        posts_table = get_table("posts")
+        for p in posts_table.find(user_uid=profile_user["uid"], order_by=["-created_at"], _limit=10):
+            activities.append({"type": "post", "content": p.get("title") or p["content"][:80], "time_ago": time_ago(p["created_at"]), "created_at": p["created_at"], "uid": p["uid"]})
+        comments_table = get_table("comments")
+        for c in comments_table.find(user_uid=profile_user["uid"], order_by=["-created_at"], _limit=10):
+            target_uid = c.get("target_uid", c.get("post_uid", ""))
+            target_type = c.get("target_type", "post")
+            activities.append({"type": "comment", "content": c["content"][:80], "time_ago": time_ago(c["created_at"]), "created_at": c["created_at"], "uid": target_uid, "target_type": target_type})
+        activities.sort(key=lambda a: a["created_at"], reverse=True)
+
+    heatmap = get_activity_heatmap(profile_user["uid"])
+    heatmap_months = get_activity_months(heatmap)
+    streak = get_streaks(profile_user["uid"])
+
+    is_following = False
+    if current_user:
+        follows = get_table("follows")
+        is_following = bool(follows.find_one(follower_uid=current_user["uid"], following_uid=profile_user["uid"]))
+
+    is_owner = bool(current_user and current_user["uid"] == profile_user["uid"])
+    viewer_is_admin = bool(current_user and current_user.get("role") == "Admin")
+    can_view_api_key = bool(current_user and (is_owner or viewer_is_admin))
+    api_key = profile_user.get("api_key", "") if can_view_api_key else ""
+    profile_is_admin = profile_user.get("role") == "Admin"
+    ai_quota = _ai_quota(profile_user["uid"], include_cost=viewer_is_admin, is_admin=profile_is_admin) if (is_owner or viewer_is_admin) else None
+
+    base = site_url(request)
+    robots = "noindex,follow" if posts_count < 2 else "index,follow"
+    bio = profile_user.get("bio", "")
+    desc = bio or f"View {profile_user['username']}'s profile on DevPlace. {posts_count} posts."
+    seo_ctx = base_seo_context(
+        request,
+        title=f"{profile_user['username']} (@{profile_user['username']})",
+        description=desc,
+        robots=robots,
+        og_type="profile",
+        og_image=avatar_url("multiavatar", profile_user["username"], 256),
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {"name": profile_user['username'], "url": f"/profile/{profile_user['username']}"},
+        ],
+        schemas=[
+            website_schema(base),
+            profile_page_schema(profile_user, posts_count, base),
+        ],
+    )
+
+    return respond(request, "profile.html", {
+        **seo_ctx,
+        "request": request,
+        "user": current_user,
+        "profile_user": profile_user,
+        "posts": posts,
+        "badges": badges,
+        "projects": projects,
+        "gists": gists,
+        "current_tab": tab,
+        "posts_count": posts_count,
+        "is_following": is_following,
+        "is_owner": is_owner,
+        "can_view_api_key": can_view_api_key,
+        "api_key": api_key,
+        "ai_quota": ai_quota,
+        "activities": activities,
+        "rank": rank,
+        "heatmap": heatmap,
+        "heatmap_months": heatmap_months,
+        "streak": streak,
+        "people": people,
+        "follow_pagination": follow_pagination,
+        "followers_count": follow_counts["followers"],
+        "following_count": follow_counts["following"],
+    }, model=ProfileOut)
+
+
 @router.post("/update")
-async def update_profile(request: Request):
+async def update_profile(request: Request, data: Annotated[ProfileForm, Form()]):
     user = require_user(request)
-    form = await request.form()
-    bio = form.get("bio", "").strip()
-    location = form.get("location", "").strip()
-    git_link = form.get("git_link", "").strip()
-    website = form.get("website", "").strip()
     users = get_table("users")
     users.update({
         "uid": user["uid"],
-        "bio": bio,
-        "location": location,
-        "git_link": git_link,
-        "website": website,
+        "bio": data.bio.strip(),
+        "location": data.location.strip(),
+        "git_link": data.git_link.strip(),
+        "website": data.website.strip(),
     }, ["uid"])
+    clear_user_cache(user["uid"])
 
     logger.info(f"Profile updated for {user['username']}")
-    return RedirectResponse(url=f"/profile/{user['username']}", status_code=302)
+    return action_result(request, f"/profile/{user['username']}")
+
+
+@router.post("/regenerate-api-key")
+async def regenerate_api_key(request: Request):
+    user = require_user(request)
+    new_key = generate_uid()
+    get_table("users").update({"uid": user["uid"], "api_key": new_key}, ["uid"])
+    clear_user_cache(user["uid"])
+    logger.info(f"API key regenerated for {user['username']}")
+    return JSONResponse({"api_key": new_key})

devplacepy/routers/project_files.py

@@ -0,0 +1,242 @@
+# retoor <retoor@molodetz.nl>
+
+import logging
+from typing import Annotated
+
+from fastapi import APIRouter, Request, Form
+from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
+
+from devplacepy.database import get_table, resolve_by_slug
+from devplacepy.content import is_owner, can_view_project
+from devplacepy.models import (
+    ProjectFileWriteForm,
+    ProjectFileMkdirForm,
+    ProjectFileMoveForm,
+    ProjectFileDeleteForm,
+    ProjectFileReplaceLinesForm,
+    ProjectFileInsertLinesForm,
+    ProjectFileDeleteLinesForm,
+    ProjectFileAppendForm,
+)
+from devplacepy.responses import respond, action_result, wants_json, json_error
+from devplacepy.schemas import ProjectFilesOut
+from devplacepy.services.jobs import queue
+from devplacepy.seo import base_seo_context
+from devplacepy.utils import get_current_user, require_user, not_found
+from devplacepy import project_files
+from devplacepy.project_files import ProjectFileError
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+
+def _load_project(project_slug: str) -> dict:
+    project = resolve_by_slug(get_table("projects"), project_slug)
+    if not project:
+        raise not_found("Project not found")
+    return project
+
+
+def _load_viewable_project(request: Request, project_slug: str) -> dict:
+    project = _load_project(project_slug)
+    if not can_view_project(project, get_current_user(request)):
+        raise not_found("Project not found")
+    return project
+
+
+def _files_url(project: dict) -> str:
+    return f"/projects/{project['slug'] or project['uid']}/files"
+
+
+def _deny(request: Request, project: dict):
+    if wants_json(request):
+        return json_error(403, "Not allowed")
+    return RedirectResponse(url=_files_url(project), status_code=302)
+
+
+def _fail(request: Request, project: dict, exc: ProjectFileError):
+    if wants_json(request):
+        return json_error(400, str(exc))
+    return RedirectResponse(url=_files_url(project), status_code=302)
+
+
+@router.get("/{project_slug}/files", response_class=HTMLResponse)
+async def project_files_page(request: Request, project_slug: str):
+    user = get_current_user(request)
+    project = _load_viewable_project(request, project_slug)
+    files = project_files.list_files(project["uid"])
+    seo_ctx = base_seo_context(
+        request,
+        title=f"Files - {project.get('title', 'Project')}",
+        description=f"Files in the {project.get('title', 'project')} project.",
+        robots="noindex,follow",
+    )
+    return respond(request, "project_files.html", {
+        **seo_ctx,
+        "request": request,
+        "user": user,
+        "project": project,
+        "files": files,
+        "is_owner": is_owner(project, user),
+        "read_only": bool(project.get("read_only")),
+        "is_private": bool(project.get("is_private")),
+    }, model=ProjectFilesOut)
+
+
+@router.get("/{project_slug}/files/raw")
+async def project_file_raw(request: Request, project_slug: str, path: str):
+    project = _load_viewable_project(request, project_slug)
+    try:
+        return JSONResponse(project_files.read_file(project["uid"], path))
+    except ProjectFileError as exc:
+        return json_error(404, str(exc))
+
+
+@router.post("/{project_slug}/files/zip")
+async def project_files_zip(request: Request, project_slug: str, path: str = ""):
+    project = _load_viewable_project(request, project_slug)
+    user = get_current_user(request)
+    subpath = ""
+    name = project.get("title") or "project"
+    if path:
+        try:
+            subpath = project_files.normalize_path(path)
+        except ProjectFileError as exc:
+            return json_error(400, str(exc))
+        if project_files.get_node(project["uid"], subpath) is None:
+            return json_error(404, "Path not found")
+        name = subpath.rsplit("/", 1)[-1]
+    owner_kind, owner_id = ("user", user["uid"]) if user else ("guest", request.client.host or "anon")
+    uid = queue.enqueue(
+        "zip",
+        {"source": {"type": "project_tree", "project_uid": project["uid"], "path": subpath}},
+        owner_kind, owner_id, name,
+    )
+    return JSONResponse({"uid": uid, "status_url": f"/zips/{uid}"})
+
+
+@router.get("/{project_slug}/files/lines")
+async def project_file_lines(request: Request, project_slug: str, path: str, start: int = 1, end: int = -1):
+    project = _load_viewable_project(request, project_slug)
+    try:
+        return JSONResponse(project_files.read_lines(project["uid"], path, start, None if end < 0 else end))
+    except ProjectFileError as exc:
+        return json_error(404, str(exc))
+
+
+@router.post("/{project_slug}/files/write")
+async def project_file_write(request: Request, project_slug: str, data: Annotated[ProjectFileWriteForm, Form()]):
+    user = require_user(request)
+    project = _load_project(project_slug)
+    if not is_owner(project, user):
+        return _deny(request, project)
+    try:
+        node = project_files.write_text_file(project["uid"], user, data.path, data.content)
+    except ProjectFileError as exc:
+        return _fail(request, project, exc)
+    logger.info("project %s file written: %s", project["uid"], node["path"])
+    return action_result(request, _files_url(project), data=project_files.node_to_dict(node))
+
+
+def _edit(request: Request, project: dict, mutate) -> JSONResponse:
+    try:
+        node = mutate()
+    except ProjectFileError as exc:
+        return _fail(request, project, exc)
+    logger.info("project %s file edited: %s", project["uid"], node["path"])
+    return action_result(request, _files_url(project), data=project_files.node_to_dict(node))
+
+
+@router.post("/{project_slug}/files/replace-lines")
+async def project_file_replace_lines(request: Request, project_slug: str, data: Annotated[ProjectFileReplaceLinesForm, Form()]):
+    user = require_user(request)
+    project = _load_project(project_slug)
+    if not is_owner(project, user):
+        return _deny(request, project)
+    return _edit(request, project, lambda: project_files.replace_lines(project["uid"], data.path, data.start, data.end, data.content))
+
+
+@router.post("/{project_slug}/files/insert-lines")
+async def project_file_insert_lines(request: Request, project_slug: str, data: Annotated[ProjectFileInsertLinesForm, Form()]):
+    user = require_user(request)
+    project = _load_project(project_slug)
+    if not is_owner(project, user):
+        return _deny(request, project)
+    return _edit(request, project, lambda: project_files.insert_lines(project["uid"], data.path, data.at, data.content))
+
+
+@router.post("/{project_slug}/files/delete-lines")
+async def project_file_delete_lines(request: Request, project_slug: str, data: Annotated[ProjectFileDeleteLinesForm, Form()]):
+    user = require_user(request)
+    project = _load_project(project_slug)
+    if not is_owner(project, user):
+        return _deny(request, project)
+    return _edit(request, project, lambda: project_files.delete_lines(project["uid"], data.path, data.start, data.end))
+
+
+@router.post("/{project_slug}/files/append")
+async def project_file_append(request: Request, project_slug: str, data: Annotated[ProjectFileAppendForm, Form()]):
+    user = require_user(request)
+    project = _load_project(project_slug)
+    if not is_owner(project, user):
+        return _deny(request, project)
+    return _edit(request, project, lambda: project_files.append_lines(project["uid"], data.path, data.content))
+
+
+@router.post("/{project_slug}/files/upload")
+async def project_file_upload(request: Request, project_slug: str):
+    user = require_user(request)
+    project = _load_project(project_slug)
+    if not is_owner(project, user):
+        return _deny(request, project)
+    form = await request.form()
+    file = form.get("file")
+    if not file or not hasattr(file, "filename") or not file.filename:
+        return _fail(request, project, ProjectFileError("no file provided"))
+    target_dir = str(form.get("path", "") or "")
+    try:
+        content = await file.read()
+        node = project_files.store_upload(project["uid"], user, target_dir, file.filename, content)
+    except ProjectFileError as exc:
+        return _fail(request, project, exc)
+    logger.info("project %s file uploaded: %s", project["uid"], node["path"])
+    return action_result(request, _files_url(project), data=project_files.node_to_dict(node))
+
+
+@router.post("/{project_slug}/files/mkdir")
+async def project_file_mkdir(request: Request, project_slug: str, data: Annotated[ProjectFileMkdirForm, Form()]):
+    user = require_user(request)
+    project = _load_project(project_slug)
+    if not is_owner(project, user):
+        return _deny(request, project)
+    try:
+        node = project_files.make_dir(project["uid"], user, data.path)
+    except ProjectFileError as exc:
+        return _fail(request, project, exc)
+    return action_result(request, _files_url(project), data=project_files.node_to_dict(node))
+
+
+@router.post("/{project_slug}/files/move")
+async def project_file_move(request: Request, project_slug: str, data: Annotated[ProjectFileMoveForm, Form()]):
+    user = require_user(request)
+    project = _load_project(project_slug)
+    if not is_owner(project, user):
+        return _deny(request, project)
+    try:
+        node = project_files.move_node(project["uid"], user, data.from_path, data.to_path)
+    except ProjectFileError as exc:
+        return _fail(request, project, exc)
+    return action_result(request, _files_url(project), data=project_files.node_to_dict(node))
+
+
+@router.post("/{project_slug}/files/delete")
+async def project_file_delete(request: Request, project_slug: str, data: Annotated[ProjectFileDeleteForm, Form()]):
+    user = require_user(request)
+    project = _load_project(project_slug)
+    if not is_owner(project, user):
+        return _deny(request, project)
+    try:
+        project_files.delete_node(project["uid"], data.path)
+    except ProjectFileError as exc:
+        return _fail(request, project, exc)
+    return action_result(request, _files_url(project), data={"path": data.path})

devplacepy/routers/projects.py

@@ -1,52 +1,58 @@
 import logging
-from datetime import datetime
-from fastapi import APIRouter, Request, HTTPException
-from fastapi.responses import HTMLResponse, RedirectResponse
-from devplacepy.database import get_table, get_vote_counts, load_comments, get_attachments, delete_attachments
+from typing import Annotated
+from sqlalchemy import or_
+from fastapi import APIRouter, Request, Form
+from devplacepy.models import ProjectForm, ProjectFlagForm, ForkForm
+from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse
+from devplacepy.database import get_table, get_users_by_uids, get_site_stats, get_user_votes, paginate, resolve_by_slug, get_fork_parent, count_forks
+from devplacepy.services.jobs import queue
+from devplacepy.content import load_detail, delete_content_item, create_content_item, detail_context, canonical_redirect, first_image_url, is_owner, can_view_project
 from devplacepy.templating import templates
-from devplacepy.utils import generate_uid, get_current_user, require_user, time_ago, make_combined_slug, create_mention_notifications
-from devplacepy.seo import base_seo_context, site_url, website_schema, breadcrumb_schema, combine, software_application_schema
+from devplacepy.utils import get_current_user, require_user, not_found, is_admin, XP_PROJECT
+from devplacepy.seo import base_seo_context, site_url, website_schema, software_application_schema, list_page_seo, next_page_url
+from devplacepy.responses import respond, action_result
+from devplacepy.schemas import ProjectsOut, ProjectDetailOut
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
 
-def get_projects_list(tab: str = "recent", search: str = "", user_uid: str = None, project_type: str = None):
+def get_projects_list(tab: str = "recent", search: str = "", user_uid: str = None, project_type: str = None, before: str = None, viewer: dict = None):
     projects = get_table("projects")
-    all_projects = list(projects.all())
 
+    filters = {}
     if user_uid:
-        all_projects = [p for p in all_projects if p["user_uid"] == user_uid]
-
+        filters["user_uid"] = user_uid
     if project_type:
-        all_projects = [p for p in all_projects if p.get("project_type") == project_type]
+        filters["project_type"] = project_type
+    if tab == "released":
+        filters["status"] = "Released"
 
-    if search:
-        search_lower = search.lower()
-        all_projects = [
-            p for p in all_projects
-            if search_lower in p.get("title", "").lower()
-            or search_lower in p.get("description", "").lower()
-        ]
+    clauses = []
+    if projects.exists:
+        columns = projects.table.columns
+        if search:
+            like = f"%{search}%"
+            clauses.append(or_(columns.title.ilike(like), columns.description.ilike(like)))
+        if not is_admin(viewer) and "is_private" in columns:
+            visible = or_(columns.is_private == None, columns.is_private == 0)
+            if viewer:
+                visible = or_(visible, columns.user_uid == viewer["uid"])
+            clauses.append(visible)
 
-    if all_projects:
-        from devplacepy.database import get_users_by_uids
-        uids = [p["user_uid"] for p in all_projects]
-        users_map = get_users_by_uids(uids)
-        for p in all_projects:
+    order = ["-stars", "-created_at"] if tab == "popular" else ["-created_at"]
+    total = projects.count(*clauses, **filters)
+    page, next_cursor = paginate(projects, *clauses, before=before, order=order, **filters)
+
+    if page:
+        users_map = get_users_by_uids([p["user_uid"] for p in page])
+        my_votes = get_user_votes(viewer["uid"], [p["uid"] for p in page]) if viewer else {}
+        for p in page:
             author = users_map.get(p["user_uid"])
             p["author_name"] = author["username"] if author else "Unknown"
+            p["my_vote"] = my_votes.get(p["uid"], 0)
 
-    if tab == "released":
-        all_projects = [p for p in all_projects if p.get("status") == "Released"]
-    elif tab == "popular":
-        all_projects.sort(key=lambda p: int(p.get("stars", 0)), reverse=True)
-    elif tab == "new":
-        all_projects.sort(key=lambda p: p.get("created_at", ""), reverse=True)
-    else:
-        all_projects.sort(key=lambda p: p.get("created_at", ""), reverse=True)
-
-    return all_projects
+    return page, next_cursor, total
 
 
 @router.get("", response_class=HTMLResponse)
@@ -56,24 +62,23 @@ async def projects_page(
     search: str = "",
     user_uid: str = None,
     project_type: str = None,
+    before: str = None,
 ):
     user = get_current_user(request)
-    projects = get_projects_list(tab, search, user_uid, project_type)
-    users = get_table("users")
-    total_members = len(list(users.all()))
+    projects, next_cursor, total_count = get_projects_list(tab, search, user_uid, project_type, before, viewer=user)
+    total_members = get_site_stats()["total_members"]
 
-    base = site_url(request)
-    seo_ctx = base_seo_context(
+    seo_ctx = list_page_seo(
         request,
         title="Projects",
-        description=f"Explore {len(projects)} developer projects on DevPlace. Games, software, mobile apps and more.",
+        description=f"Explore {total_count} developer projects on DevPlace. Games, software, mobile apps and more.",
         breadcrumbs=[
             {"name": "Home", "url": "/feed"},
             {"name": "Projects", "url": "/projects"},
         ],
-        schemas=[website_schema(base)],
+        next_url=next_page_url(request, next_cursor),
     )
-    return templates.TemplateResponse("projects.html", {
+    return respond(request, "projects.html", {
         **seo_ctx,
         "request": request,
         "user": user,
@@ -81,37 +86,31 @@ async def projects_page(
         "current_tab": tab,
         "search": search,
         "project_type": project_type,
-        "total_count": len(projects),
+        "total_count": total_count,
+        "next_cursor": next_cursor,
         "total_members": total_members,
-    })
+    }, model=ProjectsOut)
 
 
 @router.get("/{project_slug}", response_class=HTMLResponse)
 async def project_detail(request: Request, project_slug: str):
     user = get_current_user(request)
-    projects = get_table("projects")
-    project = resolve_by_slug(projects, project_slug)
-    if not project:
-        raise HTTPException(status_code=404, detail="Project not found")
-
-    from devplacepy.database import get_users_by_uids
-    users_map = get_users_by_uids([project["user_uid"]])
-    author = users_map.get(project["user_uid"])
-
-    is_owner = user and user["uid"] == project["user_uid"]
-
-    ups, downs = get_vote_counts([project["uid"]])
-    star_count = ups.get(project["uid"], 0) - downs.get(project["uid"], 0)
-
-    comments = load_comments("project", project["uid"])
-
-    project_attachments = get_attachments("project", project["uid"])
+    detail = load_detail("projects", "project", project_slug, user)
+    if not detail:
+        raise not_found("Project not found")
+    project = detail["item"]
+    if not can_view_project(project, user):
+        raise not_found("Project not found")
+    redirect = canonical_redirect("projects", project, project_slug)
+    if redirect:
+        return redirect
 
     base = site_url(request)
     seo_ctx = base_seo_context(
         request,
         title=project.get("title", "Project"),
         description=project.get("description", "")[:160],
+        og_image=first_image_url(project, detail["attachments"]),
         breadcrumbs=[
             {"name": "Home", "url": "/feed"},
             {"name": "Projects", "url": "/projects"},
@@ -119,63 +118,104 @@ async def project_detail(request: Request, project_slug: str):
         ],
         schemas=[website_schema(base), software_application_schema(project, base)],
     )
-    return templates.TemplateResponse("project_detail.html", {
-        **seo_ctx,
-        "request": request,
-        "user": user,
-        "project": project,
-        "author": author,
-        "is_owner": is_owner,
-        "star_count": star_count,
+    parent = get_fork_parent(project["uid"])
+    forked_from = {
+        "uid": parent["uid"],
+        "slug": parent.get("slug") or parent["uid"],
+        "title": parent.get("title") or "project",
+    } if parent else None
+    return respond(request, "project_detail.html", detail_context(request, user, detail, "project", seo_ctx, {
         "platforms": project.get("platforms", "").split(",") if project.get("platforms") else [],
-        "comments": comments,
-        "attachments": project_attachments,
-    })
+        "is_private": bool(project.get("is_private")),
+        "read_only": bool(project.get("read_only")),
+        "forked_from": forked_from,
+        "fork_count": count_forks(project["uid"]),
+    }), model=ProjectDetailOut)
+
+
+@router.post("/{project_slug}/zip")
+async def zip_project(request: Request, project_slug: str):
+    project = resolve_by_slug(get_table("projects"), project_slug)
+    if not project:
+        raise not_found("Project not found")
+    user = get_current_user(request)
+    if not can_view_project(project, user):
+        raise not_found("Project not found")
+    owner_kind, owner_id = ("user", user["uid"]) if user else ("guest", request.client.host or "anon")
+    uid = queue.enqueue(
+        "zip",
+        {"source": {"type": "project_tree", "project_uid": project["uid"], "path": ""}},
+        owner_kind, owner_id,
+        project.get("title") or "project",
+    )
+    return JSONResponse({"uid": uid, "status_url": f"/zips/{uid}"})
+
+
+@router.post("/{project_slug}/fork")
+async def fork_project(request: Request, project_slug: str, data: Annotated[ForkForm, Form()]):
+    user = require_user(request)
+    source = resolve_by_slug(get_table("projects"), project_slug)
+    if not source:
+        raise not_found("Project not found")
+    if not can_view_project(source, user):
+        raise not_found("Project not found")
+    title = data.title.strip()
+    uid = queue.enqueue(
+        "fork",
+        {"source_project_uid": source["uid"], "title": title, "forked_by_uid": user["uid"]},
+        "user", user["uid"],
+        title,
+    )
+    return JSONResponse({"uid": uid, "status_url": f"/forks/{uid}"})
 
 
 @router.post("/delete/{project_slug}")
 async def delete_project(request: Request, project_slug: str):
     user = require_user(request)
-    projects = get_table("projects")
-    project = resolve_by_slug(projects, project_slug)
-    if project and project["user_uid"] == user["uid"]:
-        delete_attachments("project", project["uid"])
-        projects.delete(id=project["id"])
-        logger.info(f"Project {project['uid']} deleted by {user['username']}")
-    return RedirectResponse(url="/projects", status_code=302)
+    return delete_content_item(request, "projects", "project", user, project_slug, "/projects")
 
 
 @router.post("/create")
-async def create_project(request: Request):
+async def create_project(request: Request, data: Annotated[ProjectForm, Form()]):
     user = require_user(request)
-    form = await request.form()
-    title = form.get("title", "").strip()
-    description = form.get("description", "").strip()
-    release_date = form.get("release_date", "")
-    demo_date = form.get("demo_date", "")
-    project_type = form.get("project_type", "software")
-    platforms = form.get("platforms", "").strip()
-    status = form.get("status", "In Development")
+    title = data.title.strip()
+    description = data.description.strip()
 
-    if not title or not description:
-        return RedirectResponse(url="/projects", status_code=302)
-
-    projects = get_table("projects")
-    uid = generate_uid()
-    project_slug = make_combined_slug(title, uid)
-    projects.insert({
-        "uid": uid,
-        "user_uid": user["uid"],
+    uid, project_slug = create_content_item("projects", "project", user, {
         "title": title,
-        "slug": project_slug,
         "description": description,
-        "release_date": release_date or None,
-        "demo_date": demo_date or None,
-        "project_type": project_type,
-        "platforms": platforms,
-        "status": status,
-        "stars": 0,
-        "created_at": datetime.utcnow().isoformat(),
-    })
+        "release_date": data.release_date or None,
+        "demo_date": data.demo_date or None,
+        "project_type": data.project_type,
+        "platforms": data.platforms.strip(),
+        "status": data.status,
+        "is_private": 1 if data.is_private else 0,
+        "read_only": 0,
+    }, title, XP_PROJECT, "First Project", description, data.attachment_uids)
+    url = f"/projects/{project_slug}"
+    return action_result(request, url, data={"uid": uid, "slug": project_slug, "url": url})
 
-    attachment_uids = form.getlist("attachment_uids") if hasattr(form, "getlist") else []
+
+def _set_project_flag(request: Request, project_slug: str, field: str, value: bool):
+    user = require_user(request)
+    projects = get_table("projects")
+    project = resolve_by_slug(projects, project_slug)
+    if not is_owner(project, user):
+        from devplacepy.responses import wants_json, json_error
+        if wants_json(request):
+            return json_error(403, "Not allowed")
+        return RedirectResponse(url=f"/projects/{project_slug}", status_code=302)
+    projects.update({"uid": project["uid"], field: 1 if value else 0}, ["uid"])
+    logger.info("project %s %s set to %s by %s", project["uid"], field, bool(value), user["username"])
+    url = f"/projects/{project['slug'] or project['uid']}"
+    return action_result(request, url, data={"uid": project["uid"], field: bool(value), "url": url})
+
+
+@router.post("/{project_slug}/private")
+async def set_project_private(request: Request, project_slug: str, data: Annotated[ProjectFlagForm, Form()]):
+    return _set_project_flag(request, project_slug, "is_private", data.value)
+
+
+@router.post("/{project_slug}/readonly")
+async def set_project_readonly(request: Request, project_slug: str, data: Annotated[ProjectFlagForm, Form()]):
+    return _set_project_flag(request, project_slug, "read_only", data.value)

devplacepy/routers/proxy.py

@@ -0,0 +1,157 @@
+# retoor <retoor@molodetz.nl>
+
+import asyncio
+import json
+import logging
+
+import httpx
+import websockets
+from fastapi import APIRouter, Request, WebSocket
+from starlette.responses import Response
+
+from devplacepy import config
+from devplacepy.services.containers import store
+from devplacepy.utils import not_found
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+HOP_HEADERS = {
+    "connection", "keep-alive", "proxy-authenticate", "proxy-authorization",
+    "te", "trailers", "transfer-encoding", "upgrade", "host", "content-length",
+    "content-encoding",
+}
+METHODS = ["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "HEAD"]
+
+
+def _forward_headers(request: Request, prefix: str) -> dict:
+    headers = {k: v for k, v in request.headers.items() if k.lower() not in HOP_HEADERS}
+    headers["X-Forwarded-Prefix"] = prefix
+    headers["X-Script-Name"] = prefix
+    headers["X-Forwarded-Host"] = request.headers.get("host", request.url.hostname or "")
+    headers["X-Forwarded-Proto"] = request.headers.get("x-forwarded-proto", request.url.scheme)
+    headers["Accept-Encoding"] = "identity"
+    return headers
+
+
+def _inject_base(body: bytes, prefix: str) -> bytes:
+    lowered = body.lower()
+    if b"<base" in lowered:
+        return body
+    tag = f'<base href="{prefix}/">'.encode()
+    head = lowered.find(b"<head")
+    anchor = lowered.find(b">", head) if head != -1 else lowered.find(b">", lowered.find(b"<html"))
+    if anchor == -1:
+        return tag + body
+    return body[: anchor + 1] + tag + body[anchor + 1 :]
+
+
+def _rewrite_location(value: str, prefix: str) -> str:
+    if value.startswith("/") and not value.startswith("//"):
+        return prefix + value
+    return value
+
+
+def _resolve(slug: str):
+    instance = store.find_instance_by_ingress(slug)
+    if not instance or instance.get("status") != store.ST_RUNNING:
+        return None, None
+    ports = json.loads(instance.get("ports_json") or "[]")
+    if not ports:
+        return instance, None
+    ingress_port = int(instance.get("ingress_port") or 0)
+    if ingress_port:
+        for port in ports:
+            if int(port["container"]) == ingress_port:
+                return instance, int(port["host"])
+        return instance, None
+    return instance, int(ports[0]["host"])
+
+
+@router.api_route("/{slug}", methods=METHODS)
+@router.api_route("/{slug}/{path:path}", methods=METHODS)
+async def proxy_http(request: Request, slug: str, path: str = ""):
+    instance, port = _resolve(slug)
+    if instance is None:
+        raise not_found("No running container is published at this address")
+    if port is None:
+        return Response("the published container has no reachable port", status_code=502)
+    prefix = f"/p/{slug}"
+    url = f"http://{config.CONTAINER_PROXY_HOST}:{port}/{path}"
+    headers = _forward_headers(request, prefix)
+    body = await request.body()
+    try:
+        async with httpx.AsyncClient(timeout=60.0, follow_redirects=False) as client:
+            upstream = await client.request(
+                request.method, url, params=request.query_params, headers=headers, content=body)
+    except httpx.HTTPError as exc:
+        return Response(f"upstream error: {exc}", status_code=502)
+    out_headers = {k: v for k, v in upstream.headers.items()
+                   if k.lower() not in HOP_HEADERS and k.lower() != "set-cookie"}
+    if "location" in out_headers:
+        out_headers["location"] = _rewrite_location(out_headers["location"], prefix)
+    content_type = upstream.headers.get("content-type", "")
+    content = upstream.content
+    if "text/html" in content_type.lower():
+        content = _inject_base(content, prefix)
+    response = Response(content=content, status_code=upstream.status_code,
+                        headers=out_headers, media_type=content_type or None)
+    for cookie in upstream.headers.get_list("set-cookie"):
+        response.headers.append("set-cookie", cookie)
+    return response
+
+
+@router.websocket("/{slug}")
+@router.websocket("/{slug}/{path:path}")
+async def proxy_ws(websocket: WebSocket, slug: str, path: str = ""):
+    instance, port = _resolve(slug)
+    if instance is None or port is None:
+        await websocket.close(code=1011)
+        return
+    upstream_url = f"ws://{config.CONTAINER_PROXY_HOST}:{port}/{path}"
+    if websocket.url.query:
+        upstream_url += f"?{websocket.url.query}"
+    await websocket.accept()
+    try:
+        async with websockets.connect(upstream_url, open_timeout=10, max_size=None) as upstream:
+            await _pump(websocket, upstream)
+    except Exception as exc:
+        logger.debug("ws proxy %s failed: %s", slug, exc)
+        try:
+            await websocket.close(code=1011)
+        except Exception:
+            pass
+
+
+async def _pump(client_ws: WebSocket, upstream) -> None:
+    async def client_to_upstream():
+        try:
+            while True:
+                message = await client_ws.receive()
+                if message["type"] == "websocket.disconnect":
+                    break
+                if message.get("text") is not None:
+                    await upstream.send(message["text"])
+                elif message.get("bytes") is not None:
+                    await upstream.send(message["bytes"])
+        except Exception:
+            pass
+        finally:
+            await upstream.close()
+
+    async def upstream_to_client():
+        try:
+            async for message in upstream:
+                if isinstance(message, (bytes, bytearray)):
+                    await client_ws.send_bytes(bytes(message))
+                else:
+                    await client_ws.send_text(message)
+        except Exception:
+            pass
+        finally:
+            try:
+                await client_ws.close()
+            except Exception:
+                pass
+
+    await asyncio.gather(client_to_upstream(), upstream_to_client())

devplacepy/routers/push.py

@@ -0,0 +1,65 @@
+# retoor <retoor@molodetz.nl>
+
+import logging
+from fastapi import APIRouter, Request
+from fastapi.responses import FileResponse, JSONResponse
+from devplacepy import push
+from devplacepy.config import STATIC_DIR
+from devplacepy.utils import require_user_api
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+WELCOME_PAYLOAD = {
+    "title": "DevPlace",
+    "message": "Push notifications enabled.",
+    "icon": "/static/apple-touch-icon.png",
+    "url": "/notifications",
+}
+
+
+@router.get("/push.json")
+async def push_public_key() -> JSONResponse:
+    return JSONResponse({"publicKey": push.public_key_standard_b64()})
+
+
+@router.post("/push.json")
+async def push_register(request: Request) -> JSONResponse:
+    user = require_user_api(request)
+    try:
+        body = await request.json()
+    except ValueError:
+        return JSONResponse({"error": "Invalid JSON"}, status_code=400)
+
+    keys = body.get("keys") if isinstance(body, dict) else None
+    if not (isinstance(keys, dict) and body.get("endpoint") and keys.get("p256dh") and keys.get("auth")):
+        return JSONResponse({"error": "Invalid request"}, status_code=400)
+
+    _, created = await push.register(
+        user_uid=user["uid"],
+        endpoint=body["endpoint"],
+        key_auth=keys["auth"],
+        key_p256dh=keys["p256dh"],
+    )
+
+    if created:
+        try:
+            await push.notify_user(user["uid"], WELCOME_PAYLOAD)
+        except Exception as exc:
+            logger.warning("Welcome push failed for %s: %s", user["uid"], exc)
+
+    return JSONResponse({"registered": True})
+
+
+@router.get("/service-worker.js")
+async def service_worker() -> FileResponse:
+    return FileResponse(
+        STATIC_DIR / "service-worker.js",
+        media_type="application/javascript",
+        headers={"Service-Worker-Allowed": "/", "Cache-Control": "no-cache"},
+    )
+
+
+@router.get("/manifest.json")
+async def manifest() -> FileResponse:
+    return FileResponse(STATIC_DIR / "manifest.json", media_type="application/manifest+json")

devplacepy/routers/reactions.py

@@ -0,0 +1,46 @@
+# retoor <retoor@molodetz.nl>
+import logging
+from typing import Annotated
+from datetime import datetime, timezone
+from fastapi import APIRouter, Request, Form
+from fastapi.responses import RedirectResponse, JSONResponse
+from devplacepy.database import get_table, db
+from devplacepy.utils import generate_uid, require_user
+from devplacepy.models import ReactionForm
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+REACTABLE: set[str] = {"post", "comment", "gist", "project"}
+
+
+@router.post("/{target_type}/{target_uid}")
+async def react(request: Request, target_type: str, target_uid: str, data: Annotated[ReactionForm, Form()]):
+    user = require_user(request)
+    if target_type not in REACTABLE:
+        return JSONResponse({"error": "Invalid target"}, status_code=400)
+
+    emoji = data.emoji
+    reactions = get_table("reactions")
+    existing = reactions.find_one(user_uid=user["uid"], target_uid=target_uid, target_type=target_type, emoji=emoji)
+    if existing:
+        reactions.delete(id=existing["id"])
+    else:
+        reactions.insert({
+            "uid": generate_uid(),
+            "user_uid": user["uid"],
+            "target_uid": target_uid,
+            "target_type": target_type,
+            "emoji": emoji,
+            "created_at": datetime.now(timezone.utc).isoformat(),
+        })
+
+    counts = {row["emoji"]: row["c"] for row in db.query(
+        "SELECT emoji, COUNT(*) as c FROM reactions WHERE target_type=:tt AND target_uid=:tu GROUP BY emoji",
+        tt=target_type, tu=target_uid,
+    )}
+    mine = [row["emoji"] for row in reactions.find(user_uid=user["uid"], target_uid=target_uid, target_type=target_type)]
+
+    if request.headers.get("x-requested-with") == "fetch":
+        return JSONResponse({"counts": counts, "mine": mine})
+    return RedirectResponse(url=request.headers.get("Referer", "/feed"), status_code=302)

devplacepy/routers/seo.py

@@ -1,6 +1,6 @@
 import logging
 from fastapi import APIRouter, Request
-from fastapi.responses import HTMLResponse, PlainTextResponse
+from fastapi.responses import PlainTextResponse, Response
 from devplacepy.seo import make_sitemap, site_url
 
 logger = logging.getLogger(__name__)
@@ -17,16 +17,18 @@ Disallow: /notifications/
 Disallow: /votes/
 Disallow: /avatar/
 Disallow: /follow/
+Disallow: /admin/
+Disallow: /uploads/
 Disallow: /*?tab=
 Disallow: /*?sort=
 Allow: /static/
 
 Sitemap: {base}/sitemap.xml
-""")
+""", headers={"Cache-Control": "public, max-age=3600"})
 
 
-@router.get("/sitemap.xml", response_class=HTMLResponse)
+@router.get("/sitemap.xml")
 async def sitemap_xml(request: Request):
     base = site_url(request)
     xml = make_sitemap(base)
-    return HTMLResponse(content=xml, media_type="application/xml")
+    return Response(content=xml, media_type="application/xml", headers={"Cache-Control": "public, max-age=3600"})

devplacepy/routers/services.py

@@ -2,7 +2,7 @@ import logging
 from fastapi import APIRouter, Request
 from fastapi.responses import HTMLResponse, JSONResponse
 from devplacepy.templating import templates
-from devplacepy.utils import require_admin
+from devplacepy.utils import require_admin, not_found
 from devplacepy.services.manager import service_manager
 from devplacepy.seo import base_seo_context, site_url, website_schema
 
@@ -13,12 +13,12 @@ router = APIRouter()
 @router.get("", response_class=HTMLResponse)
 async def services_page(request: Request):
     admin = require_admin(request)
-    services = service_manager.list_services()
+    services = service_manager.describe_all()
     base = site_url(request)
     seo_ctx = base_seo_context(
         request,
-        title="Services — Admin",
-        description="Monitor background services on DevPlace.",
+        title="Services - Admin",
+        description="Monitor and configure background services on DevPlace.",
         breadcrumbs=[
             {"name": "Home", "url": "/feed"},
             {"name": "Admin", "url": "/admin"},
@@ -26,7 +26,7 @@ async def services_page(request: Request):
         ],
         schemas=[website_schema(base)],
     )
-    return templates.TemplateResponse("services.html", {
+    return templates.TemplateResponse(request, "services.html", {
         **seo_ctx,
         "request": request,
         "user": admin,
@@ -38,5 +38,87 @@ async def services_page(request: Request):
 @router.get("/data")
 async def services_data(request: Request):
     require_admin(request)
-    services = service_manager.list_services()
-    return JSONResponse({"services": services})
+    return JSONResponse({"services": service_manager.describe_all()})
+
+
+@router.get("/{name}", response_class=HTMLResponse)
+async def service_detail(request: Request, name: str):
+    admin = require_admin(request)
+    svc = service_manager.get_service(name)
+    if svc is None:
+        raise not_found("Service not found")
+    info = svc.describe()
+    base = site_url(request)
+    seo_ctx = base_seo_context(
+        request,
+        title=f"{info['title']} - Services",
+        description=info["description"] or f"Configure the {info['title']} service.",
+        robots="noindex",
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {"name": "Admin", "url": "/admin"},
+            {"name": "Services", "url": "/admin/services"},
+            {"name": info["title"], "url": f"/admin/services/{name}"},
+        ],
+        schemas=[website_schema(base)],
+    )
+    return templates.TemplateResponse(request, "service_detail.html", {
+        **seo_ctx,
+        "request": request,
+        "user": admin,
+        "service": info,
+        "admin_section": "services",
+    })
+
+
+@router.get("/{name}/data")
+async def service_detail_data(request: Request, name: str):
+    require_admin(request)
+    svc = service_manager.get_service(name)
+    if svc is None:
+        return JSONResponse({"error": "unknown service"}, status_code=404)
+    return JSONResponse({"service": svc.describe()})
+
+
+@router.post("/{name}/start")
+async def service_start(request: Request, name: str):
+    require_admin(request)
+    if not service_manager.set_enabled(name, True):
+        return JSONResponse({"ok": False, "error": "unknown service"}, status_code=404)
+    return JSONResponse({"ok": True})
+
+
+@router.post("/{name}/stop")
+async def service_stop(request: Request, name: str):
+    require_admin(request)
+    if not service_manager.set_enabled(name, False):
+        return JSONResponse({"ok": False, "error": "unknown service"}, status_code=404)
+    return JSONResponse({"ok": True})
+
+
+@router.post("/{name}/run")
+async def service_run(request: Request, name: str):
+    require_admin(request)
+    if not service_manager.send_command(name, "run"):
+        return JSONResponse({"ok": False, "error": "unknown service"}, status_code=404)
+    return JSONResponse({"ok": True})
+
+
+@router.post("/{name}/clear-logs")
+async def service_clear_logs(request: Request, name: str):
+    require_admin(request)
+    if not service_manager.send_command(name, "clear"):
+        return JSONResponse({"ok": False, "error": "unknown service"}, status_code=404)
+    return JSONResponse({"ok": True})
+
+
+@router.post("/{name}/config")
+async def service_config(request: Request, name: str):
+    require_admin(request)
+    form = await request.form()
+    result = service_manager.save_config(name, dict(form))
+    if result is None:
+        return JSONResponse({"ok": False, "error": "unknown service"}, status_code=404)
+    if not result["ok"]:
+        return JSONResponse(result, status_code=400)
+    return JSONResponse(result)

devplacepy/routers/uploads.py

@@ -1,54 +1,27 @@
-import hashlib
 import logging
-import aiofiles
 from pathlib import Path
 from fastapi import APIRouter, Request
 from fastapi.responses import JSONResponse
-from devplacepy.database import get_table, get_setting
-from devplacepy.config import STATIC_DIR
-from devplacepy.utils import generate_uid, require_user
+from devplacepy.database import get_table, get_int_setting
+from devplacepy.utils import require_user_api
+from devplacepy.attachments import store_attachment, delete_attachment as _delete_attachment, is_extension_allowed
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
-IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".bmp"}
-
-
-def _store_file(content: bytes, original_filename: str) -> tuple[str, str]:
-    uid = generate_uid()
-    ext = Path(original_filename).suffix.lower() or ""
-    stored_name = f"{uid}{ext}"
-    hash_str = hashlib.sha256(stored_name.encode()).hexdigest()
-    subdir = f"{hash_str[:2]}/{hash_str[2:4]}"
-    storage_path = f"{subdir}/{stored_name}"
-    full_dir = STATIC_DIR / "uploads" / subdir
-    full_dir.mkdir(parents=True, exist_ok=True)
-    full_path = full_dir / stored_name
-    with open(str(full_path), "wb") as f:
-        f.write(content)
-    return uid, storage_path
-
 
 @router.post("/upload")
 async def upload_file(request: Request):
-    user = require_user(request)
+    user = require_user_api(request)
     form = await request.form()
     file = form.get("file")
 
     if not file or not hasattr(file, "filename") or not file.filename:
         return JSONResponse({"error": "No file provided"}, status_code=400)
 
-    max_size_mb = int(get_setting("max_upload_size_mb", "10"))
-    max_size_bytes = max_size_mb * 1024 * 1024
-    allowed_types_raw = get_setting("allowed_file_types", "").strip()
-    allowed_extensions = set()
-    if allowed_types_raw:
-        for ext in allowed_types_raw.split(","):
-            ext = ext.strip().lower()
-            if ext.startswith("."):
-                allowed_extensions.add(ext)
-            else:
-                allowed_extensions.add(f".{ext}")
+    ext = Path(file.filename).suffix.lower()
+    if not is_extension_allowed(ext):
+        return JSONResponse({"error": f"File type '{ext}' not allowed"}, status_code=415)
 
     try:
         content = await file.read()
@@ -56,61 +29,23 @@ async def upload_file(request: Request):
         logger.warning(f"Failed to read uploaded file: {e}")
         return JSONResponse({"error": "Failed to read file"}, status_code=400)
 
-    if len(content) > max_size_bytes:
+    result = store_attachment(content, file.filename, user["uid"])
+    if result is None:
+        max_size_mb = get_int_setting("max_upload_size_mb", 10)
         return JSONResponse({"error": f"File exceeds {max_size_mb}MB limit"}, status_code=413)
 
-    ext = Path(file.filename).suffix.lower()
-    if allowed_extensions and ext not in allowed_extensions:
-        return JSONResponse({"error": f"File type '{ext}' not allowed"}, status_code=415)
-
-    uid, storage_path = _store_file(content, file.filename)
-
-    attachments = get_table("attachments")
-    attachments.insert({
-        "uid": uid,
-        "resource_uid": "",
-        "resource_type": "",
-        "original_filename": file.filename,
-        "stored_filename": f"{uid}{ext}",
-        "mime_type": file.content_type or "application/octet-stream",
-        "file_size": len(content),
-        "storage_path": storage_path,
-        "created_at": __import__("datetime").datetime.utcnow().isoformat(),
-    })
-
-    is_image = ext in IMAGE_EXTENSIONS
-    file_url = f"/static/uploads/{storage_path}"
-
-    logger.info(f"File uploaded: {file.filename} ({len(content)} bytes) -> {storage_path}")
-    return JSONResponse({
-        "uid": uid,
-        "original_filename": file.filename,
-        "url": file_url,
-        "mime_type": file.content_type or "application/octet-stream",
-        "file_size": len(content),
-        "is_image": is_image,
-    }, status_code=201)
+    logger.info(f"File uploaded: {file.filename} ({len(content)} bytes) -> {result['url']}")
+    return JSONResponse(result, status_code=201)
 
 
 @router.delete("/delete/{attachment_uid}")
-async def delete_attachment(request: Request, attachment_uid: str):
-    user = require_user(request)
-    attachments = get_table("attachments")
-    att = attachments.find_one(uid=attachment_uid)
+async def delete_attachment_route(request: Request, attachment_uid: str):
+    user = require_user_api(request)
+    att = get_table("attachments").find_one(uid=attachment_uid)
     if not att:
         return JSONResponse({"error": "Attachment not found"}, status_code=404)
-    if att.get("resource_uid"):
-        resource_type = att["resource_type"]
-        if resource_type == "post":
-            post = get_table("posts").find_one(uid=att["resource_uid"])
-            if not post or post["user_uid"] != user["uid"]:
-                return JSONResponse({"error": "Not authorized"}, status_code=403)
-        elif resource_type == "comment":
-            comment = get_table("comments").find_one(uid=att["resource_uid"])
-            if not comment or comment["user_uid"] != user["uid"]:
-                return JSONResponse({"error": "Not authorized"}, status_code=403)
-    from devplacepy.database import _delete_attachment_file
-    _delete_attachment_file(att.get("storage_path", ""))
-    attachments.delete(id=att["id"])
+    if att.get("user_uid") and att["user_uid"] != user["uid"]:
+        return JSONResponse({"error": "Not authorized"}, status_code=403)
+    _delete_attachment(attachment_uid)
     logger.info(f"Attachment {attachment_uid} deleted by {user['username']}")
     return JSONResponse({"status": "deleted"})

devplacepy/routers/votes.py

@@ -1,31 +1,33 @@
 import logging
-from datetime import datetime
-from fastapi import APIRouter, Request
-from fastapi.responses import RedirectResponse
-from devplacepy.database import get_table
-from devplacepy.utils import generate_uid, require_user
+from typing import Annotated
+from datetime import datetime, timezone
+from fastapi import APIRouter, Request, Form
+from fastapi.responses import RedirectResponse, JSONResponse
+from devplacepy.database import get_table, update_target_stars, get_target_owner_uid, resolve_object_url
+from devplacepy.utils import generate_uid, require_user, create_notification, award_rewards, XP_UPVOTE
+from devplacepy.models import VoteForm
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
+NOTIFY_ON_VOTE: set[str] = {"post", "comment", "gist", "project"}
+
 
 @router.post("/{target_type}/{target_uid}")
-async def vote(request: Request, target_type: str, target_uid: str):
+async def vote(request: Request, target_type: str, target_uid: str, data: Annotated[VoteForm, Form()]):
     user = require_user(request)
-    form = await request.form()
-    value = int(form.get("value", "1"))
-
-    if value not in (1, -1):
-        return RedirectResponse(url="/feed", status_code=302)
+    value = data.value
 
     votes = get_table("votes")
     existing = votes.find_one(user_uid=user["uid"], target_uid=target_uid, target_type=target_type)
 
+    did_upvote = False
     if existing:
         if int(existing["value"]) == value:
             votes.delete(id=existing["id"])
         else:
             votes.update({"id": existing["id"], "value": value}, ["id"])
+            did_upvote = value == 1
     else:
         votes.insert({
             "uid": generate_uid(),
@@ -33,52 +35,28 @@ async def vote(request: Request, target_type: str, target_uid: str):
             "target_uid": target_uid,
             "target_type": target_type,
             "value": value,
-            "created_at": datetime.utcnow().isoformat(),
+            "created_at": datetime.now(timezone.utc).isoformat(),
         })
+        did_upvote = value == 1
 
-    up_count = len(list(votes.find(target_uid=target_uid, value=1)))
-    down_count = len(list(votes.find(target_uid=target_uid, value=-1)))
+    up_count = votes.count(target_uid=target_uid, target_type=target_type, value=1)
+    down_count = votes.count(target_uid=target_uid, target_type=target_type, value=-1)
     net = up_count - down_count
 
-    if target_type == "post":
-        posts = get_table("posts")
-        posts.update({"uid": target_uid, "stars": net}, ["uid"])
-    elif target_type == "project":
-        projects = get_table("projects")
-        projects.update({"uid": target_uid, "stars": net}, ["uid"])
-    elif target_type == "gist":
-        gists = get_table("gists")
-        gists.update({"uid": target_uid, "stars": net}, ["uid"])
+    update_target_stars(target_type, target_uid, net)
 
-    if value == 1:
-        target_owner_uid = None
-        if target_type == "post":
-            target_owner = posts.find_one(uid=target_uid)
-            if target_owner:
-                target_owner_uid = target_owner["user_uid"]
-        elif target_type == "comment":
-            comments = get_table("comments")
-            target_comment = comments.find_one(uid=target_uid)
-            if target_comment:
-                target_owner_uid = target_comment["user_uid"]
-        elif target_type == "gist":
-            gists = get_table("gists")
-            target_gist = gists.find_one(uid=target_uid)
-            if target_gist:
-                target_owner_uid = target_gist["user_uid"]
+    if did_upvote and target_type in NOTIFY_ON_VOTE:
+        owner_uid = get_target_owner_uid(target_type, target_uid)
+        if owner_uid and owner_uid != user["uid"]:
+            target_url = resolve_object_url(target_type, target_uid)
+            create_notification(owner_uid, "vote", f"{user['username']} ++'d your {target_type}", user["uid"], target_url)
+            award_rewards(owner_uid, XP_UPVOTE)
 
-        if target_owner_uid and target_owner_uid != user["uid"]:
-            label = "post" if target_type == "post" else "comment"
-            notifications = get_table("notifications")
-            notifications.insert({
-                "uid": generate_uid(),
-                "user_uid": target_owner_uid,
-                "type": "vote",
-                "message": f"{user['username']} ++'d your {label}",
-                "related_uid": user["uid"],
-                "read": False,
-                "created_at": datetime.utcnow().isoformat(),
-            })
+    if request.headers.get("x-requested-with") == "fetch":
+        current = votes.find_one(user_uid=user["uid"], target_uid=target_uid, target_type=target_type)
+        current_value = int(current["value"]) if current else 0
+        logger.debug("ajax vote response target=%s/%s net=%s value=%s", target_type, target_uid, net, current_value)
+        return JSONResponse({"net": net, "up": up_count, "down": down_count, "value": current_value})
 
     referer = request.headers.get("Referer", "/feed")
     return RedirectResponse(url=referer, status_code=302)

devplacepy/routers/zips.py

@@ -0,0 +1,59 @@
+# retoor <retoor@molodetz.nl>
+
+import logging
+from pathlib import Path
+
+from fastapi import APIRouter, Request
+from fastapi.responses import FileResponse, JSONResponse
+
+from devplacepy.services.jobs import queue
+from devplacepy.services.jobs.zip_service import ZIPS_DIR
+from devplacepy.schemas import ZipJobOut
+from devplacepy.utils import not_found
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+RETENTION_EXTEND_SECONDS = 7 * 24 * 60 * 60
+
+
+def _status_payload(job: dict) -> dict:
+    result = job.get("result", {})
+    done = job.get("status") == queue.DONE
+    return {
+        "uid": job.get("uid", ""),
+        "kind": job.get("kind", ""),
+        "status": job.get("status", ""),
+        "preferred_name": job.get("preferred_name"),
+        "download_url": result.get("download_url") if done else None,
+        "error": job.get("error") or None,
+        "bytes_in": int(job.get("bytes_in") or 0),
+        "bytes_out": int(job.get("bytes_out") or 0),
+        "item_count": int(job.get("item_count") or 0),
+        "file_count": int(result.get("file_count") or 0),
+        "dir_count": int(result.get("dir_count") or 0),
+        "created_at": job.get("created_at"),
+        "completed_at": job.get("completed_at") or None,
+    }
+
+
+@router.get("/{uid}")
+async def zip_status(request: Request, uid: str):
+    job = queue.get_job(uid)
+    if not job or job.get("kind") != "zip":
+        raise not_found("Job not found")
+    return JSONResponse(ZipJobOut.model_validate(_status_payload(job)).model_dump(mode="json"))
+
+
+@router.get("/{uid}/download")
+async def zip_download(request: Request, uid: str):
+    job = queue.get_job(uid)
+    if not job or job.get("kind") != "zip" or job.get("status") != queue.DONE:
+        raise not_found("Archive not available")
+    result = job.get("result", {})
+    local_path = result.get("local_path", "")
+    resolved = Path(local_path).resolve()
+    if not local_path or not resolved.is_relative_to(ZIPS_DIR.resolve()) or not resolved.is_file():
+        raise not_found("Archive not available")
+    queue.touch_job(uid, RETENTION_EXTEND_SECONDS)
+    return FileResponse(resolved, filename=result.get("final_name", "download.zip"), media_type="application/zip")

devplacepy/schemas.py

@@ -0,0 +1,611 @@
+"""Pydantic response models for JSON content negotiation.
+
+Every model uses ``extra="ignore"`` so it can be validated directly against the
+dict context a template already receives, dropping internal keys (request, SEO)
+and - importantly - sensitive user fields (email, api_key, password_hash) that
+are never part of ``UserOut``.
+"""
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from pydantic import BaseModel, ConfigDict
+
+
+class _Out(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+
+# ---------- envelopes ----------
+
+class ActionResult(_Out):
+    ok: bool = True
+    redirect: Optional[str] = None
+    data: Optional[Any] = None
+
+
+class ErrorOut(_Out):
+    error: dict
+
+
+class ValidationErrorOut(_Out):
+    error: str = "validation"
+    fields: dict = {}
+    messages: list = []
+
+
+class InstanceOut(_Out):
+    uid: str = ""
+    project_uid: str = ""
+    name: str = ""
+    slug: str = ""
+    container_id: Optional[str] = None
+    status: Optional[str] = None
+    desired_state: Optional[str] = None
+    restart_policy: Optional[str] = None
+    restart_count: int = 0
+    ingress_slug: Optional[str] = None
+    ingress_port: int = 0
+    boot_command: Optional[str] = None
+    cpu_limit: Optional[str] = None
+    mem_limit: Optional[str] = None
+    created_at: Optional[str] = None
+    started_at: Optional[str] = None
+    stopped_at: Optional[str] = None
+
+
+class ScheduleOut(_Out):
+    uid: str = ""
+    instance_uid: str = ""
+    action: Optional[str] = None
+    enabled: int = 0
+    next_run_at: Optional[str] = None
+    last_run_at: Optional[str] = None
+    run_count: int = 0
+
+
+class ContainersOut(_Out):
+    project: Optional[dict] = None
+    instances: list[InstanceOut] = []
+
+
+class ZipJobOut(_Out):
+    uid: str = ""
+    kind: str = ""
+    status: str = ""
+    preferred_name: Optional[str] = None
+    download_url: Optional[str] = None
+    error: Optional[str] = None
+    bytes_in: int = 0
+    bytes_out: int = 0
+    item_count: int = 0
+    file_count: int = 0
+    dir_count: int = 0
+    created_at: Optional[str] = None
+    completed_at: Optional[str] = None
+
+
+class ForkJobOut(_Out):
+    uid: str = ""
+    kind: str = ""
+    status: str = ""
+    preferred_name: Optional[str] = None
+    project_uid: Optional[str] = None
+    project_url: Optional[str] = None
+    source_project_uid: Optional[str] = None
+    error: Optional[str] = None
+    item_count: int = 0
+    created_at: Optional[str] = None
+    completed_at: Optional[str] = None
+
+
+# ---------- shared resource projections ----------
+
+class UserOut(_Out):
+    uid: str = ""
+    username: str = ""
+    avatar_seed: Optional[str] = None
+    role: Optional[str] = None
+    bio: Optional[str] = None
+    location: Optional[str] = None
+    git_link: Optional[str] = None
+    website: Optional[str] = None
+    level: Optional[int] = None
+    xp: Optional[int] = None
+    stars: Optional[int] = None
+    created_at: Optional[str] = None
+
+
+class AttachmentOut(_Out):
+    uid: str = ""
+    filename: Optional[str] = None
+    url: Optional[str] = None
+    size: Optional[int] = None
+    is_image: Optional[bool] = None
+    is_video: Optional[bool] = None
+    mime_type: Optional[str] = None
+    created_at: Optional[str] = None
+
+
+class VotesOut(_Out):
+    up: int = 0
+    down: int = 0
+
+
+class ReactionsOut(_Out):
+    counts: dict = {}
+    mine: list = []
+
+
+class PollOptionOut(_Out):
+    uid: str = ""
+    label: Optional[str] = None
+    count: Optional[int] = None
+    votes: Optional[int] = None
+    pct: Optional[int] = None
+
+
+class PollOut(_Out):
+    uid: str = ""
+    question: Optional[str] = None
+    options: list[PollOptionOut] = []
+    total: int = 0
+    my_choice: Optional[str] = None
+    voted: Optional[str] = None
+
+
+class BadgeOut(_Out):
+    name: Optional[str] = None
+    created_at: Optional[str] = None
+
+
+class PostOut(_Out):
+    uid: str = ""
+    slug: Optional[str] = None
+    user_uid: Optional[str] = None
+    title: Optional[str] = None
+    content: Optional[str] = None
+    topic: Optional[str] = None
+    stars: Optional[int] = None
+    image: Optional[str] = None
+    created_at: Optional[str] = None
+    updated_at: Optional[str] = None
+
+
+class GistOut(_Out):
+    uid: str = ""
+    slug: Optional[str] = None
+    user_uid: Optional[str] = None
+    title: Optional[str] = None
+    description: Optional[str] = None
+    source_code: Optional[str] = None
+    language: Optional[str] = None
+    stars: Optional[int] = None
+    created_at: Optional[str] = None
+    updated_at: Optional[str] = None
+
+
+class ProjectOut(_Out):
+    uid: str = ""
+    slug: Optional[str] = None
+    user_uid: Optional[str] = None
+    title: Optional[str] = None
+    description: Optional[str] = None
+    project_type: Optional[str] = None
+    status: Optional[str] = None
+    platforms: Optional[Any] = None
+    stars: Optional[int] = None
+    is_private: Optional[bool] = None
+    read_only: Optional[bool] = None
+    release_date: Optional[str] = None
+    demo_date: Optional[str] = None
+    created_at: Optional[str] = None
+    updated_at: Optional[str] = None
+
+
+class ProjectFileOut(_Out):
+    uid: str = ""
+    path: str = ""
+    name: Optional[str] = None
+    type: Optional[str] = None
+    is_binary: Optional[bool] = None
+    mime_type: Optional[str] = None
+    size: Optional[int] = None
+    url: Optional[str] = None
+    updated_at: Optional[str] = None
+
+
+class ProjectFileRawOut(ProjectFileOut):
+    content: Optional[str] = None
+
+
+class ProjectFilesOut(_Out):
+    project: ProjectOut
+    files: list[ProjectFileOut] = []
+    is_owner: bool = False
+    read_only: bool = False
+    is_private: bool = False
+
+
+class NewsOut(_Out):
+    uid: str = ""
+    slug: Optional[str] = None
+    title: Optional[str] = None
+    description: Optional[str] = None
+    content: Optional[str] = None
+    url: Optional[str] = None
+    source_name: Optional[str] = None
+    author: Optional[str] = None
+    grade: Optional[int] = None
+    status: Optional[str] = None
+    image_url: Optional[str] = None
+    article_published: Optional[str] = None
+    created_at: Optional[str] = None
+    synced_at: Optional[str] = None
+
+
+class CommentOut(_Out):
+    uid: str = ""
+    user_uid: Optional[str] = None
+    content: Optional[str] = None
+    parent_uid: Optional[str] = None
+    target_type: Optional[str] = None
+    target_uid: Optional[str] = None
+    created_at: Optional[str] = None
+
+
+class CommentItemOut(_Out):
+    comment: CommentOut
+    author: Optional[UserOut] = None
+    time_ago: Optional[str] = None
+    votes: VotesOut = VotesOut()
+    my_vote: int = 0
+    children: list["CommentItemOut"] = []
+    attachments: list[AttachmentOut] = []
+    reactions: ReactionsOut = ReactionsOut()
+
+
+class NotificationOut(_Out):
+    uid: str = ""
+    type: Optional[str] = None
+    message: Optional[str] = None
+    read: Optional[bool] = None
+    related_uid: Optional[str] = None
+    target_url: Optional[str] = None
+    created_at: Optional[str] = None
+
+
+class MessageOut(_Out):
+    uid: str = ""
+    sender_uid: Optional[str] = None
+    receiver_uid: Optional[str] = None
+    content: Optional[str] = None
+    read: Optional[bool] = None
+    created_at: Optional[str] = None
+
+
+class BugOut(_Out):
+    uid: str = ""
+    user_uid: Optional[str] = None
+    title: Optional[str] = None
+    description: Optional[str] = None
+    status: Optional[str] = None
+    created_at: Optional[str] = None
+
+
+# ---------- list-item wrappers ----------
+
+class FeedItemOut(_Out):
+    post: PostOut
+    author: Optional[UserOut] = None
+    time_ago: Optional[str] = None
+    my_vote: int = 0
+    comment_count: int = 0
+    attachments: list[AttachmentOut] = []
+    recent_comments: list[CommentItemOut] = []
+    reactions: ReactionsOut = ReactionsOut()
+    bookmarked: bool = False
+    poll: Optional[PollOut] = None
+
+
+class GistItemOut(_Out):
+    gist: GistOut
+    author: Optional[UserOut] = None
+    time_ago: Optional[str] = None
+    my_vote: int = 0
+    comment_count: int = 0
+
+
+class ProjectListItemOut(ProjectOut):
+    author_name: Optional[str] = None
+    my_vote: int = 0
+
+
+class NewsListItemOut(_Out):
+    article: NewsOut
+    time_ago: Optional[str] = None
+    image_url: Optional[str] = None
+    grade: Optional[int] = None
+
+
+class ConversationOut(_Out):
+    other_user: Optional[UserOut] = None
+    last_message: Optional[str] = None
+    last_message_at: Optional[str] = None
+    unread: bool = False
+
+
+class MessageItemOut(_Out):
+    message: MessageOut
+    sender: Optional[UserOut] = None
+    is_mine: bool = False
+    time_ago: Optional[str] = None
+    attachments: list[AttachmentOut] = []
+
+
+class NotificationItemOut(_Out):
+    notification: NotificationOut
+    actor: Optional[UserOut] = None
+    time_ago: Optional[str] = None
+
+
+class NotificationGroupOut(_Out):
+    label: Optional[str] = None
+    entries: list[NotificationItemOut] = []
+
+
+class BugItemOut(_Out):
+    bug: BugOut
+    author: Optional[UserOut] = None
+    time_ago: Optional[str] = None
+    comments: list[CommentItemOut] = []
+    attachments: list[AttachmentOut] = []
+
+
+class LeaderboardEntryOut(_Out):
+    uid: Optional[str] = None
+    username: Optional[str] = None
+    avatar_seed: Optional[str] = None
+    stars: Optional[int] = None
+    level: Optional[int] = None
+    rank: Optional[int] = None
+
+
+class SavedItemOut(_Out):
+    target_type: Optional[str] = None
+    target_uid: Optional[str] = None
+    type_label: Optional[str] = None
+    title: Optional[str] = None
+    url: Optional[str] = None
+    time_ago: Optional[str] = None
+
+
+class AdminUserOut(UserOut):
+    email: Optional[str] = None
+    is_active: Optional[bool] = None
+    posts_count: Optional[int] = None
+
+
+class AdminNewsItemOut(_Out):
+    article: NewsOut
+    time_ago: Optional[str] = None
+    synced_at: Optional[str] = None
+    grade: Optional[int] = None
+    has_image: Optional[bool] = None
+
+
+# ---------- page envelopes ----------
+
+class FeedOut(_Out):
+    posts: list[FeedItemOut] = []
+    current_tab: Optional[str] = None
+    current_topic: Optional[str] = None
+    next_cursor: Optional[str] = None
+    total_members: Optional[int] = None
+    posts_today: Optional[int] = None
+    total_projects: Optional[int] = None
+    total_gists: Optional[int] = None
+    top_authors: list[UserOut] = []
+    daily_topic: Optional[Any] = None
+
+
+class PostDetailOut(_Out):
+    post: PostOut
+    author: Optional[UserOut] = None
+    is_owner: bool = False
+    star_count: int = 0
+    my_vote: int = 0
+    time_ago: Optional[str] = None
+    comments: list[CommentItemOut] = []
+    attachments: list[AttachmentOut] = []
+    reactions: ReactionsOut = ReactionsOut()
+    bookmarked: bool = False
+    poll: Optional[PollOut] = None
+    comment_count: Optional[int] = None
+    related_posts: list[FeedItemOut] = []
+
+
+class ProjectsOut(_Out):
+    projects: list[ProjectListItemOut] = []
+    current_tab: Optional[str] = None
+    search: Optional[str] = None
+    project_type: Optional[str] = None
+    total_count: Optional[int] = None
+    next_cursor: Optional[str] = None
+    total_members: Optional[int] = None
+
+
+class ProjectDetailOut(_Out):
+    project: ProjectOut
+    author: Optional[UserOut] = None
+    is_owner: bool = False
+    star_count: int = 0
+    my_vote: int = 0
+    time_ago: Optional[str] = None
+    comments: list[CommentItemOut] = []
+    attachments: list[AttachmentOut] = []
+    reactions: ReactionsOut = ReactionsOut()
+    bookmarked: bool = False
+    platforms: Optional[Any] = None
+    is_private: bool = False
+    read_only: bool = False
+    forked_from: Optional[dict] = None
+    fork_count: int = 0
+
+
+class GistsOut(_Out):
+    gists: list[GistItemOut] = []
+    total_count: Optional[int] = None
+    next_cursor: Optional[str] = None
+    current_language: Optional[str] = None
+
+
+class GistDetailOut(_Out):
+    gist: GistOut
+    author: Optional[UserOut] = None
+    is_owner: bool = False
+    star_count: int = 0
+    my_vote: int = 0
+    time_ago: Optional[str] = None
+    comments: list[CommentItemOut] = []
+    attachments: list[AttachmentOut] = []
+    reactions: ReactionsOut = ReactionsOut()
+    bookmarked: bool = False
+
+
+class NewsListOut(_Out):
+    articles: list[NewsListItemOut] = []
+    next_cursor: Optional[str] = None
+
+
+class NewsDetailOut(_Out):
+    article: NewsOut
+    canonical_slug: Optional[str] = None
+    image_url: Optional[str] = None
+    grade: Optional[int] = None
+    time_ago: Optional[str] = None
+    comments: list[CommentItemOut] = []
+    bookmarked: bool = False
+
+
+class ProfileOut(_Out):
+    profile_user: UserOut
+    posts: list[FeedItemOut] = []
+    badges: list[BadgeOut] = []
+    projects: list[ProjectOut] = []
+    gists: list[GistItemOut] = []
+    current_tab: Optional[str] = None
+    posts_count: Optional[int] = None
+    is_following: bool = False
+    is_owner: bool = False
+    can_view_api_key: bool = False
+    api_key: Optional[str] = None
+    ai_quota: Optional[dict] = None
+    activities: list[Any] = []
+    rank: Optional[int] = None
+    heatmap: Optional[Any] = None
+    heatmap_months: Optional[Any] = None
+    streak: Optional[Any] = None
+    people: list[Any] = []
+    follow_pagination: Optional[Any] = None
+    followers_count: Optional[int] = None
+    following_count: Optional[int] = None
+
+
+class MessagesOut(_Out):
+    conversations: list[ConversationOut] = []
+    messages: list[MessageItemOut] = []
+    other_user: Optional[UserOut] = None
+    current_conversation: Optional[str] = None
+    search: Optional[str] = None
+
+
+class NotificationsOut(_Out):
+    notification_groups: list[NotificationGroupOut] = []
+    next_cursor: Optional[str] = None
+
+
+class LeaderboardOut(_Out):
+    entries: list[LeaderboardEntryOut] = []
+    user_rank: Optional[int] = None
+    total_members: Optional[int] = None
+    posts_today: Optional[int] = None
+    total_projects: Optional[int] = None
+    total_gists: Optional[int] = None
+    top_authors: list[UserOut] = []
+    featured_news: list[NewsOut] = []
+
+
+class BugsOut(_Out):
+    bugs: list[BugItemOut] = []
+
+
+class SavedOut(_Out):
+    items: list[SavedItemOut] = []
+    next_cursor: Optional[str] = None
+
+
+class AdminUsersOut(_Out):
+    users: list[AdminUserOut] = []
+    pagination: Optional[Any] = None
+    admin_section: Optional[str] = None
+
+
+class AdminNewsOut(_Out):
+    articles: list[AdminNewsItemOut] = []
+    pagination: Optional[Any] = None
+    admin_section: Optional[str] = None
+
+
+class AdminSettingsOut(_Out):
+    settings: dict = {}
+    admin_section: Optional[str] = None
+
+
+class GatewayUsageOut(_Out):
+    window_hours: int = 48
+    generated_at: Optional[str] = None
+    requests: int = 0
+    volume: dict = {}
+    tokens: dict = {}
+    latency: dict = {}
+    errors: dict = {}
+    cost: dict = {}
+    behavior: dict = {}
+    hourly: list = []
+    notes: dict = {}
+    admin_section: Optional[str] = None
+
+
+class UserAiUsageOut(_Out):
+    owner_id: str = ""
+    window_hours: int = 24
+    generated_at: Optional[str] = None
+    requests: int = 0
+    success: int = 0
+    failed: int = 0
+    success_pct: float = 0.0
+    error_pct: float = 0.0
+    tokens: dict = {}
+    cost: dict = {}
+    latency: dict = {}
+    first_used: Optional[str] = None
+    last_used: Optional[str] = None
+    by_model: list = []
+    by_backend: list = []
+    hourly: list = []
+    notes: dict = {}
+
+
+# ---------- auth pages ----------
+
+class AuthPageOut(_Out):
+    page: str = ""
+    next_url: Optional[str] = None
+    registration_closed: Optional[bool] = None
+    sent: Optional[bool] = None
+    token: Optional[str] = None
+    errors: list = []
+
+
+CommentItemOut.model_rebuild()

devplacepy/seo.py

@@ -1,26 +1,39 @@
 import json
 import logging
-from datetime import datetime
+import os
+import time
+from urllib.parse import urlencode
 from xml.etree.ElementTree import Element, tostring
 from xml.dom import minidom
+from devplacepy.config import SITE_URL
+from devplacepy.utils import strip_html
 
 logger = logging.getLogger(__name__)
 
 
 SITE_NAME = "DevPlace"
+SITEMAP_URL_LIMIT = 5000
+SITEMAP_TTL = int(os.environ.get("DEVPLACE_SITEMAP_TTL", "3600"))
+_sitemap_cache = {}
 
 
 def truncate(text, max_len=160):
     if not text:
         return ""
-    text = " ".join(text.split())[:max_len]
-    if len(text) >= max_len:
-        text = text.rsplit(" ", 1)[0] + "..."
-    return text
+    text = " ".join(text.split())
+    if len(text) <= max_len:
+        return text
+    text = text[:max_len - 3].rsplit(" ", 1)[0]
+    return text + "..."
+
+
+def public_base_url() -> str:
+    from devplacepy.database import get_setting
+    return (get_setting("site_url", "").strip() or SITE_URL or "").rstrip("/")
 
 
 def site_url(request):
-    return str(request.base_url).rstrip("/")
+    return public_base_url() or str(request.base_url).rstrip("/")
 
 
 def website_schema(base_url):
@@ -68,13 +81,12 @@ def discussion_forum_posting(post, author, comment_count, star_count, base_url):
             "url": f"{base_url}/profile/{author['username']}" if author else ""
         },
         "datePublished": post.get("created_at", ""),
+        "dateModified": post.get("updated_at") or post.get("created_at", ""),
         "interactionStatistic": [
             {"@type": "InteractionCounter", "interactionType": "https://schema.org/LikeAction", "userInteractionCount": star_count},
             {"@type": "InteractionCounter", "interactionType": "https://schema.org/CommentAction", "userInteractionCount": comment_count}
         ]
     }
-    if post.get("title"):
-        schema["headline"] = post["title"]
     return schema
 
 
@@ -98,7 +110,7 @@ def software_application_schema(project, base_url):
         "@type": "SoftwareApplication",
         "name": project.get("title", "Untitled"),
         "description": truncate(project.get("description", ""), 300),
-        "url": f"{base_url}/projects",
+        "url": f"{base_url}/projects/{project.get('slug') or project['uid']}",
         "applicationCategory": "DeveloperApplication",
         "operatingSystem": project.get("platforms", "Cross-platform"),
         "author": {
@@ -106,6 +118,7 @@ def software_application_schema(project, base_url):
             "name": project.get("author_name", "Unknown")
         },
         "datePublished": project.get("created_at", ""),
+        "dateModified": project.get("updated_at") or project.get("created_at", ""),
         "offers": {
             "@type": "Offer",
             "price": "0",
@@ -114,6 +127,56 @@ def software_application_schema(project, base_url):
     }
 
 
+def organization_schema(base_url):
+    return {
+        "@type": "Organization",
+        "name": SITE_NAME,
+        "url": base_url,
+        "logo": f"{base_url}{DEFAULT_OG_IMAGE}",
+    }
+
+
+def news_article_schema(article, base_url, image_url=""):
+    url = f"{base_url}/news/{article.get('slug') or article['uid']}"
+    schema = {
+        "@type": "NewsArticle",
+        "headline": (article.get("title") or "Untitled")[:110],
+        "description": truncate(strip_html(article.get("description", "") or ""), 200),
+        "url": url,
+        "datePublished": article.get("synced_at", "") or article.get("created_at", ""),
+        "dateModified": article.get("synced_at", "") or article.get("created_at", ""),
+        "mainEntityOfPage": {"@type": "WebPage", "@id": url},
+        "author": {"@type": "Organization", "name": article.get("source_name") or SITE_NAME},
+        "publisher": organization_schema(base_url),
+    }
+    if image_url:
+        schema["image"] = image_url
+    return schema
+
+
+def software_source_code_schema(gist, base_url):
+    return {
+        "@type": "SoftwareSourceCode",
+        "name": gist.get("title") or "Gist",
+        "description": truncate(strip_html(gist.get("description", "") or ""), 200),
+        "url": f"{base_url}/gists/{gist.get('slug') or gist['uid']}",
+        "programmingLanguage": gist.get("language", "") or "text",
+        "dateCreated": gist.get("created_at", ""),
+        "dateModified": gist.get("updated_at") or gist.get("created_at", ""),
+    }
+
+
+def _json_ld_dumps(payload):
+    raw = json.dumps(payload, ensure_ascii=False)
+    return (
+        raw.replace("<", "\\u003c")
+           .replace(">", "\\u003e")
+           .replace("&", "\\u0026")
+           .replace("
", "\\u2028")
+           .replace("
", "\\u2029")
+    )
+
+
 def combine(schemas):
     if not schemas:
         return None
@@ -124,34 +187,89 @@ def combine(schemas):
             cleaned.append(s)
     if not cleaned:
         return None
-    if len(cleaned) == 1:
-        return json.dumps({"@context": "https://schema.org", **cleaned[0]}, ensure_ascii=False)
-    return json.dumps({"@context": "https://schema.org", "@graph": cleaned}, ensure_ascii=False)
+    payload = (
+        {"@context": "https://schema.org", **cleaned[0]}
+        if len(cleaned) == 1
+        else {"@context": "https://schema.org", "@graph": cleaned}
+    )
+    return _json_ld_dumps(payload)
 
 
-DEFAULT_OG_IMAGE = "/static/og-default.svg"
+DEFAULT_OG_IMAGE = "/static/og-default.png"
 
 
-def base_seo_context(request, title="", description="", robots="index,follow", og_type="website", og_image=None, breadcrumbs=None, schemas=None):
+def absolute_url(base, path):
+    if not path:
+        return ""
+    return path if path.startswith("http") else f"{base}{path}"
+
+
+def base_seo_context(request, title="", description="", robots="index,follow", og_type="website", og_image=None, breadcrumbs=None, schemas=None, prev_url=None, next_url=None):
     base = site_url(request)
-    page_title = f"{title} — {SITE_NAME}" if title else SITE_NAME
-    canonical = str(request.url).split("?")[0]
-    og_img = og_image or f"{base}{DEFAULT_OG_IMAGE}"
+    page_title = f"{title} - {SITE_NAME}" if title else SITE_NAME
+    canonical = f"{base}{request.url.path}"
+    page = request.query_params.get("page")
+    if page and page not in ("", "1"):
+        canonical = f"{canonical}?page={page}"
+    clean_description = truncate(strip_html(description), 160)
+    og_img = absolute_url(base, og_image) or f"{base}{DEFAULT_OG_IMAGE}"
+    page_schemas = list(schemas or [])
+    if breadcrumbs:
+        page_schemas.append(breadcrumb_schema(breadcrumbs, base))
     return {
         "page_title": page_title,
-        "meta_description": description,
+        "meta_description": clean_description,
         "meta_robots": robots,
         "canonical_url": canonical,
         "og_title": title or SITE_NAME,
-        "og_description": description,
+        "og_description": clean_description,
         "og_image": og_img,
         "og_type": og_type,
         "breadcrumbs": breadcrumbs or [],
-        "page_schema": combine(schemas or []),
+        "page_schema": combine(page_schemas),
+        "prev_url": absolute_url(base, prev_url),
+        "next_url": absolute_url(base, next_url),
     }
 
 
+def next_page_url(request, next_cursor):
+    if not next_cursor:
+        return None
+    params = dict(request.query_params)
+    params["before"] = next_cursor
+    return f"{request.url.path}?{urlencode(params)}"
+
+
+def list_page_seo(request, title="", description="", breadcrumbs=None, prev_url=None, next_url=None):
+    base = site_url(request)
+    return base_seo_context(
+        request,
+        title=title,
+        description=description,
+        breadcrumbs=breadcrumbs,
+        schemas=[website_schema(base)],
+        prev_url=prev_url,
+        next_url=next_url,
+    )
+
+
 def make_sitemap(base_url):
+    cached = _sitemap_cache.get(base_url)
+    if cached and time.time() - cached[0] < SITEMAP_TTL:
+        return cached[1]
+    xml = _build_sitemap(base_url)
+    _sitemap_cache[base_url] = (time.time(), xml)
+    return xml
+
+
+def _collect(table, limit, label, **query):
+    rows = list(table.find(_limit=limit, **query))
+    if len(rows) >= limit:
+        logger.warning("sitemap: %s truncated at %d entries", label, limit)
+    return rows
+
+
+def _build_sitemap(base_url):
     from devplacepy.database import get_table, db
 
     def url_element(loc, lastmod=None, changefreq=None, priority=None):
@@ -180,9 +298,11 @@ def make_sitemap(base_url):
     urlset.append(url_element(f"{base_url}/feed", changefreq="hourly", priority="0.9"))
     urlset.append(url_element(f"{base_url}/news", changefreq="hourly", priority="0.9"))
     urlset.append(url_element(f"{base_url}/projects", changefreq="daily", priority="0.8"))
+    urlset.append(url_element(f"{base_url}/gists", changefreq="daily", priority="0.8"))
+    urlset.append(url_element(f"{base_url}/leaderboard", changefreq="daily", priority="0.7"))
 
     if "posts" in db.tables:
-        posts = list(get_table("posts").find(order_by=["-created_at"], _limit=500))
+        posts = _collect(get_table("posts"), SITEMAP_URL_LIMIT, "posts", order_by=["-created_at"])
         for p in posts:
             urlset.append(url_element(
                 f"{base_url}/posts/{p.get('slug') or p['uid']}",
@@ -192,8 +312,10 @@ def make_sitemap(base_url):
             ))
 
     if "projects" in db.tables:
-        projects = list(get_table("projects").find(order_by=["-created_at"], _limit=1000))
+        projects = _collect(get_table("projects"), SITEMAP_URL_LIMIT, "projects", order_by=["-created_at"])
         for p in projects:
+            if p.get("is_private"):
+                continue
             urlset.append(url_element(
                 f"{base_url}/projects/{p.get('slug') or p['uid']}",
                 lastmod=p.get("created_at", ""),
@@ -202,7 +324,7 @@ def make_sitemap(base_url):
             ))
 
     if "gists" in db.tables:
-        gists = list(get_table("gists").find(order_by=["-created_at"], _limit=500))
+        gists = _collect(get_table("gists"), SITEMAP_URL_LIMIT, "gists", order_by=["-created_at"])
         for g in gists:
             urlset.append(url_element(
                 f"{base_url}/gists/{g.get('slug') or g['uid']}",
@@ -211,11 +333,28 @@ def make_sitemap(base_url):
                 priority="0.6"
             ))
 
+    if "news" in db.tables:
+        articles = _collect(get_table("news"), SITEMAP_URL_LIMIT, "news", status="published", order_by=["-synced_at"])
+        for a in articles:
+            urlset.append(url_element(
+                f"{base_url}/news/{a.get('slug') or a['uid']}",
+                lastmod=a.get("synced_at", "") or a.get("created_at", ""),
+                changefreq="weekly",
+                priority="0.7"
+            ))
+
     if "users" in db.tables:
-        users = list(get_table("users").find(order_by=["-created_at"], _limit=200))
+        post_counts = {}
+        if "posts" in db.tables:
+            for row in db.query("SELECT user_uid, COUNT(*) AS c FROM posts GROUP BY user_uid"):
+                post_counts[row["user_uid"]] = row["c"]
+        users = _collect(get_table("users"), SITEMAP_URL_LIMIT, "users", order_by=["-created_at"])
         for u in users:
+            if post_counts.get(u["uid"], 0) < 2:
+                continue
             urlset.append(url_element(
                 f"{base_url}/profile/{u['username']}",
+                lastmod=u.get("created_at", ""),
                 changefreq="weekly",
                 priority="0.4"
             ))

devplacepy/services/base.py

@@ -1,93 +1,391 @@
 import asyncio
+import json
 import logging
 from abc import ABC, abstractmethod
 from collections import deque
-from datetime import datetime
+from datetime import datetime, timedelta, timezone
+
+from devplacepy.database import db, get_table, get_setting, get_int_setting
 
 logger = logging.getLogger(__name__)
 
 
+class ConfigField:
+    def __init__(self, key, label, type="str", default="", help="",
+                 options=None, secret=False, minimum=None, maximum=None, group="General"):
+        self.key = key
+        self.label = label
+        self.type = type
+        self.default = default
+        self.help = help
+        self.options = options or []
+        self.secret = secret
+        self.minimum = minimum
+        self.maximum = maximum
+        self.group = group
+
+    def coerce(self, raw):
+        if self.type == "int":
+            try:
+                value = int(str(raw).strip())
+            except (TypeError, ValueError):
+                raise ValueError(f"{self.label} must be a whole number")
+            if self.minimum is not None and value < self.minimum:
+                raise ValueError(f"{self.label} must be at least {self.minimum}")
+            if self.maximum is not None and value > self.maximum:
+                raise ValueError(f"{self.label} must be at most {self.maximum}")
+            return value
+        if self.type == "float":
+            try:
+                value = float(str(raw).strip())
+            except (TypeError, ValueError):
+                raise ValueError(f"{self.label} must be a number")
+            if self.minimum is not None and value < self.minimum:
+                raise ValueError(f"{self.label} must be at least {self.minimum}")
+            if self.maximum is not None and value > self.maximum:
+                raise ValueError(f"{self.label} must be at most {self.maximum}")
+            return value
+        if self.type == "bool":
+            text = str(raw).strip()
+            if text in ("0", "1"):
+                return text == "1"
+            raise ValueError(f"{self.label} must be enabled or disabled")
+        if self.type == "select":
+            allowed = [option["value"] for option in self.options]
+            if raw not in allowed:
+                raise ValueError(f"{self.label} has an invalid selection")
+            return raw
+        if self.type == "url":
+            text = str(raw).strip()
+            if text and not (text.startswith("http://") or text.startswith("https://")):
+                raise ValueError(f"{self.label} must be a http(s) URL")
+            return text
+        return str(raw)
+
+    def to_storage(self, value):
+        if isinstance(value, bool):
+            return "1" if value else "0"
+        return str(value)
+
+    def read(self):
+        raw = get_setting(self.key, "")
+        if raw == "":
+            return self.default
+        try:
+            return self.coerce(raw)
+        except ValueError:
+            return self.default
+
+    def display_value(self):
+        if self.secret:
+            return ""
+        raw = get_setting(self.key, "")
+        if raw == "":
+            return self.to_storage(self.default)
+        return raw
+
+    def spec(self):
+        return {
+            "key": self.key,
+            "label": self.label,
+            "type": self.type,
+            "help": self.help,
+            "options": self.options,
+            "secret": self.secret,
+            "minimum": self.minimum,
+            "maximum": self.maximum,
+            "value": self.display_value(),
+            "is_set": bool(get_setting(self.key, "")) if self.secret else None,
+            "group": self.group,
+        }
+
+
 class BaseService(ABC):
+    config_fields = []
+    interval_key = None
+    min_interval = 1
+    default_enabled = True
+    title = ""
+    description = ""
+    DEFAULT_LOG_SIZE = 20
+    TICK_SECONDS = 1
+    PERSIST_SECONDS = 3
+    STALE_SECONDS = 15
+
     def __init__(self, name: str, interval_seconds: int = 3600):
         self.name = name
         self.interval_seconds = interval_seconds
-        self.log_buffer = deque(maxlen=20)
+        self.interval_key = self.interval_key or f"service_{name}_interval"
+        self.enabled_key = f"service_{name}_enabled"
+        self.command_key = f"service_{name}_command"
+        self.log_size_key = f"service_{name}_log_size"
+        self.log_buffer = deque(maxlen=self.DEFAULT_LOG_SIZE)
         self._task = None
         self._running = False
+        self._shutdown = False
+        self._started_at = None
         self._last_run = None
         self._next_run = None
-        self._started_at = None
+        self._next_due = None
+        self._run_pending = False
+        self._last_command = None
+        self._last_persist = None
+        self.enabled_field = ConfigField(
+            self.enabled_key, "Enabled", type="bool", default=self.default_enabled,
+            help="When enabled the service runs on its interval and starts on boot.",
+            group="General")
+        self.interval_field = ConfigField(
+            self.interval_key, "Run interval (seconds)", type="int",
+            default=interval_seconds, minimum=self.min_interval,
+            help=f"Delay between runs. Minimum {self.min_interval} seconds.",
+            group="General")
+        self.log_size_field = ConfigField(
+            self.log_size_key, "Log buffer size", type="int",
+            default=self.DEFAULT_LOG_SIZE, minimum=1, maximum=200,
+            help="Number of recent log lines to keep.",
+            group="Advanced")
+
+    def all_fields(self) -> list:
+        return [self.enabled_field, self.interval_field, *self.config_fields, self.log_size_field]
+
+    def _grouped_fields(self) -> list:
+        groups: dict = {}
+        order: list = []
+        for field in self.all_fields():
+            spec = field.spec()
+            group = spec["group"]
+            if group not in groups:
+                groups[group] = []
+                order.append(group)
+            groups[group].append(spec)
+        return [{"name": name, "fields": groups[name]} for name in order]
 
     @property
     def status(self) -> str:
         return "running" if self._running else "stopped"
 
-    @property
-    def last_run(self) -> str | None:
-        return self._last_run
+    def is_enabled(self) -> bool:
+        return get_setting(self.enabled_key, "1" if self.default_enabled else "0") == "1"
 
-    @property
-    def next_run(self) -> str | None:
-        return self._next_run
+    def current_interval(self) -> int:
+        return max(self.min_interval, get_int_setting(self.interval_key, self.interval_seconds))
 
-    @property
-    def uptime(self) -> str | None:
-        if self._started_at is None:
-            return None
-        delta = datetime.utcnow() - self._started_at
-        return str(delta).split(".")[0]
+    def get_config(self) -> dict:
+        return {field.key: field.read() for field in self.all_fields()}
 
     def log(self, message: str) -> None:
-        stamp = datetime.utcnow().strftime("%H:%M:%S")
-        entry = f"[{stamp}] {message}"
-        self.log_buffer.append(entry)
+        stamp = datetime.now(timezone.utc).strftime("%H:%M:%S")
+        self.log_buffer.append(f"[{stamp}] {message}")
         logger.info(f"[{self.name}] {message}")
 
     @abstractmethod
     async def run_once(self) -> None:
         pass
 
-    async def _run_loop(self) -> None:
-        self._running = True
-        self._started_at = datetime.utcnow()
-        self.log(f"Service started (interval={self.interval_seconds}s)")
-        while self._running:
-            try:
-                self._last_run = datetime.utcnow().isoformat()
-                self._next_run = None
-                await self.run_once()
-            except asyncio.CancelledError:
-                self.log("Service cancelled")
-                break
-            except Exception as e:
-                self.log(f"Error in run_once: {e}")
-            if not self._running:
-                break
-            self._next_run = datetime.utcnow().isoformat()
-            self.log(f"Sleeping for {self.interval_seconds}s")
-            try:
-                await asyncio.sleep(self.interval_seconds)
-            except asyncio.CancelledError:
-                self.log("Service cancelled during sleep")
-                break
-        self._running = False
-        self.log("Service stopped")
+    async def on_enable(self) -> None:
+        pass
 
-    async def start(self) -> None:
-        if self._running:
-            self.log("Already running")
+    async def on_disable(self) -> None:
+        pass
+
+    def collect_metrics(self) -> dict:
+        return {}
+
+    def _safe_metrics(self) -> dict:
+        try:
+            return self.collect_metrics()
+        except Exception as e:
+            logger.warning(f"Could not collect metrics for {self.name}: {e}")
+            return {}
+
+    def start_supervisor(self) -> None:
+        if self._task is not None:
             return
-        self._running = True
+        self._shutdown = False
         self._task = asyncio.create_task(self._run_loop())
 
-    async def stop(self) -> None:
-        if not self._running:
-            return
-        self._running = False
-        if self._task is not None:
-            self._task.cancel()
+    def request_shutdown(self) -> None:
+        self._shutdown = True
+
+    async def _run_loop(self) -> None:
+        self._last_command = get_setting(self.command_key, "")
+        self.log("Supervisor started")
+        self._persist_state(force=True)
+        while not self._shutdown:
             try:
-                await asyncio.wait_for(self._task, timeout=10.0)
-            except (asyncio.CancelledError, asyncio.TimeoutError):
-                pass
-            self._task = None
+                await self._tick()
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                self.log(f"Loop error: {e}")
+            try:
+                await asyncio.sleep(self.TICK_SECONDS)
+            except asyncio.CancelledError:
+                break
+        if self._started_at is not None:
+            await self._safe_disable()
+        self._running = False
+        self._started_at = None
+        self._persist_state(force=True)
+        self.log("Supervisor stopped")
+
+    async def _tick(self) -> None:
+        self._sync_log_size()
+        self._handle_commands()
+        if self.is_enabled():
+            if self._started_at is None:
+                self._started_at = datetime.now(timezone.utc)
+                self._running = True
+                self.log("Service enabled")
+                await self.on_enable()
+            due = self._next_due is None or datetime.now(timezone.utc) >= self._next_due
+            if self._run_pending or due:
+                self._run_pending = False
+                await self._execute_run()
+        elif self._started_at is not None:
+            await self._safe_disable()
+            self._started_at = None
+            self._running = False
+            self._next_due = None
+            self._next_run = None
+            self.log("Service disabled")
+        self._persist_state()
+
+    async def _safe_disable(self) -> None:
+        try:
+            await self.on_disable()
+        except Exception as e:
+            self.log(f"on_disable error: {e}")
+
+    async def _execute_run(self) -> None:
+        self.interval_seconds = self.current_interval()
+        self._last_run = datetime.now(timezone.utc).isoformat()
+        self._next_run = None
+        self._persist_state(force=True)
+        try:
+            await self.run_once()
+        except asyncio.CancelledError:
+            raise
+        except Exception as e:
+            self.log(f"Error in run_once: {e}")
+        self.interval_seconds = self.current_interval()
+        self._next_due = datetime.now(timezone.utc) + timedelta(seconds=self.interval_seconds)
+        self._next_run = self._next_due.isoformat()
+        self.log(f"Next run in {self.interval_seconds}s")
+        self._persist_state(force=True)
+
+    def _handle_commands(self) -> None:
+        raw = get_setting(self.command_key, "")
+        if not raw or raw == self._last_command:
+            return
+        self._last_command = raw
+        verb = raw.split(":", 1)[0]
+        if verb == "run":
+            self._run_pending = True
+            self.log("Run requested")
+        elif verb == "clear":
+            self.log_buffer.clear()
+            self.log("Logs cleared")
+
+    def _sync_log_size(self) -> None:
+        size = max(1, get_int_setting(self.log_size_key, self.DEFAULT_LOG_SIZE))
+        if size != self.log_buffer.maxlen:
+            self.log_buffer = deque(self.log_buffer, maxlen=size)
+
+    def _persist_state(self, force: bool = False) -> None:
+        now = datetime.now(timezone.utc)
+        if not force and self._last_persist is not None:
+            if (now - self._last_persist).total_seconds() < self.PERSIST_SECONDS:
+                return
+        self._last_persist = now
+        record = {
+            "name": self.name,
+            "status": self.status,
+            "last_run": self._last_run or "",
+            "next_run": self._next_run or "",
+            "started_at": self._started_at.isoformat() if self._started_at else "",
+            "heartbeat": now.isoformat(),
+            "logs": json.dumps(list(self.log_buffer)),
+            "metrics": json.dumps(self._safe_metrics()),
+            "updated_at": now.isoformat(),
+        }
+        try:
+            table = get_table("service_state")
+            existing = table.find_one(name=self.name)
+            if existing:
+                table.update({**record, "id": existing["id"]}, ["id"])
+            else:
+                table.insert(record)
+        except Exception as e:
+            logger.warning(f"Could not persist state for {self.name}: {e}")
+
+    def _read_state(self) -> dict:
+        if "service_state" not in db.tables:
+            return {}
+        row = db["service_state"].find_one(name=self.name)
+        return row or {}
+
+    def _display_status(self, enabled: bool, state: dict) -> str:
+        if not enabled:
+            return "stopped"
+        heartbeat = state.get("heartbeat")
+        if not heartbeat:
+            return "stalled"
+        try:
+            beat = datetime.fromisoformat(heartbeat)
+        except ValueError:
+            return "stalled"
+        if (datetime.now(timezone.utc) - beat).total_seconds() > self.STALE_SECONDS:
+            return "stalled"
+        return "running"
+
+    def _uptime(self, state: dict, status: str) -> str | None:
+        if status != "running":
+            return None
+        started = state.get("started_at")
+        if not started:
+            return None
+        try:
+            since = datetime.fromisoformat(started)
+        except ValueError:
+            return None
+        return str(datetime.now(timezone.utc) - since).split(".")[0]
+
+    def _logs(self, state: dict) -> list:
+        raw = state.get("logs")
+        if not raw:
+            return []
+        try:
+            return json.loads(raw)
+        except (ValueError, TypeError):
+            return []
+
+    def _metrics(self, state: dict) -> dict:
+        raw = state.get("metrics")
+        if not raw:
+            return {}
+        try:
+            return json.loads(raw)
+        except (ValueError, TypeError):
+            return {}
+
+    def describe(self) -> dict:
+        state = self._read_state()
+        enabled = self.is_enabled()
+        status = self._display_status(enabled, state)
+        return {
+            "name": self.name,
+            "title": self.title or self.name.capitalize(),
+            "description": self.description,
+            "enabled": enabled,
+            "status": status,
+            "interval_seconds": self.current_interval(),
+            "last_run": state.get("last_run") or None,
+            "next_run": state.get("next_run") or None,
+            "heartbeat": state.get("heartbeat") or None,
+            "uptime": self._uptime(state, status),
+            "log_buffer": self._logs(state),
+            "metrics": self._metrics(state),
+            "fields": [field.spec() for field in self.all_fields()],
+            "field_groups": self._grouped_fields(),
+        }

devplacepy/services/bot/__init__.py

@@ -0,0 +1,3 @@
+from devplacepy.services.bot.service import BotsService
+
+__all__ = ["BotsService"]

devplacepy/services/bot/browser.py

@@ -0,0 +1,162 @@
+import asyncio
+import logging
+import random
+from typing import Optional
+
+from playwright.async_api import Page, Playwright, async_playwright
+
+logger = logging.getLogger(__name__)
+
+
+class BotBrowser:
+    def __init__(self, headless: bool = True):
+        self._headless = headless
+        self._playwright: Optional[Playwright] = None
+        self._browser = None
+        self._context = None
+        self._page: Optional[Page] = None
+
+    async def launch(self) -> None:
+        self._playwright = await async_playwright().start()
+        self._browser = await self._playwright.chromium.launch(
+            headless=self._headless,
+            args=[
+                "--disable-blink-features=AutomationControlled", "--no-first-run",
+                "--disable-dev-shm-usage", "--window-size=1280,900",
+            ],
+        )
+        self._context = await self._browser.new_context(
+            user_agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36",
+            viewport={"width": 1280, "height": 900},
+            locale="en-US",
+        )
+        self._page = await self._context.new_page()
+
+    async def close(self) -> None:
+        context, self._context = self._context, None
+        browser, self._browser = self._browser, None
+        playwright, self._playwright = self._playwright, None
+        self._page = None
+        for label, resource, shutdown in (
+            ("context", context, lambda r: r.close()),
+            ("browser", browser, lambda r: r.close()),
+            ("playwright", playwright, lambda r: r.stop()),
+        ):
+            if resource is None:
+                continue
+            try:
+                await shutdown(resource)
+            except Exception as e:
+                logger.debug("Browser %s close error: %s", label, e)
+
+    @property
+    def page(self) -> Page:
+        return self._page
+
+    async def goto(self, url: str) -> None:
+        try:
+            await self._page.goto(url, wait_until="domcontentloaded", timeout=15000)
+        except Exception as e:
+            logger.debug("goto (domcontentloaded) failed for %s: %s", url[:80], e)
+            try:
+                await self._page.goto(url, timeout=20000)
+            except Exception as e2:
+                logger.warning("goto failed for %s: %s", url[:80], e2)
+        await self._idle(0.5, 1.5)
+
+    async def _idle(self, lo: float = 0.5, hi: float = 2.0) -> None:
+        await asyncio.sleep(random.uniform(lo, hi))
+
+    async def scroll(self, amount: Optional[int] = None) -> None:
+        if amount is None:
+            amount = random.randint(300, 800)
+        try:
+            await self._page.evaluate(f"window.scrollBy(0, {amount})")
+        except Exception as e:
+            logger.debug("scroll failed: %s", e)
+        await self._idle(0.2, 0.6)
+
+    async def scroll_to_bottom(self) -> None:
+        try:
+            await self._page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+        except Exception as e:
+            logger.debug("scroll_to_bottom failed: %s", e)
+
+    async def fill(self, sel: str, val: str) -> bool:
+        try:
+            el = self._page.locator(sel).first
+            if await el.count() == 0:
+                logger.debug("fill: selector '%s' not found", sel)
+                return False
+            await el.click(timeout=3000)
+            await self._idle(0.05, 0.15)
+            await el.fill("")
+            for ch in val:
+                await self._page.keyboard.type(ch, delay=random.randint(20, 60))
+                if ch == " ":
+                    await asyncio.sleep(random.uniform(0.02, 0.08))
+            return True
+        except Exception as e:
+            logger.debug("fill failed for '%s': %s", sel, e)
+            return False
+
+    async def click(self, sel: str) -> bool:
+        try:
+            el = self._page.locator(sel).first
+            if await el.count() == 0:
+                logger.debug("click: selector '%s' not found", sel)
+                return False
+            box = await el.bounding_box()
+            if box:
+                tx = box["x"] + box["width"] / 2 + random.uniform(-3, 3)
+                ty = box["y"] + box["height"] / 2 + random.uniform(-3, 3)
+                await self._page.mouse.move(tx, ty, steps=random.randint(8, 18))
+                await self._idle(0.02, 0.06)
+            await el.click(timeout=5000)
+            return True
+        except Exception as e:
+            logger.debug("click failed for '%s': %s", sel, e)
+            return False
+
+    async def text(self, sel: str) -> str:
+        try:
+            return (await self._page.locator(sel).first.inner_text()) or ""
+        except Exception as e:
+            logger.debug("text failed for '%s': %s", sel, e)
+            return ""
+
+    async def attr(self, sel: str, attr: str) -> str:
+        try:
+            val = await self._page.locator(sel).first.get_attribute(attr)
+            return val or ""
+        except Exception as e:
+            logger.debug("attr '%s' on '%s' failed: %s", attr, sel, e)
+            return ""
+
+    async def html(self) -> str:
+        try:
+            return await self._page.content()
+        except Exception as e:
+            logger.debug("html() failed: %s", e)
+            return ""
+
+    async def url(self) -> str:
+        try:
+            return self._page.url
+        except Exception as e:
+            logger.debug("url() failed: %s", e)
+            return ""
+
+    async def title(self) -> str:
+        try:
+            return await self._page.title()
+        except Exception as e:
+            logger.debug("title() failed: %s", e)
+            return ""
+
+    async def reload(self) -> None:
+        try:
+            await self._page.reload(timeout=15000)
+            await self._idle()
+        except Exception as e:
+            logger.debug("reload failed: %s", e)

devplacepy/services/bot/config.py

@@ -0,0 +1,73 @@
+from pathlib import Path
+
+from devplacepy.config import INTERNAL_GATEWAY_URL, INTERNAL_MODEL
+
+BASE_URL_DEFAULT = "https://pravda.education"
+API_URL_DEFAULT = INTERNAL_GATEWAY_URL
+NEWS_API_DEFAULT = "https://news.app.molodetz.nl/api"
+MODEL_DEFAULT = INTERNAL_MODEL
+INPUT_COST_PER_1M_DEFAULT = 0.27
+OUTPUT_COST_PER_1M_DEFAULT = 1.10
+
+COST_WINDOW_SECONDS = 600
+COST_WARMUP_SECONDS = 120
+
+STATE_DIR = Path.home() / ".devplace_bots"
+ARTICLE_REGISTRY_PATH = Path.home() / ".dpbot_article_registry.json"
+
+PROJECT_STATUSES = ["In Development", "Released"]
+
+PERSONAS = [
+    "enthusiastic_junior",
+    "grumpy_senior",
+    "hobbyist_maker",
+    "academic_type",
+    "minimalist",
+    "storyteller",
+    "rebel",
+    "mentor",
+]
+
+REACT_RATES = {
+    "enthusiastic_junior": 0.50,
+    "hobbyist_maker": 0.40,
+    "storyteller": 0.30,
+    "mentor": 0.30,
+    "rebel": 0.25,
+    "academic_type": 0.20,
+    "minimalist": 0.12,
+    "grumpy_senior": 0.12,
+}
+REACT_RATE_DEFAULT = 0.20
+
+CATEGORIES = ["devlog", "showcase", "question", "rant", "fun", "random"]
+
+GIST_LANGUAGES = [
+    "python", "javascript", "typescript", "bash", "go", "rust",
+    "sql", "html", "css", "java", "c", "cpp", "ruby", "php", "lua",
+]
+
+FEED_TOPICS = ["devlog", "showcase", "question", "rant", "fun"]
+PROJECT_TYPES = ["game", "game_asset", "software", "mobile_app", "website"]
+
+SEARCH_TERMS = {
+    "enthusiastic_junior": ["vibe coding", "first app", "react", "python", "ai tools"],
+    "grumpy_senior": ["rust", "perl", "legacy", "tech debt", "kubernetes"],
+    "hobbyist_maker": ["arduino", "raspberry pi", "side project", "3d printing", "game"],
+    "academic_type": ["algorithms", "type theory", "distributed systems", "compilers"],
+    "minimalist": ["cli", "vim", "golang", "suckless"],
+    "storyteller": ["postmortem", "war story", "migration", "outage"],
+    "rebel": ["hot take", "overrated", "monorepo", "microservices"],
+    "mentor": ["best practices", "code review", "testing", "career"],
+}
+
+MIN_COMMENT_LEN = 40
+MIN_POST_LEN = 120
+RESTATEMENT_OVERLAP_THRESHOLD = 0.6
+
+GENERIC_COMMENT_PHRASES = [
+    "great point", "good point", "i agree", "totally agree", "well said",
+    "nice post", "great post", "thanks for sharing", "interesting read",
+    "this is interesting", "so true", "couldn't agree more", "spot on",
+    "great article", "love this", "this is great", "good read",
+]

devplacepy/services/bot/llm.py

@@ -0,0 +1,303 @@
+import json
+import logging
+import random
+import re
+import time
+import urllib.error
+import urllib.request
+
+from devplacepy.services.bot.config import CATEGORIES, GIST_LANGUAGES
+
+logger = logging.getLogger(__name__)
+
+
+class LLMClient:
+    def __init__(self, api_key: str, api_url: str, model: str,
+                 input_cost_per_1m: float, output_cost_per_1m: float):
+        if not api_key:
+            raise RuntimeError("LLM API key not set")
+        self.api_key = api_key
+        self.api_url = api_url
+        self.model = model
+        self.input_cost_per_1m = input_cost_per_1m
+        self.output_cost_per_1m = output_cost_per_1m
+        self.total_cost = 0.0
+        self.total_calls = 0
+        self.total_in_tokens = 0
+        self.total_out_tokens = 0
+
+    def _call(self, system: str, prompt: str, temperature: float = 0.7) -> str:
+        for attempt in range(3):
+            try:
+                payload = {
+                    "model": self.model,
+                    "messages": [{"role": "system", "content": system}, {"role": "user", "content": prompt}],
+                    "temperature": temperature,
+                }
+                logger.info("LLM >>> model=%s system=%s prompt=%s",
+                            self.model, json.dumps(system[:500]), json.dumps(prompt[:500]))
+                data = json.dumps(payload).encode()
+                req = urllib.request.Request(
+                    self.api_url, data=data,
+                    headers={"Content-Type": "application/json", "Authorization": f"Bearer {self.api_key}"},
+                )
+                with urllib.request.urlopen(req, timeout=30) as resp:
+                    raw = resp.read()
+                    logger.info("LLM <<< %s", raw[:2000].decode(errors="replace"))
+                    result = json.loads(raw)
+                    usage = result.get("usage", {})
+                    in_tokens = usage.get("prompt_tokens", 0)
+                    out_tokens = usage.get("completion_tokens", 0)
+                    cost = (in_tokens * self.input_cost_per_1m / 1_000_000) + \
+                           (out_tokens * self.output_cost_per_1m / 1_000_000)
+                    self.total_cost += cost
+                    self.total_calls += 1
+                    self.total_in_tokens += in_tokens
+                    self.total_out_tokens += out_tokens
+                    return self.clean(result["choices"][0]["message"]["content"])
+            except (urllib.error.HTTPError, urllib.error.URLError, json.JSONDecodeError, KeyError) as e:
+                logger.warning("LLM call attempt %d/3 failed: %s", attempt + 1, e)
+                if attempt == 2:
+                    raise
+                time.sleep(2 ** attempt)
+        return ""
+
+    @staticmethod
+    def clean(text: str, preserve_md: bool = False) -> str:
+        if not preserve_md:
+            text = re.sub(r"\*\*|__|\*|_", "", text)
+        text = text.replace("-", "-").replace("–", "-")
+        text = text.replace("‘", "'").replace("’", "'")
+        text = text.replace("“", '"').replace("”", '"')
+        text = re.sub(r"\s{2,}", " ", text)
+        return text.strip()
+
+    @staticmethod
+    def strip_md(text: str) -> str:
+        return LLMClient.clean(text)[:200]
+
+    @staticmethod
+    def strip_code_fences(text: str) -> str:
+        text = text.strip()
+        if text.startswith("```"):
+            lines = text.split("\n")
+            if lines and lines[0].lstrip().startswith("```"):
+                lines = lines[1:]
+            if lines and lines[-1].strip().startswith("```"):
+                lines = lines[:-1]
+            text = "\n".join(lines)
+        return text.strip()
+
+    def generate_post(self, title: str, desc: str, persona: str = "", category: str = "") -> str:
+        persona_extras = {
+            "enthusiastic_junior": "Be excited. Use **bold** for emphasis. Short excited sentences. End with a question sometimes.",
+            "grumpy_senior": "Be slightly cynical but helpful. Short blunt sentences. No fluff. Call out bad practices.",
+            "hobbyist_maker": "Be casual and friendly. Mention side projects and tinkering. Use *italics* for tools/libraries.",
+            "academic_type": "Be precise and structured. Use *italics* for terminology. Well thought out arguments.",
+            "storyteller": "Use **bold** for key points. Write anecdotal, narrative style. Longer flowing sentences.",
+            "rebel": "Be informal. Skip punctuation sometimes. Use slang. Type like you're in a hurry.",
+            "mentor": "Be helpful and explanatory. Use **bold** for key takeaways. Include practical advice.",
+            "minimalist": "One paragraph. Short blunt declarative sentences. No markdown. No fluff.",
+        }
+        category_extras = {
+            "devlog": "Write it as a personal devlog entry - share your learning process and insights about this topic.",
+            "showcase": "Write it as a showcase - express genuine excitement about what makes this impressive.",
+            "question": "Write it as a discussion starter - ask thoughtful questions, invite others to share perspectives.",
+            "rant": "Write it as an opinionated rant - strong viewpoint, passionate criticism, but keep it substantive.",
+            "fun": "Write it lighthearted and playful, but stay tied to the actual tech topic. Humor about the technology itself, not off-topic jokes or lyrics.",
+            "random": "Be natural and conversational - share your thoughts like any casual discussion.",
+        }
+        persona_extra = f" {persona_extras.get(persona, 'Be casual. No markdown.')}" if persona else " Be casual. No markdown."
+        category_extra = f" {category_extras.get(category, '')}" if category else ""
+        preserve = persona in ("enthusiastic_junior", "hobbyist_maker", "academic_type", "storyteller", "mentor")
+        text = self._call(
+            f"You are a dev writing a social media post reacting to tech news. Write 2-4 short paragraphs."
+            f"{persona_extra}{category_extra} Do not summarize the article; assume the reader already saw it. "
+            f"Add your own value: a concrete opinion, an implication, a personal experience, or a pointed question. "
+            f"Reference a specific detail rather than restating the headline. No em dashes. Under 300 words.",
+            f"News: {title}\n\n{desc[:800]}",
+        )
+        return self.clean(text, preserve_md=preserve)
+
+    def select_category(self, title: str, desc: str) -> str:
+        text = self._call(
+            "You are a classifier. Given tech news, choose the single best post category. Reply with ONLY the category name.",
+            f"Categories:\n- devlog: learning journey, building something, personal dev experience\n- showcase: impressive release, new tool, achievement worth highlighting\n- question: uncertainty, asking for advice, curiosity about implications\n- rant: frustration, bad practices, criticism of industry trends\n- fun: amusing, surprising, entertaining but not deeply serious\n- random: anything else that doesn't clearly fit above\n\nNews: {title}\n\n{desc[:600]}",
+            temperature=0.2,
+        )
+        for c in CATEGORIES:
+            if c in text.lower():
+                return c
+        return "random"
+
+    def select_reaction(self, content_snippet: str, persona: str = "") -> str:
+        from devplacepy.constants import REACTION_EMOJI
+
+        flavor = {
+            "enthusiastic_junior": "You react readily and warmly.",
+            "grumpy_senior": "You react rarely, only to genuinely notable content.",
+            "hobbyist_maker": "You react to anything hands-on, clever, or fun.",
+            "academic_type": "You react only to substantive, rigorous content.",
+            "minimalist": "You react very sparingly.",
+            "storyteller": "You react to anything with a human angle.",
+            "rebel": "You react to bold or contrarian takes.",
+            "mentor": "You react supportively to effort and learning.",
+        }.get(persona, "")
+        options = " ".join(REACTION_EMOJI)
+        verdict = self._call(
+            "You are a developer browsing a community feed. Decide whether a post deserves an "
+            "emoji reaction and which single emoji best fits its content and sentiment. "
+            f"{flavor} Reply with EXACTLY one of these emojis: {options} "
+            "or the word NONE if the content is mundane or would not move you to react. "
+            "Output only the emoji or NONE, nothing else.",
+            f"Post:\n{content_snippet[:1200]}",
+            temperature=0.4,
+        )
+        text = verdict or ""
+        for emoji in REACTION_EMOJI:
+            if emoji in text or emoji.replace("\ufe0f", "") in text:
+                return emoji
+        return ""
+
+    def generate_comment(self, post_snippet: str, persona: str = "",
+                         mention_target: str = "", parent_context: str = "") -> str:
+        extras = {
+            "enthusiastic_junior": "Be excited. Use **bold** for agreement. Short replies.",
+            "grumpy_senior": "Be blunt and short. No markdown. One sarcastic remark or actual advice.",
+            "rebel": "Super casual. Skip caps sometimes. Short.",
+            "mentor": "Be helpful. Use **bold** for key point.",
+            "storyteller": "Share a quick related story. Use *italics* for emphasis.",
+            "minimalist": "Shortest possible reply. One sentence max.",
+        }
+        extra = f" {extras.get(persona, 'Be casual. No markdown.')}" if persona else " Be casual. No markdown."
+        preserve = persona in ("enthusiastic_junior", "mentor", "storyteller")
+        mention_rule = ""
+        if mention_target:
+            mention_rule = (
+                f" Address @{mention_target} naturally in the first sentence "
+                f"(weave the @{mention_target} mention inline, do not append it at the end)."
+            )
+        ctx = ""
+        if parent_context:
+            ctx = f"\n\nReplying to this comment:\n{parent_context[:800]}"
+        system = (
+            f"You are a dev replying to a post. Write 1-3 short sentences.{extra}{mention_rule} "
+            "No em dashes. Reference a specific detail from the post. "
+            "Do not open with or rely on generic filler like 'great point', 'I agree', 'nice', "
+            "'thanks for sharing', or 'interesting'. Add something real: a concrete experience, "
+            "a caveat, a counterexample, respectful disagreement, or a pointed question. "
+            "Never just paraphrase or agree blandly."
+        )
+        reply = self.clean(
+            self._call(system, f"Post:\n{post_snippet[:1500]}{ctx}").strip(),
+            preserve_md=preserve,
+        )
+        return reply[:2000]
+
+    @staticmethod
+    def _overlap_ratio(text: str, context: str) -> float:
+        def tokens(s: str) -> list[str]:
+            return [w for w in re.findall(r"[a-z0-9]+", s.lower()) if len(w) > 3]
+
+        candidate = tokens(text)
+        source = set(tokens(context))
+        if not candidate or not source:
+            return 0.0
+        hits = sum(1 for w in candidate if w in source)
+        return hits / len(candidate)
+
+    def quality_check(self, kind: str, text: str, context: str = "") -> tuple[bool, str]:
+        from devplacepy.services.bot.config import (
+            MIN_COMMENT_LEN, MIN_POST_LEN, RESTATEMENT_OVERLAP_THRESHOLD, GENERIC_COMMENT_PHRASES,
+        )
+        stripped = self.clean(text or "").strip()
+        min_len = MIN_COMMENT_LEN if kind == "comment" else MIN_POST_LEN
+        if len(stripped) < min_len:
+            return False, f"too short ({len(stripped)} < {min_len})"
+        lowered = stripped.lower()
+        if kind == "comment":
+            for phrase in GENERIC_COMMENT_PHRASES:
+                if phrase in lowered:
+                    return False, f"generic phrase '{phrase}'"
+        if context and self._overlap_ratio(stripped, context) > RESTATEMENT_OVERLAP_THRESHOLD:
+            return False, "restates the source"
+        verdict = self._call(
+            "You are a strict content quality reviewer for a developer community. "
+            "Reject text that is generic, low-effort, pure filler, or merely summarizes its "
+            "source without adding an opinion, experience, or insight. "
+            "Reply with exactly PASS or 'FAIL: <short reason>'.",
+            f"Type: {kind}\nText:\n{stripped[:1200]}",
+            temperature=0.0,
+        )
+        if verdict.strip().upper().startswith("PASS"):
+            return True, "ok"
+        return False, verdict.strip()[:120] or "judge rejected"
+
+    def generate_bug(self, topic: str) -> tuple[str, str]:
+        text = self._call(
+            "Write a bug report. One line title. Then 2-3 sentences describing what happened and what should have happened. Like a real dev reporting a bug. No em dashes.",
+            f"Bug: {topic}",
+        )
+        lines = text.strip().split("\n")
+        title = lines[0].strip().lstrip("#").strip()[:120] or f"Bug report: {topic}"
+        desc = " ".join(line.lstrip("-* ").strip() for line in lines[1:] if line.strip())
+        return title, desc[:2000] or text[:500]
+
+    def generate_bio(self) -> str:
+        return self._call(
+            "Write a short bio for a developer profile. 1-2 sentences. What they work on, what they like. Sound human. No em dashes.",
+            "Bio:",
+        )
+
+    def generate_profile_fields(self, handle: str) -> tuple[str, str, str]:
+        location = self.clean(self._call(
+            "Name one plausible city and country for a developer. Reply with ONLY 'City, Country'. No extra words.",
+            "Location:", temperature=0.9,
+        ))[:80]
+        slug = re.sub(r"[^a-z0-9_-]", "", handle.lower()) or "dev"
+        git_link = f"https://github.com/{slug}"
+        website = f"https://{slug}.dev"
+        return location, git_link, website
+
+    def generate_dm(self, persona: str = "", context: str = "") -> str:
+        extra = {
+            "enthusiastic_junior": "Be friendly and excited.",
+            "grumpy_senior": "Be blunt but not rude.",
+            "rebel": "Be super casual.",
+            "mentor": "Be warm and encouraging.",
+            "storyteller": "Open with a small hook.",
+            "minimalist": "One short sentence.",
+        }.get(persona, "Be casual and friendly.")
+        ctx = f"\n\nEarlier message:\n{context[:400]}" if context else ""
+        return self.clean(self._call(
+            f"Write one short, friendly direct message to another developer. 1-2 sentences. {extra} No em dashes.",
+            f"Write a DM to start or continue a chat.{ctx}",
+        ))[:500]
+
+    def generate_project_title(self) -> str:
+        return self._call("Come up with a project name. 2-4 words. A tool, game, or app idea. No em dashes.", "Name:", temperature=0.9)
+
+    def generate_project_desc(self, title: str) -> str:
+        return self._call(
+            "You are a developer. Write a compelling 2-3 sentence project description including tech stack and purpose.",
+            f"Project: {title}",
+        )
+
+    def generate_gist(self, persona: str = "") -> tuple[str, str, str, str]:
+        language = random.choice(GIST_LANGUAGES)
+        title = self.clean(self._call(
+            "Name a short, useful code snippet. 2-5 words. No quotes. No markdown.",
+            f"Language: {language}. Snippet name:",
+            temperature=0.9,
+        ))[:120]
+        description = self.clean(self._call(
+            "Write a one-sentence description of what a code snippet does, like a dev sharing something handy. No em dashes.",
+            f"Snippet: {title}\nLanguage: {language}",
+        ))[:400]
+        code = self.strip_code_fences(self._call(
+            f"Write a short, correct, self-contained {language} snippet of 5 to 20 lines for the description. "
+            "Output ONLY raw code. No markdown fences. No commentary.",
+            f"Title: {title}\nDescription: {description}\nLanguage: {language}",
+            temperature=0.4,
+        ))[:4000]
+        return title, description, language, code

devplacepy/services/bot/news_fetcher.py

@@ -0,0 +1,31 @@
+import json
+import logging
+import time
+import urllib.request
+
+logger = logging.getLogger(__name__)
+
+
+class NewsFetcher:
+    TTL_SECONDS = 1800
+
+    def __init__(self, news_api: str):
+        self.news_api = news_api
+        self.articles: list[dict] = []
+        self.fetched_at: float = 0.0
+
+    def fetch(self) -> list[dict]:
+        fresh = self.articles and (time.monotonic() - self.fetched_at) < self.TTL_SECONDS
+        if fresh:
+            return self.articles
+        try:
+            req = urllib.request.Request(self.news_api, headers={"User-Agent": "dpbot/1.0"})
+            with urllib.request.urlopen(req, timeout=10) as resp:
+                articles = json.loads(resp.read()).get("articles", [])
+            if articles:
+                self.articles = articles
+                self.fetched_at = time.monotonic()
+                logger.info("Fetched %d news articles", len(self.articles))
+        except Exception as e:
+            logger.warning("News fetch failed: %s", e)
+        return self.articles

devplacepy/services/bot/registry.py

@@ -0,0 +1,94 @@
+import fcntl
+import json
+import logging
+import os
+import random
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+
+class ArticleRegistry:
+    def __init__(self, path: Path):
+        self.path = path
+
+    def _lock(self) -> tuple:
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        try:
+            fd = os.open(str(self.path), os.O_RDWR | os.O_CREAT)
+            fcntl.flock(fd, fcntl.LOCK_EX)
+            data = {}
+            if os.path.getsize(str(self.path)) > 0:
+                chunks = []
+                while True:
+                    chunk = os.read(fd, 65536)
+                    if not chunk:
+                        break
+                    chunks.append(chunk)
+                raw = b"".join(chunks).decode(errors="replace")
+                if raw.strip():
+                    try:
+                        data = json.loads(raw)
+                    except json.JSONDecodeError:
+                        data = {}
+            if not isinstance(data, dict):
+                data = {}
+            now = datetime.now()
+            kept: dict = {}
+            for title, meta in data.items():
+                if not isinstance(meta, dict):
+                    continue
+                ts = meta.get("time", "")
+                try:
+                    age_days = (now - datetime.fromisoformat(ts)).total_seconds() / 86400
+                except (ValueError, TypeError):
+                    age_days = 0
+                if age_days < 7:
+                    kept[title] = meta
+            return fd, kept
+        except Exception as e:
+            logger.warning("Failed to lock article registry: %s", e)
+            return None, {}
+
+    @staticmethod
+    def _unlock(fd) -> None:
+        if fd is not None:
+            try:
+                fcntl.flock(fd, fcntl.LOCK_UN)
+                os.close(fd)
+            except Exception as e:
+                logger.debug("Failed to unlock article registry: %s", e)
+
+    def reserve(self, title: str, owner: str) -> bool:
+        fd, registry = self._lock()
+        if fd is None:
+            return True
+        try:
+            if title in registry:
+                return False
+            registry[title] = {"bot": owner, "time": datetime.now().isoformat()}
+            self.path.write_text(json.dumps(registry, indent=2, default=str))
+            return True
+        finally:
+            self._unlock(fd)
+
+    def reserve_unused(self, articles: list[dict], owner: str, clean) -> Optional[dict]:
+        if not articles:
+            return None
+        fd, registry = self._lock()
+        if fd is None:
+            return random.choice(articles) if articles else None
+        try:
+            used = set(registry.keys())
+            random.shuffle(articles)
+            for a in articles:
+                title = clean(a.get("title", "")[:200])
+                if title and title not in used:
+                    registry[title] = {"bot": owner, "time": datetime.now().isoformat()}
+                    self.path.write_text(json.dumps(registry, indent=2, default=str))
+                    return a
+            return None
+        finally:
+            self._unlock(fd)

devplacepy/services/bot/runtime.py

@@ -0,0 +1,18 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+from devplacepy.services.bot import config
+
+
+@dataclass
+class BotRuntimeConfig:
+    state_path: Path
+    base_url: str = config.BASE_URL_DEFAULT
+    api_url: str = config.API_URL_DEFAULT
+    news_api: str = config.NEWS_API_DEFAULT
+    model: str = config.MODEL_DEFAULT
+    api_key: str = ""
+    input_cost_per_1m: float = config.INPUT_COST_PER_1M_DEFAULT
+    output_cost_per_1m: float = config.OUTPUT_COST_PER_1M_DEFAULT
+    headless: bool = True
+    registry_path: Path = config.ARTICLE_REGISTRY_PATH

devplacepy/services/bot/service.py

@@ -0,0 +1,241 @@
+import asyncio
+import importlib
+import logging
+from collections import deque
+from datetime import datetime, timedelta, timezone
+
+from devplacepy.services.base import BaseService, ConfigField
+from devplacepy.services.bot import config
+
+logger = logging.getLogger(__name__)
+
+
+class BotsService(BaseService):
+    default_enabled = False
+    min_interval = 5
+    title = "Bots"
+    description = (
+        "Runs a fleet of Playwright-driven AI personas that browse a DevPlace "
+        "instance and post, comment, vote, and react like real users. Fleet size, "
+        "target site, and LLM settings are configurable, with live cost and usage "
+        "metrics per bot."
+    )
+    config_fields = [
+        ConfigField("bot_fleet_size", "Fleet size (bots)", type="int", default=1, minimum=0, maximum=20,
+                    help="Number of concurrent bot accounts to run.", group="Fleet"),
+        ConfigField("bot_headless", "Headless browser", type="bool", default=True,
+                    help="Run Chromium headless. Disable only for local debugging on a desktop.",
+                    group="Fleet"),
+        ConfigField("bot_max_actions", "Max actions per bot", type="int", default=0, minimum=0,
+                    help="Stop a bot after this many actions (0 = run indefinitely).", group="Fleet"),
+        ConfigField("bot_base_url", "Target site URL", type="url", default=config.BASE_URL_DEFAULT,
+                    help="DevPlace instance the bots browse and post to.", group="Target"),
+        ConfigField("bot_news_api", "News API URL", type="url", default=config.NEWS_API_DEFAULT,
+                    help="Source of articles the bots react to.", group="Target"),
+        ConfigField("bot_api_url", "LLM API URL", type="url", default=config.API_URL_DEFAULT,
+                    help="OpenAI-compatible chat-completions endpoint.", group="LLM"),
+        ConfigField("bot_model", "LLM model", type="str", default=config.MODEL_DEFAULT,
+                    help="Model name sent to the LLM API.", group="LLM"),
+        ConfigField("bot_api_key", "LLM API key", type="password", default="", secret=True,
+                    help="Defaults to the gateway's internal key (the bots use the local AI gateway).",
+                    group="LLM"),
+        ConfigField("bot_input_cost_per_1m", "Input cost per 1M tokens ($)", type="float",
+                    default=config.INPUT_COST_PER_1M_DEFAULT, minimum=0,
+                    help="Used to compute live spend per bot and across the fleet.", group="LLM"),
+        ConfigField("bot_output_cost_per_1m", "Output cost per 1M tokens ($)", type="float",
+                    default=config.OUTPUT_COST_PER_1M_DEFAULT, minimum=0,
+                    help="Used to compute live spend per bot and across the fleet.", group="LLM"),
+    ]
+
+    def __init__(self):
+        super().__init__(name="bots", interval_seconds=15)
+        self._fleet = {}
+        self._state_dir = config.STATE_DIR
+        self._cost_samples = deque()
+        self._cost_anchor = None
+
+    def _resolve_api_key(self, cfg: dict) -> str:
+        from devplacepy.database import internal_gateway_key
+        return cfg["bot_api_key"] or internal_gateway_key()
+
+    @staticmethod
+    def _ensure_importable() -> None:
+        importlib.import_module("devplacepy.services.bot.bot")
+
+    async def run_once(self) -> None:
+        cfg = self.get_config()
+        desired = cfg["bot_fleet_size"]
+        api_key = self._resolve_api_key(cfg)
+        if not api_key:
+            if self._fleet:
+                await self._stop_fleet()
+            self.log("No LLM API key available (bot_api_key / gateway internal key); fleet idle")
+            return
+        try:
+            self._ensure_importable()
+        except ImportError as e:
+            if self._fleet:
+                await self._stop_fleet()
+            self.log(f"Bot dependencies unavailable ({e}); install the 'bots' extra")
+            return
+
+        for slot in list(self._fleet):
+            task = self._fleet[slot]["task"]
+            if task.done():
+                self._report_exit(slot, task)
+                del self._fleet[slot]
+
+        while len(self._fleet) > desired:
+            await self._stop_slot(max(self._fleet))
+
+        for slot in range(desired):
+            if slot not in self._fleet:
+                self._launch_slot(slot, cfg, api_key)
+
+        self.log(f"Fleet: {len(self._fleet)}/{desired} bots active")
+
+    def _report_exit(self, slot: int, task) -> None:
+        if task.cancelled():
+            return
+        exc = task.exception()
+        if exc:
+            self.log(f"bot{slot} exited with error: {exc}; relaunching")
+        else:
+            self.log(f"bot{slot} finished; relaunching")
+
+    def _launch_slot(self, slot: int, cfg: dict, api_key: str) -> None:
+        from devplacepy.services.bot.bot import DevPlaceBot
+        from devplacepy.services.bot.runtime import BotRuntimeConfig
+        rc = BotRuntimeConfig(
+            state_path=self._state_dir / f"state_slot{slot}.json",
+            base_url=cfg["bot_base_url"],
+            api_url=cfg["bot_api_url"],
+            news_api=cfg["bot_news_api"],
+            model=cfg["bot_model"],
+            api_key=api_key,
+            input_cost_per_1m=cfg["bot_input_cost_per_1m"],
+            output_cost_per_1m=cfg["bot_output_cost_per_1m"],
+            headless=cfg["bot_headless"],
+            registry_path=config.ARTICLE_REGISTRY_PATH,
+        )
+        bot = DevPlaceBot(rc, on_event=self.log)
+        task = asyncio.create_task(self._run_slot(slot, bot, cfg["bot_max_actions"]))
+        self._fleet[slot] = {"bot": bot, "task": task}
+        self.log(f"Launched bot{slot}")
+
+    async def _run_slot(self, slot: int, bot, max_actions: int) -> None:
+        try:
+            await bot.run_forever(action_limit=max_actions)
+        except asyncio.CancelledError:
+            raise
+        except Exception as e:
+            self.log(f"bot{slot} crashed: {e}")
+
+    async def on_disable(self) -> None:
+        await self._stop_fleet()
+
+    async def _stop_fleet(self) -> None:
+        slots = list(self._fleet)
+        if slots:
+            await asyncio.gather(*(self._stop_slot(slot) for slot in slots), return_exceptions=True)
+
+    async def _stop_slot(self, slot: int) -> None:
+        entry = self._fleet.pop(slot, None)
+        if not entry:
+            return
+        task = entry["task"]
+        task.cancel()
+        try:
+            await asyncio.wait_for(task, timeout=30)
+        except (asyncio.CancelledError, asyncio.TimeoutError):
+            pass
+        except Exception as e:
+            self.log(f"bot{slot} stop error: {e}")
+        await self._close_bot(slot, entry.get("bot"))
+        self.log(f"Stopped bot{slot}")
+
+    async def _close_bot(self, slot: int, bot) -> None:
+        browser = getattr(bot, "b", None)
+        if browser is None:
+            return
+        try:
+            await asyncio.wait_for(browser.close(), timeout=15)
+        except (asyncio.CancelledError, asyncio.TimeoutError):
+            pass
+        except Exception as e:
+            self.log(f"bot{slot} browser close error: {e}")
+
+    def collect_metrics(self) -> dict:
+        rows = []
+        total_cost = 0.0
+        total_calls = total_in = total_out = 0
+        posts = comments = votes = running = 0
+        for slot in sorted(self._fleet):
+            entry = self._fleet[slot]
+            bot = entry["bot"]
+            alive = not entry["task"].done()
+            running += 1 if alive else 0
+            llm = getattr(bot, "llm", None)
+            st = getattr(bot, "state", None)
+            cost = llm.total_cost if llm else 0.0
+            calls = llm.total_calls if llm else 0
+            total_cost += cost
+            total_calls += calls
+            total_in += llm.total_in_tokens if llm else 0
+            total_out += llm.total_out_tokens if llm else 0
+            if st:
+                posts += st.created_posts
+                comments += st.comments_posted
+                votes += st.votes_cast
+            rows.append([
+                f"bot{slot}",
+                (st.username if st else "") or "-",
+                (st.persona if st else "") or "-",
+                "running" if alive else "stopped",
+                st.created_posts if st else 0,
+                st.comments_posted if st else 0,
+                st.votes_cast if st else 0,
+                calls,
+                f"${cost:.4f}",
+            ])
+        window, observed_cost, ready = self._sample_cost(datetime.now(timezone.utc), total_cost)
+        cost_per_hour = observed_cost / window * 3600 if ready else 0.0
+        projected_24h = observed_cost / window * 86400 if ready else 0.0
+        rate_value = f"${cost_per_hour:.4f}/h" if ready else "warming up"
+        projected_value = f"${projected_24h:.2f}" if ready else "warming up"
+        stats = [
+            {"label": "Bots running", "value": running},
+            {"label": "Fleet cost", "value": f"${total_cost:.4f}"},
+            {"label": "Observed window", "value": f"{window:.0f}s"},
+            {"label": "Cost rate", "value": rate_value},
+            {"label": "Projected 24h cost", "value": projected_value},
+            {"label": "LLM calls", "value": total_calls},
+            {"label": "Tokens in", "value": total_in},
+            {"label": "Tokens out", "value": total_out},
+            {"label": "Posts", "value": posts},
+            {"label": "Comments", "value": comments},
+            {"label": "Votes", "value": votes},
+        ]
+        table = {
+            "columns": ["Bot", "User", "Persona", "Status", "Posts", "Comments", "Votes", "Calls", "Cost"],
+            "rows": rows,
+        }
+        return {"stats": stats, "table": table}
+
+    def _sample_cost(self, now: datetime, total_cost: float) -> tuple[float, float, bool]:
+        if self._started_at is None:
+            self._cost_samples.clear()
+            self._cost_anchor = None
+            return 0.0, 0.0, False
+        if self._cost_anchor != self._started_at:
+            self._cost_samples.clear()
+            self._cost_anchor = self._started_at
+        self._cost_samples.append((now, total_cost))
+        cutoff = now - timedelta(seconds=config.COST_WINDOW_SECONDS)
+        while len(self._cost_samples) > 2 and self._cost_samples[0][0] < cutoff:
+            self._cost_samples.popleft()
+        start_at, start_cost = self._cost_samples[0]
+        window = (now - start_at).total_seconds()
+        observed_cost = max(0.0, total_cost - start_cost)
+        ready = window >= config.COST_WARMUP_SECONDS
+        return window, observed_cost, ready

devplacepy/services/bot/state.py

@@ -0,0 +1,65 @@
+import json
+import logging
+import os
+from dataclasses import dataclass, field, asdict
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class BotState:
+    username: str = ""
+    email: str = ""
+    password: str = ""
+    uid: str = ""
+    persona: str = ""
+    log: list[str] = field(default_factory=list)
+    known_users: list[str] = field(default_factory=list)
+    known_posts: list[str] = field(default_factory=list)
+    created_posts: int = 0
+    posted_content_hashes: list[str] = field(default_factory=list)
+    comments_posted: int = 0
+    quality_rejections: int = 0
+    votes_cast: int = 0
+    bugs_filed: int = 0
+    projects_created: int = 0
+    gists_created: int = 0
+    profiles_viewed: int = 0
+    started_at: str = ""
+    own_post_urls: list[str] = field(default_factory=list)
+    voted_post_ids: list[str] = field(default_factory=list)
+    commented_post_ids: list[str] = field(default_factory=list)
+    notifications_seen: list[str] = field(default_factory=list)
+    mentions_received: int = 0
+    mentions_replied: int = 0
+    replies_to_own_posts: int = 0
+    news_comments: int = 0
+    projects_starred: int = 0
+    messages_sent: int = 0
+    searches_run: int = 0
+    poll_votes_cast: int = 0
+    reactions_cast: int = 0
+    polls_voted: list[str] = field(default_factory=list)
+    reacted_targets: list[str] = field(default_factory=list)
+    reading_speed_wpm: int = 0
+    total_cost: float = 0.0
+    total_calls: int = 0
+    total_in_tokens: int = 0
+    total_out_tokens: int = 0
+
+    def save(self, path: Path) -> None:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        tmp = path.with_suffix(path.suffix + ".tmp")
+        tmp.write_text(json.dumps(asdict(self), indent=2, default=str))
+        os.replace(str(tmp), str(path))
+
+    @classmethod
+    def load(cls, path: Path) -> "BotState":
+        if path.exists():
+            try:
+                data = json.loads(path.read_text())
+                return cls(**data)
+            except Exception as e:
+                logger.warning("Failed to load state: %s", e)
+        return cls()

devplacepy/services/containers/__init__.py

@@ -0,0 +1 @@
+# retoor <retoor@molodetz.nl>

devplacepy/services/containers/api.py

@@ -0,0 +1,331 @@
+# retoor <retoor@molodetz.nl>
+
+import asyncio
+import json
+import re
+import socket
+from pathlib import Path
+
+import httpx
+
+from devplacepy import config, project_files
+from devplacepy.services.containers import store
+from devplacepy.services.containers.backend.base import Mount, PortMapping, RunSpec
+from devplacepy.services.containers.runtime import get_backend
+from devplacepy.services.devii.tasks.schedule import Schedule, next_run, now_utc, to_iso, from_iso
+
+IMAGE_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{0,62}$")
+INGRESS_SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9-]{0,62}$")
+MEM_RE = re.compile(r"^\d+(\.\d+)?[bkmgBKMG]?$")
+CPU_RE = re.compile(r"^\d+(\.\d+)?$")
+ENV_KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+INSTANCE_LABEL = "devplace.instance"
+PROJECT_LABEL = "devplace.project"
+HOST_PORT_MIN = 20001
+HOST_PORT_MAX = 65535
+
+
+class ContainerError(ValueError):
+    pass
+
+
+def _validate_limits(cpu_limit: str, mem_limit: str) -> None:
+    if cpu_limit and not CPU_RE.match(str(cpu_limit)):
+        raise ContainerError("cpu limit must be a number, e.g. 1 or 1.5")
+    if mem_limit and not MEM_RE.match(str(mem_limit)):
+        raise ContainerError("memory limit must look like 512m, 1g, or a byte count")
+
+
+def parse_ports(value) -> list:
+    ports = []
+    if not value:
+        return ports
+    items = value if isinstance(value, list) else str(value).replace(",", "\n").splitlines()
+    for item in items:
+        item = str(item).strip()
+        if not item:
+            continue
+        proto = "tcp"
+        if "/" in item:
+            item, proto = item.split("/", 1)
+        if ":" in item:
+            host, container = item.split(":", 1)
+        else:
+            host, container = "0", item
+        if not host.isdigit() or not container.isdigit():
+            raise ContainerError(f"port '{item}' must be numeric host:container or a bare container port")
+        ports.append(PortMapping(int(host), int(container), proto.strip() or "tcp"))
+    return ports
+
+
+def used_host_ports() -> set:
+    ports = set()
+    for instance in store.all_instances():
+        for mapping in json.loads(instance.get("ports_json") or "[]"):
+            host = int(mapping.get("host") or 0)
+            if host:
+                ports.add(host)
+    return ports
+
+
+def _host_port_free(port: int) -> bool:
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        try:
+            sock.bind(("0.0.0.0", port))
+            return True
+        except OSError:
+            return False
+
+
+def allocate_host_port(reserved: set) -> int:
+    for port in range(HOST_PORT_MIN, HOST_PORT_MAX + 1):
+        if port in reserved:
+            continue
+        if _host_port_free(port):
+            return port
+    raise ContainerError(f"no free host port available in range {HOST_PORT_MIN}-{HOST_PORT_MAX}")
+
+
+def assign_host_ports(port_list: list) -> list:
+    reserved = used_host_ports()
+    assigned = []
+    for mapping in port_list:
+        host = mapping.host
+        if not host:
+            host = allocate_host_port(reserved)
+        elif host in reserved:
+            raise ContainerError(f"host port {host} is already published by another instance")
+        reserved.add(host)
+        assigned.append(PortMapping(host, mapping.container, mapping.proto))
+    return assigned
+
+
+def parse_env(value) -> dict:
+    env = {}
+    if not value:
+        return env
+    if isinstance(value, dict):
+        items = value.items()
+    else:
+        items = (line.split("=", 1) for line in str(value).splitlines() if "=" in line)
+    for key, val in items:
+        key = str(key).strip()
+        if not ENV_KEY_RE.match(key):
+            raise ContainerError(f"invalid environment variable name: {key}")
+        env[key] = str(val)
+    return env
+
+
+# ---------------- instances ----------------
+
+def validate_ingress(slug: str, port, port_list) -> tuple:
+    slug = (slug or "").strip().lower()
+    if not slug:
+        return "", 0
+    if not INGRESS_SLUG_RE.match(slug):
+        raise ContainerError("ingress slug must be lowercase letters, digits, or '-' (max 63 chars)")
+    for other in store.all_instances():
+        if other.get("ingress_slug") == slug:
+            raise ContainerError(f"ingress slug '{slug}' is already in use")
+    ingress_port = int(port) if port else 0
+    container_ports = {p.container for p in port_list}
+    if ingress_port and ingress_port not in container_ports:
+        raise ContainerError(f"ingress_port {ingress_port} must be one of the container ports you mapped")
+    if not ingress_port and len(container_ports) != 1:
+        raise ContainerError("set ingress_port to choose which mapped container port to publish")
+    return slug, ingress_port
+
+
+async def create_instance(project: dict, *, name: str,
+                          boot_command: str = "", env="", cpu_limit: str = "", mem_limit: str = "",
+                          ports="", volumes="", restart_policy: str = "never",
+                          autostart: bool = True, ingress_slug: str = "", ingress_port=None,
+                          actor=("system", "system")) -> dict:
+    if not await get_backend().image_exists(config.CONTAINER_IMAGE):
+        raise ContainerError(f"the '{config.CONTAINER_IMAGE}' image is not built - run 'make ppy'")
+    if restart_policy not in store.RESTART_POLICIES:
+        raise ContainerError(f"restart policy must be one of {', '.join(store.RESTART_POLICIES)}")
+    _validate_limits(cpu_limit, mem_limit)
+    port_list = assign_host_ports(parse_ports(ports))
+    env_map = parse_env(env)
+    name = (name or "").strip()
+    if not name:
+        raise ContainerError("instance name is required")
+    ingress_slug, ingress_port = validate_ingress(ingress_slug, ingress_port, port_list)
+
+    workspace = Path(config.CONTAINER_WORKSPACES_DIR) / project["uid"]
+    await asyncio.to_thread(project_files.export_to_dir, project["uid"], "", str(workspace))
+
+    row = {
+        "project_uid": project["uid"],
+        "created_by": actor[1] if actor and actor[0] == "user" else "",
+        "owner_uid": project.get("user_uid", ""),
+        "name": name, "boot_command": boot_command or "",
+        "env_json": json.dumps(env_map),
+        "cpu_limit": str(cpu_limit or ""), "mem_limit": str(mem_limit or ""),
+        "ports_json": json.dumps([{"host": p.host, "container": p.container, "proto": p.proto} for p in port_list]),
+        "volumes_json": volumes if isinstance(volumes, str) else json.dumps(volumes or []),
+        "restart_policy": restart_policy,
+        "ingress_slug": ingress_slug, "ingress_port": ingress_port,
+        "desired_state": store.DESIRED_RUNNING if autostart else store.DESIRED_STOPPED,
+        "status": store.ST_CREATED,
+        "workspace_dir": str(workspace),
+    }
+    instance = store.create_instance(row)
+    store.record_event(instance, "created", actor[0], actor[1], {"image": config.CONTAINER_IMAGE})
+    return instance
+
+
+def set_desired_state(instance: dict, desired: str, *, actor=("system", "system")) -> dict:
+    if desired not in (store.DESIRED_RUNNING, store.DESIRED_STOPPED, store.DESIRED_PAUSED):
+        raise ContainerError("desired state must be running, stopped, or paused")
+    store.update_instance(instance["uid"], {"desired_state": desired})
+    store.record_event(instance, f"desire_{desired}", actor[0], actor[1])
+    return store.get_instance(instance["uid"])
+
+
+def request_restart(instance: dict, *, actor=("system", "system")) -> dict:
+    store.update_instance(instance["uid"], {"desired_state": store.DESIRED_RUNNING, "status": store.ST_RESTARTING})
+    store.record_event(instance, "restart", actor[0], actor[1])
+    return store.get_instance(instance["uid"])
+
+
+def mark_for_removal(instance: dict, *, actor=("system", "system")) -> None:
+    store.update_instance(instance["uid"], {"desired_state": store.DESIRED_STOPPED, "status": store.ST_REMOVING})
+    store.record_event(instance, "remove", actor[0], actor[1])
+
+
+def pravda_env(instance: dict) -> dict:
+    from devplacepy import database, seo
+    base_url = seo.public_base_url()
+    api_key = ""
+    for uid in (instance.get("created_by"), instance.get("owner_uid")):
+        if not uid:
+            continue
+        user = database.get_users_by_uids([uid]).get(uid)
+        if user and user.get("api_key"):
+            api_key = user["api_key"]
+            break
+    slug = instance.get("ingress_slug") or ""
+    ingress_url = (f"{base_url}/p/{slug}" if base_url else f"/p/{slug}") if slug else ""
+    return {
+        "PRAVDA_BASE_URL": base_url,
+        "PRAVDA_OPENAI_URL": f"{base_url}/openai/v1" if base_url else "",
+        "PRAVDA_API_KEY": api_key,
+        "PRAVDA_USER_UID": instance.get("owner_uid") or "",
+        "PRAVDA_CONTAINER_NAME": instance.get("name") or "",
+        "PRAVDA_CONTAINER_UID": instance.get("uid") or "",
+        "PRAVDA_INGRESS_URL": ingress_url,
+    }
+
+
+def run_spec_for(instance: dict, image_tag: str) -> RunSpec:
+    env = {**json.loads(instance.get("env_json") or "{}"), **pravda_env(instance)}
+    ports = [PortMapping(p["host"], p["container"], p.get("proto", "tcp"))
+             for p in json.loads(instance.get("ports_json") or "[]")]
+    mounts = [Mount(instance["workspace_dir"], "/app", "rw")]
+    for extra in json.loads(instance.get("volumes_json") or "[]"):
+        if isinstance(extra, dict) and extra.get("host") and extra.get("container"):
+            mounts.append(Mount(extra["host"], extra["container"], extra.get("mode", "rw")))
+    boot = (instance.get("boot_command") or "").strip()
+    command = ["/bin/sh", "-c", boot] if boot else ["sleep", "infinity"]
+    return RunSpec(
+        image=image_tag, name=instance["slug"],
+        labels={INSTANCE_LABEL: instance["uid"], PROJECT_LABEL: instance["project_uid"]},
+        env=env, cpu_limit=instance.get("cpu_limit", ""), mem_limit=instance.get("mem_limit", ""),
+        ports=ports, mounts=mounts, restart_policy=instance.get("restart_policy", "never"), command=command,
+    )
+
+
+async def sync_workspace(instance: dict, user: dict) -> int:
+    workspace = instance.get("workspace_dir")
+    if not workspace:
+        raise ContainerError("instance has no workspace")
+    count = await asyncio.to_thread(project_files.import_from_dir, instance["project_uid"], workspace, user)
+    store.record_event(instance, "sync", "user", user["uid"], {"imported": count})
+    return count
+
+
+def add_schedule(instance: dict, action: str, schedule: Schedule) -> dict:
+    if action not in ("start", "stop"):
+        raise ContainerError("schedule action must be start or stop")
+    first = schedule.first_run(now_utc())
+    return store.create_schedule(instance, action, schedule.columns(), to_iso(first))
+
+
+# ---------------- aggregation ----------------
+
+def _percentile(values: list, pct: float) -> float:
+    if not values:
+        return 0.0
+    ordered = sorted(values)
+    index = min(len(ordered) - 1, int(round((pct / 100.0) * (len(ordered) - 1))))
+    return float(ordered[index])
+
+
+def instance_stats(instance_uid: str) -> dict:
+    metrics = store.recent_metrics(instance_uid, limit=720)
+    cpu = [m.get("cpu_pct", 0) for m in metrics]
+    mem = [m.get("mem_bytes", 0) for m in metrics]
+    return {
+        "samples": len(metrics),
+        "cpu_avg": round(sum(cpu) / len(cpu), 2) if cpu else 0.0,
+        "cpu_p95": round(_percentile(cpu, 95), 2),
+        "mem_max": max(mem) if mem else 0,
+        "mem_avg": int(sum(mem) / len(mem)) if mem else 0,
+    }
+
+
+def _port_reachable(host: str, port: int, timeout: float = 0.3) -> bool:
+    try:
+        with socket.create_connection((host, port), timeout=timeout):
+            return True
+    except OSError:
+        return False
+
+
+def _http_probe(host: str, port: int, timeout: float = 1.0) -> str:
+    try:
+        with httpx.Client(timeout=timeout) as client:
+            response = client.get(f"http://{host}:{port}/")
+        return f"HTTP {response.status_code}"
+    except Exception as exc:  # noqa: BLE001 - diagnostic, any failure is informative
+        return f"unreachable: {type(exc).__name__}"
+
+
+def _ingress_host_port(instance: dict, port_maps: list) -> int:
+    ingress_port = int(instance.get("ingress_port") or 0)
+    if ingress_port:
+        for mapping in port_maps:
+            if int(mapping.get("container") or 0) == ingress_port:
+                return int(mapping.get("host") or 0)
+        return 0
+    return int(port_maps[0].get("host") or 0) if port_maps else 0
+
+
+def instance_runtime(instance: dict) -> dict:
+    boot = (instance.get("boot_command") or "").strip()
+    port_maps = json.loads(instance.get("ports_json") or "[]")
+    host = config.CONTAINER_PROXY_HOST
+    ports = []
+    for mapping in port_maps:
+        host_port = int(mapping.get("host") or 0)
+        ports.append({
+            "container": int(mapping.get("container") or 0),
+            "host": host_port,
+            "proto": mapping.get("proto", "tcp"),
+            "reachable": _port_reachable(host, host_port) if host_port else False,
+        })
+    ingress_host_port = _ingress_host_port(instance, port_maps)
+    ingress_serving = _http_probe(host, ingress_host_port) if ingress_host_port else "no ingress port mapped"
+    return {
+        "command": boot or "image CMD (no boot_command set)",
+        "ports": ports,
+        "ingress_port": int(instance.get("ingress_port") or 0),
+        "ingress_serving": ingress_serving,
+        "status_ok_for_ingress": instance.get("status") == store.ST_RUNNING,
+        "restart_count": int(instance.get("restart_count") or 0),
+        "exit_code": instance.get("exit_code"),
+        "container_id": (instance.get("container_id") or "")[:12],
+    }

devplacepy/services/containers/backend/__init__.py

@@ -0,0 +1,23 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.services.containers.backend.base import (
+    Backend,
+    BuildResult,
+    ExecResult,
+    Mount,
+    PortMapping,
+    PsRow,
+    RunSpec,
+    StatsSample,
+)
+
+__all__ = [
+    "Backend",
+    "BuildResult",
+    "ExecResult",
+    "Mount",
+    "PortMapping",
+    "PsRow",
+    "RunSpec",
+    "StatsSample",
+]

devplacepy/services/containers/backend/base.py

@@ -0,0 +1,124 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Awaitable, Callable, Optional
+
+LogCallback = Callable[[str], Awaitable[None]]
+
+
+@dataclass
+class PortMapping:
+    host: int
+    container: int
+    proto: str = "tcp"
+
+
+@dataclass
+class Mount:
+    host: str
+    container: str
+    mode: str = "rw"
+
+
+@dataclass
+class RunSpec:
+    image: str
+    name: str
+    labels: dict = field(default_factory=dict)
+    env: dict = field(default_factory=dict)
+    cpu_limit: str = ""
+    mem_limit: str = ""
+    ports: list = field(default_factory=list)
+    mounts: list = field(default_factory=list)
+    restart_policy: str = "no"
+    command: list = field(default_factory=list)
+
+
+@dataclass
+class BuildResult:
+    success: bool
+    image_id: str = ""
+    error: str = ""
+
+
+@dataclass
+class ExecResult:
+    exit_code: int
+    output: str = ""
+
+
+@dataclass
+class StatsSample:
+    cpu_pct: float = 0.0
+    mem_bytes: int = 0
+    mem_pct: float = 0.0
+    net_rx: int = 0
+    net_tx: int = 0
+    blk_read: int = 0
+    blk_write: int = 0
+
+
+@dataclass
+class PsRow:
+    container_id: str
+    name: str
+    state: str
+    status: str
+    exit_code: Optional[int]
+    labels: dict = field(default_factory=dict)
+
+
+class Backend(ABC):
+    @abstractmethod
+    async def build(self, *, context_dir: str, dockerfile_text: str, tags: list,
+                    on_log: Optional[LogCallback] = None, network: str = "") -> BuildResult: ...
+
+    @abstractmethod
+    async def run(self, spec: RunSpec) -> str: ...
+
+    @abstractmethod
+    async def start(self, cid: str) -> None: ...
+
+    @abstractmethod
+    async def stop(self, cid: str, timeout: int = 10) -> None: ...
+
+    @abstractmethod
+    async def restart(self, cid: str) -> None: ...
+
+    @abstractmethod
+    async def pause(self, cid: str) -> None: ...
+
+    @abstractmethod
+    async def unpause(self, cid: str) -> None: ...
+
+    @abstractmethod
+    async def rm(self, cid: str, force: bool = False) -> None: ...
+
+    @abstractmethod
+    async def exec(self, cid: str, cmd: list, *, tty: bool = False,
+                   on_log: Optional[LogCallback] = None) -> ExecResult: ...
+
+    @abstractmethod
+    async def logs(self, cid: str, *, follow: bool = False, tail: int = 200,
+                   on_log: Optional[LogCallback] = None) -> None: ...
+
+    @abstractmethod
+    async def stats_once(self, cids: list) -> dict: ...
+
+    @abstractmethod
+    async def ps(self, *, label_filter: str = "devplace.instance") -> list: ...
+
+    @abstractmethod
+    async def inspect(self, cid: str) -> dict: ...
+
+    @abstractmethod
+    async def image_prune(self) -> None: ...
+
+    @abstractmethod
+    async def remove_image(self, ref: str, force: bool = False) -> None: ...
+
+    @abstractmethod
+    async def image_exists(self, ref: str) -> bool: ...

devplacepy/services/containers/backend/docker_cli.py

@@ -0,0 +1,280 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import re
+import tempfile
+from pathlib import Path
+from typing import Optional
+
+from devplacepy.services.containers.backend.base import (
+    Backend,
+    BuildResult,
+    ExecResult,
+    LogCallback,
+    PsRow,
+    RunSpec,
+    StatsSample,
+)
+
+logger = logging.getLogger("containers.docker")
+
+DOCKER = "docker"
+DEFAULT_TIMEOUT = 600
+BUILD_TIMEOUT = 3600
+_EXIT_RE = re.compile(r"Exited \((\d+)\)")
+_SIZE_RE = re.compile(r"([0-9.]+)\s*([kKmMgGtT]?i?)B")
+
+
+def build_run_argv(spec: RunSpec) -> list:
+    argv = [DOCKER, "run", "-d", "--name", spec.name]
+    for key, value in spec.labels.items():
+        argv += ["--label", f"{key}={value}"]
+    for key, value in spec.env.items():
+        argv += ["-e", f"{key}={value}"]
+    if spec.cpu_limit:
+        argv += ["--cpus", str(spec.cpu_limit)]
+    if spec.mem_limit:
+        argv += ["--memory", str(spec.mem_limit)]
+    for port in spec.ports:
+        argv += ["-p", f"{port.host}:{port.container}/{port.proto}"]
+    for mount in spec.mounts:
+        argv += ["-v", f"{mount.host}:{mount.container}:{mount.mode}"]
+    if spec.restart_policy and spec.restart_policy != "never":
+        policy = "no" if spec.restart_policy in ("no", "never") else spec.restart_policy
+        argv += ["--restart", policy]
+    argv.append(spec.image)
+    argv += list(spec.command)
+    return argv
+
+
+def build_image_argv(context_dir: str, dockerfile_path: str, tags: list, iidfile: str, network: str = "") -> list:
+    argv = [DOCKER, "build", "--label", "devplace.build=1", "--iidfile", iidfile, "-f", dockerfile_path]
+    if network:
+        argv += ["--network", network]
+    for tag in tags:
+        argv += ["-t", tag]
+    argv.append(context_dir)
+    return argv
+
+
+def parse_size(text: str) -> int:
+    match = _SIZE_RE.search(text or "")
+    if not match:
+        return 0
+    value = float(match.group(1))
+    unit = match.group(2).lower().rstrip("i")
+    factor = {"": 1, "k": 1024, "m": 1024 ** 2, "g": 1024 ** 3, "t": 1024 ** 4}.get(unit, 1)
+    return int(value * factor)
+
+
+def parse_stats_line(row: dict) -> StatsSample:
+    def _pair(text: str) -> tuple:
+        parts = (text or "").split("/")
+        left = parse_size(parts[0]) if parts else 0
+        right = parse_size(parts[1]) if len(parts) > 1 else 0
+        return left, right
+
+    rx, tx = _pair(row.get("NetIO", ""))
+    rd, wr = _pair(row.get("BlockIO", ""))
+    return StatsSample(
+        cpu_pct=float((row.get("CPUPerc", "0%") or "0%").rstrip("%") or 0),
+        mem_bytes=_pair(row.get("MemUsage", ""))[0],
+        mem_pct=float((row.get("MemPerc", "0%") or "0%").rstrip("%") or 0),
+        net_rx=rx, net_tx=tx, blk_read=rd, blk_write=wr,
+    )
+
+
+class DockerCliBackend(Backend):
+    def __init__(self, binary: str = DOCKER, timeout: int = DEFAULT_TIMEOUT,
+                 build_timeout: int = BUILD_TIMEOUT) -> None:
+        self._binary = binary
+        self._timeout = timeout
+        self._build_timeout = build_timeout
+
+    async def _run(self, argv: list, *, timeout: Optional[int] = None) -> tuple:
+        proc = await asyncio.create_subprocess_exec(
+            *argv, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
+        try:
+            out, err = await asyncio.wait_for(proc.communicate(), timeout=timeout or self._timeout)
+        except asyncio.TimeoutError:
+            proc.kill()
+            raise RuntimeError(f"docker command timed out: {' '.join(argv[:3])}")
+        return proc.returncode, out.decode("utf-8", "replace"), err.decode("utf-8", "replace")
+
+    async def _stream(self, argv: list, on_log: Optional[LogCallback], *, timeout: Optional[int] = None) -> int:
+        proc = await asyncio.create_subprocess_exec(
+            *argv, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT)
+
+        async def pump():
+            assert proc.stdout is not None
+            async for raw in proc.stdout:
+                line = raw.decode("utf-8", "replace").rstrip("\n")
+                if on_log is not None:
+                    await on_log(line)
+            return await proc.wait()
+
+        try:
+            return await asyncio.wait_for(pump(), timeout=timeout or self._timeout)
+        except asyncio.TimeoutError:
+            proc.kill()
+            raise RuntimeError(f"docker stream timed out: {' '.join(argv[:3])}")
+
+    async def _check(self, argv: list) -> str:
+        code, out, err = await self._run(argv)
+        if code != 0:
+            raise RuntimeError(f"{' '.join(argv[:2])} failed ({code}): {(err or out).strip()[:500]}")
+        return out.strip()
+
+    async def build(self, *, context_dir, dockerfile_text, tags, on_log=None, network="") -> BuildResult:
+        ctx = Path(context_dir)
+        dockerfile_path = ctx / "Dockerfile.devplace"
+        iid_path = await asyncio.to_thread(self._prepare_context, dockerfile_path, dockerfile_text)
+        argv = build_image_argv(str(ctx), str(dockerfile_path), tags, iid_path, network)
+        try:
+            code = await self._stream(argv, on_log, timeout=self._build_timeout)
+            if code != 0:
+                return BuildResult(success=False, error=f"docker build exited {code}")
+            image_id = (await asyncio.to_thread(Path(iid_path).read_text)).strip()
+            return BuildResult(success=True, image_id=image_id)
+        finally:
+            await asyncio.to_thread(self._cleanup_context, iid_path, dockerfile_path)
+
+    @staticmethod
+    def _prepare_context(dockerfile_path: Path, dockerfile_text: str) -> str:
+        dockerfile_path.write_text(dockerfile_text, encoding="utf-8")
+        iid = tempfile.NamedTemporaryFile(suffix=".iid", delete=False)
+        iid.close()
+        return iid.name
+
+    @staticmethod
+    def _cleanup_context(iid_path: str, dockerfile_path: Path) -> None:
+        Path(iid_path).unlink(missing_ok=True)
+        dockerfile_path.unlink(missing_ok=True)
+
+    async def run(self, spec: RunSpec) -> str:
+        return await self._check(build_run_argv(spec))
+
+    async def start(self, cid: str) -> None:
+        await self._check([self._binary, "start", cid])
+
+    async def stop(self, cid: str, timeout: int = 10) -> None:
+        await self._check([self._binary, "stop", "-t", str(timeout), cid])
+
+    async def restart(self, cid: str) -> None:
+        await self._check([self._binary, "restart", cid])
+
+    async def pause(self, cid: str) -> None:
+        await self._check([self._binary, "pause", cid])
+
+    async def unpause(self, cid: str) -> None:
+        await self._check([self._binary, "unpause", cid])
+
+    async def rm(self, cid: str, force: bool = False) -> None:
+        argv = [self._binary, "rm"]
+        if force:
+            argv.append("-f")
+        argv.append(cid)
+        await self._check(argv)
+
+    async def exec(self, cid, cmd, *, tty=False, on_log=None) -> ExecResult:
+        argv = [self._binary, "exec"]
+        if tty:
+            argv.append("-t")
+        argv.append(cid)
+        argv += list(cmd)
+        if on_log is not None:
+            code = await self._stream(argv, on_log)
+            return ExecResult(exit_code=code)
+        code, out, err = await self._run(argv)
+        return ExecResult(exit_code=code or 0, output=(out + err))
+
+    async def logs(self, cid, *, follow=False, tail=200, on_log=None) -> None:
+        argv = [self._binary, "logs", "--tail", str(tail)]
+        if follow:
+            argv.append("--follow")
+        argv.append(cid)
+        await self._stream(argv, on_log)
+
+    async def stats_once(self, cids: list) -> dict:
+        if not cids:
+            return {}
+        argv = [self._binary, "stats", "--no-stream", "--format", "{{json .}}", *cids]
+        code, out, _ = await self._run(argv, timeout=30)
+        samples = {}
+        if code != 0:
+            return samples
+        for line in out.splitlines():
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                row = json.loads(line)
+            except ValueError:
+                continue
+            name = row.get("Name") or row.get("ID") or ""
+            samples[name] = parse_stats_line(row)
+        return samples
+
+    async def ps(self, *, label_filter: str = "devplace.instance") -> list:
+        argv = [self._binary, "ps", "-a", "--filter", f"label={label_filter}", "--format", "{{json .}}"]
+        code, out, _ = await self._run(argv, timeout=30)
+        rows = []
+        if code != 0:
+            return rows
+        for line in out.splitlines():
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                row = json.loads(line)
+            except ValueError:
+                continue
+            labels = {}
+            for pair in (row.get("Labels", "") or "").split(","):
+                if "=" in pair:
+                    key, value = pair.split("=", 1)
+                    labels[key] = value
+            status = row.get("Status", "") or ""
+            match = _EXIT_RE.search(status)
+            rows.append(PsRow(
+                container_id=row.get("ID", ""),
+                name=row.get("Names", ""),
+                state=(row.get("State", "") or "").lower(),
+                status=status,
+                exit_code=int(match.group(1)) if match else None,
+                labels=labels,
+            ))
+        return rows
+
+    async def inspect(self, cid: str) -> dict:
+        code, out, _ = await self._run([self._binary, "inspect", cid], timeout=30)
+        if code != 0:
+            return {}
+        try:
+            data = json.loads(out)
+            return data[0] if isinstance(data, list) and data else {}
+        except ValueError:
+            return {}
+
+    async def image_prune(self) -> None:
+        try:
+            await self._run([self._binary, "image", "prune", "-f", "--filter", "label=devplace.build"], timeout=120)
+        except RuntimeError as exc:
+            logger.warning("image prune failed: %s", exc)
+
+    async def remove_image(self, ref: str, force: bool = False) -> None:
+        argv = [self._binary, "rmi"]
+        if force:
+            argv.append("-f")
+        argv.append(ref)
+        code, _, err = await self._run(argv, timeout=60)
+        if code != 0 and "No such image" not in (err or ""):
+            logger.warning("rmi %s failed: %s", ref, (err or "").strip()[:200])
+
+    async def image_exists(self, ref: str) -> bool:
+        code, _, _ = await self._run([self._binary, "image", "inspect", ref], timeout=30)
+        return code == 0

devplacepy/services/containers/backend/fake.py

@@ -0,0 +1,106 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from devplacepy.services.containers.backend.base import (
+    Backend,
+    BuildResult,
+    ExecResult,
+    PsRow,
+    RunSpec,
+    StatsSample,
+)
+
+
+class FakeBackend(Backend):
+    def __init__(self) -> None:
+        self.containers: dict = {}
+        self.built_images: list = []
+        self.removed: list = []
+        self.removed_images: list = []
+        self.pruned = 0
+        self._counter = 0
+        self.fail_build = False
+
+    async def build(self, *, context_dir, dockerfile_text, tags, on_log=None, network="") -> BuildResult:
+        if on_log is not None:
+            await on_log(f"FAKE build {tags} net={network}")
+        if self.fail_build:
+            return BuildResult(success=False, error="fake build failure")
+        self.built_images.append(list(tags))
+        return BuildResult(success=True, image_id=f"sha256:fake{len(self.built_images)}")
+
+    async def run(self, spec: RunSpec) -> str:
+        self._counter += 1
+        cid = f"fake{self._counter:012d}"
+        self.containers[cid] = {
+            "name": spec.name, "state": "running", "labels": dict(spec.labels),
+            "exit_code": None, "image": spec.image, "spec": spec,
+        }
+        return cid
+
+    def _set_state(self, cid: str, state: str, exit_code=None) -> None:
+        if cid in self.containers:
+            self.containers[cid]["state"] = state
+            if exit_code is not None:
+                self.containers[cid]["exit_code"] = exit_code
+
+    async def start(self, cid: str) -> None:
+        self._set_state(cid, "running")
+
+    async def stop(self, cid: str, timeout: int = 10) -> None:
+        self._set_state(cid, "exited", exit_code=0)
+
+    async def restart(self, cid: str) -> None:
+        self._set_state(cid, "running")
+
+    async def pause(self, cid: str) -> None:
+        self._set_state(cid, "paused")
+
+    async def unpause(self, cid: str) -> None:
+        self._set_state(cid, "running")
+
+    async def rm(self, cid: str, force: bool = False) -> None:
+        self.removed.append(cid)
+        self.containers.pop(cid, None)
+
+    async def exec(self, cid, cmd, *, tty=False, on_log=None) -> ExecResult:
+        line = f"FAKE exec {' '.join(cmd)}"
+        if on_log is not None:
+            await on_log(line)
+        return ExecResult(exit_code=0, output=line)
+
+    async def logs(self, cid, *, follow=False, tail=200, on_log=None) -> None:
+        if on_log is not None:
+            await on_log(f"FAKE logs for {cid}")
+
+    async def stats_once(self, cids: list) -> dict:
+        return {self.containers[cid]["name"]: StatsSample(cpu_pct=1.0, mem_bytes=1024)
+                for cid in cids if cid in self.containers}
+
+    async def ps(self, *, label_filter: str = "devplace.instance") -> list:
+        key, _, value = label_filter.partition("=")
+        rows = []
+        for cid, data in self.containers.items():
+            labels = data["labels"]
+            if key not in labels:
+                continue
+            if value and labels.get(key) != value:
+                continue
+            rows.append(PsRow(
+                container_id=cid, name=data["name"], state=data["state"],
+                status=data["state"], exit_code=data["exit_code"], labels=labels,
+            ))
+        return rows
+
+    async def inspect(self, cid: str) -> dict:
+        return self.containers.get(cid, {})
+
+    async def image_prune(self) -> None:
+        self.pruned += 1
+
+    async def remove_image(self, ref: str, force: bool = False) -> None:
+        self.removed_images.append(ref)
+
+    async def image_exists(self, ref: str) -> bool:
+        return True

devplacepy/services/containers/files/.vimrc

@@ -0,0 +1,245 @@
+et nocompatible
+filetype off
+set rtp+=~/.vim/bundle/Vundle.vim
+
+let g:ycm_auto_trigger = 1
+
+filetype plugin indent on    " required
+
+set clipboard=unnamedplus
+
+function! GitBranch()
+  let l:branch =  system("bash -c 'uptime'")
+  return l:branch
+endfunction
+
+call plug#begin()
+   " List your plugins here
+  Plug 'prabirshrestha/asyncomplete.vim'
+Plug 'prabirshrestha/asyncomplete-lsp.vim'
+  Plug 'tpope/vim-sensible'
+  Plug 'Exafunction/codeium.vim', { 'branch': 'main' }
+  Plug 'prabirshrestha/vim-lsp'
+    Plug 'mattn/vim-lsp-settings'
+set statusline=\{…\}%3{codeium#GetStatusString()}\%h%m%r\ [%l,%c]\ %p%%
+highlight StatusLine guifg=#ffffff guibg=#005f87
+highlight StatusLineNC guifg=#aaaaaa guibg=#303030
+highlight ErrorMsg ctermfg=Red ctermbg=NONE guifg=#FF0000 guibg=NONE
+
+call plug#end()
+
+call vundle#begin()
+Plugin 'VundleVim/Vundle.vim'
+Plugin 'ycm-core/YouCompleteMe'
+call vundle#end()
+
+set mouse=a
+set backspace=indent,eol,start
+syntax on
+filetype plugin indent on
+
+set showtabline=2
+set enc=utf-8
+set fenc=utf-8
+set termencoding=utf-8
+set autoindent
+set smartindent
+set tabstop=4
+set shiftwidth=4
+set expandtab
+set number
+set showmatch
+set comments=sl:/*,mb:\ *,elx:\ */
+
+let mapleader = ","
+
+function! ShowInTerminal(content)
+  " Open a new terminal split
+  belowright split | terminal
+  " Wait for terminal to initialize
+  call feedkeys("i")  " go into insert mode to send keys
+  call chansend(b:terminal_job_id, a:content . "\n")
+endfunction
+
+function! AIR() 
+    let l:ai_prompt = input("Enter AI instructions: ")
+    let $AI_PROMPT = l:ai_prompt
+    let l:result = system('r ' . l:ai_prompt)
+    call ShowInTerminal(l:result)
+
+endfunction
+
+function! AIPromptSelection()
+    " Save cursor position and the unnamed register
+    let l:pos          = getpos('.')
+    let l:save_reg     = @"
+    let l:save_regtype = getregtype('"')
+
+    " Yank the visual selection into the unnamed register
+    normal! gvy
+    let l:selected_text = @"
+
+    " Restore the unnamed register
+    call setreg('"', l:save_reg, l:save_regtype)
+
+    " Prompt the user for AI instructions
+    let l:ai_prompt = input('Enter AI instructions: ')
+    let $AI_PROMPT  = l:ai_prompt
+
+    " Call your external AI formatter (replace `c` with your command)
+    let l:formatted = system('c', l:selected_text)
+
+    " Handle errors or replace the selection with the AI response
+    if v:shell_error
+        echohl ErrorMsg | echom 'Formatting failed' | echohl None
+    else
+        " Delete the original selection
+        normal! gvd
+        " Insert the formatted text *before* the cursor (same spot)
+        put! =l:formatted
+        " Restore the original cursor position
+        call setpos('.', l:pos)
+    endif
+
+    " Restore the unnamed register again (good measure)
+    call setreg('"', l:save_reg, l:save_regtype)
+endfunction
+
+" Map the AI prompt function to a key combination
+vnoremap <Leader>f :<C-u>call AIPromptSelection()<CR>
+nnoremap <Leader>r :call AIR()<CR>
+
+" Existing keymappings and other configurations...
+inoremap <C-n> <ESC>:tabnext<CR>
+inoremap <C-p> <ESC>:tabnext<CR>
+nnoremap <C-n> :tabnext<CR>
+nnoremap <C-p> :tabnext<CR>
+nnoremap <Tab> :tabnext<CR>
+nnoremap <C-Tab> :tabprevious<CR>
+inoremap <C-t> <ESC>:terminal<CR><CR><C-w>r<CR><C-w>-<C-w>-<C-w>-<C-w>-<C-w>-<C-w>-<C-w>-
+nnoremap <C-e> :tabnew<Space>
+inoremap <C-e> <ESC>:tabnew<Space>
+nnoremap e :tabnew<Space>
+nnoremap q <ESC>:q<CR>
+
+set undofile
+set undodir=~/.vim/undo
+
+" Existing commands and other configurations...
+command! GPT py3file ~/bin/gpt
+command! Refactor py3file /home/retoor/.vim/plugin/refactor.py
+command! HelloVim py3file ~/.vim/plugin/hello.py | py3 say_hello()
+
+if has("autocmd")
+    au BufReadPost * if line("'\"") > 0 && line("'\"") <= line("$") | exe "normal! g'\"" | endif
+endif
+
+execute pathogen#infect()
+execute pathogen#helptags()
+
+" Helper: get visually selected text
+function! GetVisualSelection()
+    let [line_start, col_start] = [line("'<"), col("'<")]
+    let [line_end, col_end] = [line("'>"), col("'>")]
+    let lines = getline(line_start, line_end)
+    if len(lines) == 0
+        return ''
+    endif
+    " Trim col for first/last line if selection is characterwise
+    let lines[0] = lines[0][col_start - 1 :]
+    let lines[-1] = lines[-1][: col_end - (line_start == line_end ? 1 : 2)]
+    return join(lines, "\n")
+endfunction
+
+function! AiEditSelection()
+    " (1) Get user prompt
+    let l:instruction = input("AI instruction: ")
+    if empty(l:instruction)
+        echo "No instruction given, cancelled."
+        return
+    endif
+
+    " (2) Get visual selection
+    let l:origText = GetVisualSelection()
+    if empty(l:origText)
+        echo "No selection."
+        return
+    endif
+
+    " (3) Compose the API prompt
+    let l:prompt = l:instruction . "\n\nHere is the text:\n" . l:origText . "\n\nOutput only the transformed text. Do not include explanations, markdown, or code blocks."
+
+    " (4) Send to the same gateway pagent uses (PRAVDA_OPENAI_URL / PRAVDA_API_KEY)
+    let l:base = substitute($PRAVDA_OPENAI_URL, '/\+$', '', '')
+    if empty(l:base)
+        let l:url = 'https://openai.app.molodetz.nl/v1/chat/completions'
+    elseif l:base =~# '/chat/completions$'
+        let l:url = l:base
+    else
+        let l:url = l:base . '/chat/completions'
+    endif
+    let l:api_key = !empty($PRAVDA_API_KEY) ? $PRAVDA_API_KEY : $DEEPSEEK_API_KEY
+    let l:json = '{"model":"deepseek-chat","messages":[{"role":"user","content":'.json_encode(l:prompt).'}]}'
+    let l:cmd = 'curl -sS -X POST ' . shellescape(l:url) . ' -H "Authorization: Bearer ' . l:api_key . '" -H "Content-Type: application/json" -d ' . shellescape(l:json)
+    let l:reply = system(l:cmd)
+    " --- Extract output (simple JSON parsing) ---
+    let l:text = matchstr(l:reply, '"content":\s*"\zs\(.\{-}\)\ze"\s*}')
+    let l:text = substitute(l:text, '\\n', "\n", 'g')
+    let l:text = substitute(l:text, '\\"', '"', 'g')
+
+    " (5) Replace selected text
+    normal! gv
+    " Use c to change or d (delete and then 'put') depending on visual mode
+    normal! c
+    call feedkeys(l:text, 'n')
+endfunction
+
+" Visual mode mapping
+xnoremap <silent> <Leader>a :<C-u>call AiEditSelection()<CR>
+
+command! GreetPython py3 greet()
+vnoremap <leader>gr :GPTRefactor<CR>
+vnoremap <leader>gd :GPTRefactorDefault<CR>
+
+
+if executable('pylsp')
+    " pip install python-lsp-server
+    au User lsp_setup call lsp#register_server({
+        \ 'name': 'pylsp',
+        \ 'cmd': {server_info->['pylsp']},
+        \ 'allowlist': ['python'],
+        \ })
+endif
+
+function! s:on_lsp_buffer_enabled() abort
+    setlocal omnifunc=lsp#complete
+    setlocal signcolumn=yes
+    if exists('+tagfunc') | setlocal tagfunc=lsp#tagfunc | endif
+    nmap <buffer> gd <plug>(lsp-definition)
+    nmap <buffer> gs <plug>(lsp-document-symbol-search)
+    nmap <buffer> gS <plug>(lsp-workspace-symbol-search)
+    nmap <buffer> gr <plug>(lsp-references)
+    nmap <buffer> gi <plug>(lsp-implementation)
+    nmap <buffer> gt <plug>(lsp-type-definition)
+    nmap <buffer> <leader>rn <plug>(lsp-rename)
+    nmap <buffer> [g <plug>(lsp-previous-diagnostic)
+    nmap <buffer> ]g <plug>(lsp-next-diagnostic)
+    nmap <buffer> K <plug>(lsp-hover)
+    nnoremap <buffer> <expr><c-f> lsp#scroll(+4)
+    nnoremap <buffer> <expr><c-d> lsp#scroll(-4)
+
+    let g:lsp_format_sync_timeout = 1000
+    autocmd! BufWritePre *.rs,*.go call execute('LspDocumentFormatSync')
+
+    " refer to doc to add more commands
+endfunction
+
+augroup lsp_install
+    au!
+    " call s:on_lsp_buffer_enabled only for languages that has the server registered.
+    autocmd User lsp_buffer_enabled call s:on_lsp_buffer_enabled()
+augroup END
+
+let g:lsp_document_highlight_enabled = 1
+let g:lsp_diagnostics_enabled = 0         " disable diagnostics support
+

devplacepy/services/containers/files/sudo

@@ -0,0 +1,66 @@
+#!/bin/sh
+# devplace sudo superclone: sudo-compatible CLI, no privilege change.
+# It never swaps user; it executes the command as the current user (pravda),
+# so nothing a container does can ever create a root-owned file on the host.
+
+_dp_usage() {
+    cat <<'EOF'
+usage: sudo -h | -K | -k | -V
+usage: sudo -v [-ABkNnS] [-g group] [-h host] [-p prompt] [-u user]
+usage: sudo -l [-ABkNnS] [-g group] [-h host] [-p prompt] [-U user] [-u user] [command]
+usage: sudo [-ABbEHnPS] [-C num] [-g group] [-h host] [-p prompt] [-R dir] [-T timeout] [-u user] [VAR=value] [-i | -s] [command [arg ...]]
+EOF
+}
+
+_dp_login=0
+_dp_shell=0
+
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --) shift; break ;;
+        -V|--version) echo "Sudo version 1.9.15p5"; echo "Sudoers policy plugin version 1.9.15p5"; exit 0 ;;
+        --help) _dp_usage; exit 0 ;;
+        -K|-k|--remove-timestamp|--reset-timestamp) exit 0 ;;
+        -v|--validate) exit 0 ;;
+        -l|-ll|--list) echo "User $(id -un 2>/dev/null) may run any command"; exit 0 ;;
+        -i|--login) _dp_login=1; shift ;;
+        -s|--shell) _dp_shell=1; shift ;;
+        --user=*|--group=*|--prompt=*|--chdir=*|--chroot=*|--host=*|--role=*|--type=*|--other-user=*|--command-timeout=*|--close-from=*|--preserve-env=*) shift ;;
+        -A|--askpass|-b|--background|-B|--bell|-E|--preserve-env|-H|--set-home|-n|--non-interactive|-N|--no-update|-P|--preserve-groups|-S|--stdin) shift ;;
+        -u|--user|-g|--group|-p|--prompt|-C|--close-from|-D|--chdir|-R|--chroot|-T|--command-timeout|-U|--other-user|-r|--role|-t|--type|-c|--class|-h|--host)
+            if [ $# -ge 2 ]; then shift 2; else shift; fi ;;
+        -*) shift ;;
+        *) break ;;
+    esac
+done
+
+_dp_env=""
+while [ $# -gt 0 ]; do
+    case "$1" in
+        [A-Za-z_]*=*) _dp_env="$_dp_env $1"; shift ;;
+        *) break ;;
+    esac
+done
+
+SUDO_USER="$(id -un 2>/dev/null)"; export SUDO_USER
+SUDO_UID="$(id -u 2>/dev/null)"; export SUDO_UID
+SUDO_GID="$(id -g 2>/dev/null)"; export SUDO_GID
+SUDO_COMMAND="$*"; export SUDO_COMMAND
+
+if [ "$_dp_login" -eq 1 ] || [ "$_dp_shell" -eq 1 ]; then
+    _dp_sh="${SHELL:-/bin/sh}"
+    if [ $# -gt 0 ]; then
+        [ -n "$_dp_env" ] && exec env $_dp_env "$_dp_sh" -c "$*"
+        exec "$_dp_sh" -c "$*"
+    fi
+    [ -n "$_dp_env" ] && exec env $_dp_env "$_dp_sh"
+    exec "$_dp_sh"
+fi
+
+if [ $# -eq 0 ]; then
+    _dp_usage >&2
+    exit 1
+fi
+
+[ -n "$_dp_env" ] && exec env $_dp_env "$@"
+exec "$@"

devplacepy/services/containers/locks.py

@@ -0,0 +1,5 @@
+# retoor <retoor@molodetz.nl>
+
+import asyncio
+
+NUMBERING_LOCK = asyncio.Lock()

devplacepy/services/containers/runtime.py

@@ -0,0 +1,16 @@
+# retoor <retoor@molodetz.nl>
+
+_backend = None
+
+
+def get_backend():
+    global _backend
+    if _backend is None:
+        from devplacepy.services.containers.backend.docker_cli import DockerCliBackend
+        _backend = DockerCliBackend()
+    return _backend
+
+
+def set_backend(backend) -> None:
+    global _backend
+    _backend = backend

devplacepy/services/containers/service.py

@@ -0,0 +1,213 @@
+# retoor <retoor@molodetz.nl>
+
+import json
+
+from devplacepy import config
+from devplacepy.services.base import BaseService, ConfigField
+from devplacepy.services.containers import api, store
+from devplacepy.services.containers.runtime import get_backend
+from devplacepy.services.devii.tasks.schedule import next_run, now_utc, to_iso
+
+AUTO_RESTART = ("always", "on-failure", "unless-stopped")
+
+
+class ContainerService(BaseService):
+    default_enabled = False
+    min_interval = 1
+    title = "Containers"
+    description = (
+        "Reconciles desired container state against the docker daemon: launches, stops, restarts, "
+        "reaps orphans, fires schedules, and samples per-instance metrics. Requires access to the "
+        "docker socket."
+    )
+    config_fields = [
+        ConfigField("container_metrics_every", "Metrics sample every (ticks)", type="int",
+                    default=1, minimum=1, maximum=60,
+                    help="Sample docker stats once every N reconcile ticks.", group="Containers"),
+    ]
+
+    def __init__(self):
+        super().__init__(name="containers", interval_seconds=5)
+        self._metric_tick = 0
+
+    async def run_once(self) -> None:
+        backend = get_backend()
+        try:
+            ps_rows = await backend.ps(label_filter=api.INSTANCE_LABEL)
+        except Exception as exc:
+            self.log(f"docker ps failed: {exc}")
+            return
+
+        by_uid = {}
+        for row in ps_rows:
+            uid = row.labels.get(api.INSTANCE_LABEL)
+            if uid:
+                by_uid[uid] = row
+
+        instances = store.all_instances()
+        known = {inst["uid"] for inst in instances}
+
+        for uid, row in by_uid.items():
+            if uid not in known:
+                try:
+                    await backend.rm(row.container_id, force=True)
+                    self.log(f"reaped orphan container {row.name}")
+                except Exception as exc:
+                    self.log(f"orphan rm failed for {row.name}: {exc}")
+
+        for inst in instances:
+            try:
+                await self._reconcile(backend, inst, by_uid.get(inst["uid"]))
+            except Exception as exc:
+                self.log(f"reconcile {inst.get('name')} failed: {exc}")
+
+        await self._fire_schedules()
+
+        self._metric_tick += 1
+        if self._metric_tick % max(1, self.get_config().get("container_metrics_every", 1)) == 0:
+            await self._sample_metrics(backend, by_uid)
+
+    async def _reconcile(self, backend, inst, ps) -> None:
+        uid = inst["uid"]
+        desired = inst["desired_state"]
+        status = inst["status"]
+
+        if status == store.ST_REMOVING:
+            if ps is not None:
+                await backend.rm(ps.container_id, force=True)
+            store.delete_instance(uid)
+            self.log(f"removed instance {inst['name']}")
+            return
+
+        if desired == store.DESIRED_RUNNING:
+            if ps is None:
+                await self._launch(backend, inst)
+            elif ps.state == "running":
+                if status != store.ST_RUNNING:
+                    store.update_instance(uid, {"status": store.ST_RUNNING, "container_id": ps.container_id,
+                                                "started_at": inst.get("started_at") or store.now()})
+            elif ps.state == "paused":
+                await backend.unpause(ps.container_id)
+                store.update_instance(uid, {"status": store.ST_RUNNING})
+            elif ps.state == "created":
+                await backend.start(ps.container_id)
+                store.update_instance(uid, {"status": store.ST_RUNNING})
+            else:
+                await self._handle_exit(backend, inst, ps)
+
+        elif desired == store.DESIRED_STOPPED:
+            if ps is not None and ps.state in ("running", "restarting", "paused"):
+                await backend.stop(ps.container_id)
+            if status != store.ST_STOPPED:
+                store.update_instance(uid, {"status": store.ST_STOPPED, "stopped_at": store.now()})
+
+        elif desired == store.DESIRED_PAUSED:
+            if ps is not None and ps.state == "running":
+                await backend.pause(ps.container_id)
+            store.update_instance(uid, {"status": store.ST_PAUSED})
+
+    async def _launch(self, backend, inst) -> None:
+        spec = api.run_spec_for(inst, config.CONTAINER_IMAGE)
+        try:
+            cid = await backend.run(spec)
+        except Exception as exc:
+            store.update_instance(inst["uid"], {"status": store.ST_CRASHED})
+            store.record_event(inst, "launch_failed", "reconciler", "", {"reason": str(exc)})
+            self.log(f"launch {inst['name']} failed: {exc}")
+            return
+        store.update_instance(inst["uid"], {"container_id": cid, "status": store.ST_RUNNING, "started_at": store.now()})
+        store.record_event(inst, "start", "reconciler", "")
+        self.log(f"launched instance {inst['name']}")
+
+    async def _exit_logs(self, backend, container_id: str) -> str:
+        lines: list = []
+
+        async def collect(line: str) -> None:
+            lines.append(line)
+
+        try:
+            await backend.logs(container_id, follow=False, tail=20, on_log=collect)
+        except Exception:
+            return ""
+        return "\n".join(lines)[-2000:]
+
+    async def _handle_exit(self, backend, inst, ps) -> None:
+        uid = inst["uid"]
+        exit_code = ps.exit_code or 0
+        logs = await self._exit_logs(backend, ps.container_id)
+        policy = inst.get("restart_policy", "never")
+        if policy in AUTO_RESTART and not (policy == "on-failure" and exit_code == 0):
+            try:
+                await backend.start(ps.container_id)
+                store.update_instance(uid, {"status": store.ST_RUNNING,
+                                            "restart_count": int(inst.get("restart_count") or 0) + 1})
+                store.record_event(inst, "policy_restart", "reconciler", "",
+                                   {"exit_code": exit_code, "logs": logs})
+                return
+            except Exception as exc:
+                self.log(f"policy restart {inst['name']} failed: {exc}")
+        terminal = store.ST_CRASHED if exit_code != 0 else store.ST_STOPPED
+        store.update_instance(uid, {"status": terminal, "desired_state": store.DESIRED_STOPPED,
+                                    "exit_code": exit_code, "stopped_at": store.now()})
+        if terminal == store.ST_CRASHED:
+            store.record_event(inst, "crash", "reconciler", "", {"exit_code": exit_code, "logs": logs})
+
+    async def _fire_schedules(self) -> None:
+        now = now_utc()
+        now_iso = to_iso(now)
+        for sched in store.due_schedules(now_iso):
+            inst = store.get_instance(sched["instance_uid"])
+            if inst is None:
+                store.delete_schedule(sched["uid"])
+                continue
+            action = sched["action"]
+            desired = store.DESIRED_RUNNING if action == "start" else store.DESIRED_STOPPED
+            store.update_instance(inst["uid"], {"desired_state": desired})
+            store.record_event(inst, f"schedule_{action}", "scheduler", "")
+            cols = json.loads(sched.get("schedule_json") or "{}")
+            run_count = int(sched.get("run_count") or 0) + 1
+            upcoming = next_run(cols.get("kind"), cols.get("every_seconds"), cols.get("cron"), now)
+            changes = {"last_run_at": now_iso, "run_count": run_count}
+            max_runs = cols.get("max_runs")
+            if upcoming is None or (max_runs and run_count >= max_runs):
+                changes["enabled"] = 0
+                changes["next_run_at"] = ""
+            else:
+                changes["next_run_at"] = to_iso(upcoming)
+            store.update_schedule(sched["uid"], changes)
+            self.log(f"schedule fired: {action} {inst['name']}")
+
+    async def _sample_metrics(self, backend, by_uid) -> None:
+        running = {uid: row for uid, row in by_uid.items() if row.state == "running"}
+        if not running:
+            return
+        try:
+            samples = await backend.stats_once([row.container_id for row in running.values()])
+        except Exception as exc:
+            self.log(f"docker stats failed: {exc}")
+            return
+        name_to_uid = {row.name: uid for uid, row in running.items()}
+        for name, sample in samples.items():
+            uid = name_to_uid.get(name)
+            if uid:
+                store.insert_metric(uid, sample)
+
+    def collect_metrics(self) -> dict:
+        instances = store.all_instances()
+        counts = {}
+        for inst in instances:
+            counts[inst["status"]] = counts.get(inst["status"], 0) + 1
+        running = counts.get(store.ST_RUNNING, 0)
+        stats = [
+            {"label": "Instances", "value": len(instances)},
+            {"label": "Running", "value": running},
+            {"label": "Stopped", "value": counts.get(store.ST_STOPPED, 0)},
+            {"label": "Crashed", "value": counts.get(store.ST_CRASHED, 0)},
+            {"label": "Paused", "value": counts.get(store.ST_PAUSED, 0)},
+        ]
+        rows = [[
+            inst.get("name", "")[:32], inst.get("status", ""), inst.get("desired_state", ""),
+            inst.get("restart_policy", ""), int(inst.get("restart_count") or 0),
+        ] for inst in instances[:15]]
+        table = {"columns": ["Instance", "Status", "Desired", "Policy", "Restarts"], "rows": rows}
+        return {"stats": stats, "table": table}

devplacepy/services/containers/store.py

@@ -0,0 +1,161 @@
+# retoor <retoor@molodetz.nl>
+
+import json
+from datetime import datetime, timezone
+
+from devplacepy.database import db, get_table
+from devplacepy.utils import generate_uid, make_combined_slug
+
+ST_CREATED = "created"
+ST_STARTING = "starting"
+ST_RUNNING = "running"
+ST_STOPPED = "stopped"
+ST_PAUSED = "paused"
+ST_CRASHED = "crashed"
+ST_RESTARTING = "restarting"
+ST_REMOVING = "removing"
+ST_REMOVED = "removed"
+
+DESIRED_RUNNING = "running"
+DESIRED_STOPPED = "stopped"
+DESIRED_PAUSED = "paused"
+
+RESTART_POLICIES = ("never", "always", "on-failure", "unless-stopped")
+METRICS_RING = 720
+
+
+def now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _exists(table: str) -> bool:
+    return table in db.tables
+
+
+# ---------------- instances ----------------
+
+def create_instance(row: dict) -> dict:
+    uid = generate_uid()
+    base = {
+        "uid": uid, "slug": make_combined_slug(row.get("name", "instance"), uid),
+        "container_id": "", "status": ST_CREATED, "exit_code": 0, "restart_count": 0,
+        "ingress_slug": "", "ingress_port": 0,
+        "started_at": "", "stopped_at": "", "created_at": now(), "updated_at": now(),
+    }
+    base.update(row)
+    base["uid"] = uid
+    get_table("instances").insert(base)
+    return get_instance(uid)
+
+
+def get_instance(uid: str):
+    table = get_table("instances")
+    if not _exists("instances"):
+        return None
+    return table.find_one(uid=uid) or table.find_one(slug=uid)
+
+
+def list_instances(project_uid: str = None) -> list:
+    if not _exists("instances"):
+        return []
+    filters = {}
+    if project_uid:
+        filters["project_uid"] = project_uid
+    return sorted(get_table("instances").find(**filters),
+                  key=lambda r: r.get("created_at", ""), reverse=True)
+
+
+def all_instances() -> list:
+    if not _exists("instances"):
+        return []
+    return list(get_table("instances").find())
+
+
+def find_instance_by_ingress(slug: str):
+    if not slug or not _exists("instances"):
+        return None
+    matches = list(get_table("instances").find(ingress_slug=slug))
+    for instance in matches:
+        if instance.get("status") == ST_RUNNING:
+            return instance
+    return matches[0] if matches else None
+
+
+def update_instance(uid: str, changes: dict) -> None:
+    get_table("instances").update({"uid": uid, "updated_at": now(), **changes}, ["uid"])
+
+
+def delete_instance(uid: str) -> None:
+    get_table("instances").delete(uid=uid)
+
+
+# ---------------- events / metrics / schedules ----------------
+
+def record_event(instance: dict, event: str, actor_kind: str, actor_id: str, detail: dict = None) -> None:
+    get_table("instance_events").insert({
+        "uid": generate_uid(), "instance_uid": instance["uid"], "project_uid": instance.get("project_uid", ""),
+        "event": event, "actor_kind": actor_kind, "actor_id": actor_id or "",
+        "detail": json.dumps(detail or {}), "created_at": now(),
+    })
+
+
+def list_events(instance_uid: str, limit: int = 100) -> list:
+    if not _exists("instance_events"):
+        return []
+    rows = sorted(get_table("instance_events").find(instance_uid=instance_uid),
+                  key=lambda r: r.get("created_at", ""), reverse=True)
+    return rows[:limit]
+
+
+def insert_metric(instance_uid: str, sample) -> None:
+    table = get_table("instance_metrics")
+    table.insert({
+        "uid": generate_uid(), "instance_uid": instance_uid, "ts": now(),
+        "cpu_pct": sample.cpu_pct, "mem_bytes": sample.mem_bytes, "mem_pct": sample.mem_pct,
+        "net_rx": sample.net_rx, "net_tx": sample.net_tx,
+        "blk_read": sample.blk_read, "blk_write": sample.blk_write,
+    })
+    rows = sorted(table.find(instance_uid=instance_uid), key=lambda r: r.get("ts", ""))
+    if len(rows) > METRICS_RING:
+        for old in rows[:len(rows) - METRICS_RING]:
+            table.delete(uid=old["uid"])
+
+
+def recent_metrics(instance_uid: str, limit: int = 120) -> list:
+    if not _exists("instance_metrics"):
+        return []
+    rows = sorted(get_table("instance_metrics").find(instance_uid=instance_uid),
+                  key=lambda r: r.get("ts", ""))
+    return rows[-limit:]
+
+
+def create_schedule(instance: dict, action: str, schedule_columns: dict, next_run_at: str) -> dict:
+    uid = generate_uid()
+    get_table("instance_schedules").insert({
+        "uid": uid, "instance_uid": instance["uid"], "project_uid": instance.get("project_uid", ""),
+        "action": action, "schedule_json": json.dumps(schedule_columns), "enabled": 1,
+        "next_run_at": next_run_at, "last_run_at": "", "run_count": 0,
+        "created_at": now(), "updated_at": now(),
+    })
+    return get_table("instance_schedules").find_one(uid=uid)
+
+
+def list_schedules(instance_uid: str) -> list:
+    if not _exists("instance_schedules"):
+        return []
+    return list(get_table("instance_schedules").find(instance_uid=instance_uid))
+
+
+def due_schedules(now_iso: str) -> list:
+    if not _exists("instance_schedules"):
+        return []
+    return [r for r in get_table("instance_schedules").find(enabled=1)
+            if r.get("next_run_at") and r["next_run_at"] <= now_iso]
+
+
+def update_schedule(uid: str, changes: dict) -> None:
+    get_table("instance_schedules").update({"uid": uid, "updated_at": now(), **changes}, ["uid"])
+
+
+def delete_schedule(uid: str) -> None:
+    get_table("instance_schedules").delete(uid=uid)

devplacepy/services/devii/__init__.py

@@ -0,0 +1,5 @@
+# retoor <retoor@molodetz.nl>
+
+from .service import DeviiService
+
+__all__ = ["DeviiService"]

devplacepy/services/devii/__main__.py

@@ -0,0 +1,6 @@
+# retoor <retoor@molodetz.nl>
+
+from .cli import main
+
+if __name__ == "__main__":
+    main()

devplacepy/services/devii/actions/__init__.py

@@ -0,0 +1,7 @@
+# retoor <retoor@molodetz.nl>
+
+from .catalog import ACTIONS, PLATFORM_CATALOG
+from .dispatcher import Dispatcher
+from .spec import Action, Catalog, Param
+
+__all__ = ["ACTIONS", "PLATFORM_CATALOG", "Dispatcher", "Action", "Catalog", "Param"]

devplacepy/services/devii/actions/avatar_actions.py

@@ -0,0 +1,149 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from .spec import Action, Param
+
+
+def arg(name: str, description: str, required: bool = False, kind: str = "string") -> Param:
+    return Param(name=name, location="body", description=description, required=required, type=kind)
+
+
+DEVII = (
+    "Controls devii, the on-screen avatar that is you rendered as a classic animated "
+    "desktop character. Only effective in the web interface; in other interfaces it reports "
+    "that no avatar is attached."
+)
+
+AVATAR_ACTIONS: tuple[Action, ...] = (
+    Action(
+        name="avatar_show",
+        method="LOCAL",
+        path="",
+        summary="Show devii (yourself) on screen",
+        description=DEVII,
+        handler="avatar",
+        requires_auth=False,
+    ),
+    Action(
+        name="avatar_hide",
+        method="LOCAL",
+        path="",
+        summary="Hide devii from the screen (plays a goodbye/hide animation)",
+        description=(
+            DEVII + " This already plays a goodbye animation, so do not queue a separate "
+            "wave/goodbye animation immediately before calling it - hiding interrupts a still-"
+            "queued animation."
+        ),
+        handler="avatar",
+        requires_auth=False,
+    ),
+    Action(
+        name="avatar_speak",
+        method="LOCAL",
+        path="",
+        summary="Show a line of text in devii's speech balloon",
+        description=DEVII,
+        handler="avatar",
+        requires_auth=False,
+        params=(
+            arg("text", "The text devii should say.", required=True),
+            arg("hold", "Keep the balloon open until the next action.", kind="boolean"),
+        ),
+    ),
+    Action(
+        name="avatar_list_animations",
+        method="LOCAL",
+        path="",
+        summary="List every animation the current character supports",
+        description=DEVII,
+        handler="avatar",
+        requires_auth=False,
+    ),
+    Action(
+        name="avatar_play_animation",
+        method="LOCAL",
+        path="",
+        summary="Play a specific named animation (use avatar_list_animations first)",
+        description=(
+            DEVII + " A following avatar_hide or avatar_stop interrupts an animation that is "
+            "still playing, so do not hide immediately after if you want it to finish."
+        ),
+        handler="avatar",
+        requires_auth=False,
+        params=(arg("name", "Exact animation name.", required=True),),
+    ),
+    Action(
+        name="avatar_random_animation",
+        method="LOCAL",
+        path="",
+        summary="Play a random non-idle animation to react expressively",
+        description=DEVII,
+        handler="avatar",
+        requires_auth=False,
+    ),
+    Action(
+        name="avatar_move_to",
+        method="LOCAL",
+        path="",
+        summary="Walk devii to a pixel position on screen",
+        description=DEVII,
+        handler="avatar",
+        requires_auth=False,
+        params=(
+            arg("x", "Target x in pixels.", required=True, kind="integer"),
+            arg("y", "Target y in pixels.", required=True, kind="integer"),
+            arg("duration", "Movement duration in milliseconds.", kind="integer"),
+        ),
+    ),
+    Action(
+        name="avatar_gesture_at",
+        method="LOCAL",
+        path="",
+        summary="Make devii gesture toward a pixel position",
+        description=DEVII,
+        handler="avatar",
+        requires_auth=False,
+        params=(
+            arg("x", "Target x in pixels.", required=True, kind="integer"),
+            arg("y", "Target y in pixels.", required=True, kind="integer"),
+        ),
+    ),
+    Action(
+        name="avatar_get_viewport",
+        method="LOCAL",
+        path="",
+        summary="Get the screen size and devii's current position",
+        description=DEVII,
+        handler="avatar",
+        requires_auth=False,
+    ),
+    Action(
+        name="avatar_stop",
+        method="LOCAL",
+        path="",
+        summary="Stop all of devii's queued animations and speech immediately",
+        description=DEVII,
+        handler="avatar",
+        requires_auth=False,
+    ),
+    Action(
+        name="avatar_list_characters",
+        method="LOCAL",
+        path="",
+        summary="List every character devii can appear as",
+        description=DEVII,
+        handler="avatar",
+        requires_auth=False,
+    ),
+    Action(
+        name="avatar_switch_character",
+        method="LOCAL",
+        path="",
+        summary="Change which character devii is rendered as",
+        description=DEVII,
+        handler="avatar",
+        requires_auth=False,
+        params=(arg("name", "Character name (e.g. Clippy, Merlin, Bonzi, Genie, Rover).", required=True),),
+    ),
+)

devplacepy/services/devii/actions/catalog.py

@@ -0,0 +1,938 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from .spec import Action, Catalog, Param
+
+
+def path(name: str, description: str, required: bool = True) -> Param:
+    return Param(name=name, location="path", description=description, required=required)
+
+
+def query(name: str, description: str, required: bool = False) -> Param:
+    return Param(name=name, location="query", description=description, required=required)
+
+
+def body(name: str, description: str, required: bool = False) -> Param:
+    return Param(name=name, location="body", description=description, required=required)
+
+
+def upload(name: str, description: str) -> Param:
+    return Param(name=name, location="file", description=description, required=True)
+
+
+ATTACHMENTS = "Comma separated attachment uids returned by upload_file."
+TARGET_TYPE = "Target type, e.g. post, comment, project, gist."
+
+ACTIONS: tuple[Action, ...] = (
+    Action(
+        name="auth_status",
+        method="GET",
+        path="",
+        summary="Report whether the current session is authenticated",
+        handler="status",
+        requires_auth=False,
+    ),
+    Action(
+        name="login",
+        method="POST",
+        path="/auth/login",
+        summary="Authenticate with email and password and start a session",
+        handler="login",
+        requires_auth=False,
+        params=(
+            body("email", "Account email address.", required=True),
+            body("password", "Account password.", required=True),
+            body("remember_me", "Keep the session alive longer ('on' or '')."),
+        ),
+    ),
+    Action(
+        name="logout",
+        method="GET",
+        path="/auth/logout",
+        summary="End the current session",
+        handler="logout",
+        requires_auth=True,
+    ),
+    Action(
+        name="signup",
+        method="POST",
+        path="/auth/signup",
+        summary="Create a new account",
+        requires_auth=False,
+        params=(
+            body("username", "Desired username.", required=True),
+            body("email", "Email address.", required=True),
+            body("password", "Password (minimum six characters).", required=True),
+            body("confirm_password", "Password confirmation.", required=True),
+        ),
+    ),
+    Action(
+        name="forgot_password",
+        method="POST",
+        path="/auth/forgot-password",
+        summary="Request a password reset email",
+        requires_auth=False,
+        params=(body("email", "Account email address.", required=True),),
+    ),
+    Action(
+        name="reset_password",
+        method="POST",
+        path="/auth/reset-password/{token}",
+        summary="Reset a password using a reset token",
+        requires_auth=False,
+        params=(
+            path("token", "Reset token from the email link."),
+            body("password", "New password.", required=True),
+            body("confirm_password", "New password confirmation.", required=True),
+        ),
+    ),
+    Action(
+        name="view_feed",
+        method="GET",
+        path="/feed",
+        summary="View the activity feed",
+        params=(
+            query("tab", "Feed tab to view."),
+            query("topic", "Filter by topic."),
+            query("before", "Pagination cursor."),
+        ),
+    ),
+    Action(
+        name="create_post",
+        method="POST",
+        path="/posts/create",
+        summary="Create a new post",
+        params=(
+            body("content", "Post body text.", required=True),
+            body("title", "Optional post title."),
+            body("topic", "Optional topic."),
+            body("project_uid", "Attach the post to a project uid."),
+            body("attachment_uids", ATTACHMENTS),
+            body("poll_question", "Optional poll question."),
+            body("poll_options", "Poll options as a JSON array of strings, or one option per line, or comma separated. At least two are required for the poll to be created."),
+        ),
+    ),
+    Action(
+        name="view_post",
+        method="GET",
+        path="/posts/{post_slug}",
+        summary="View a single post by slug",
+        params=(path("post_slug", "Exact post slug copied from a /posts/... link in a feed or listing response; do not build it from the title."),),
+    ),
+    Action(
+        name="edit_post",
+        method="POST",
+        path="/posts/edit/{post_slug}",
+        summary="Edit an existing post",
+        params=(
+            path("post_slug", "Exact post slug copied from a /posts/... link in a feed or listing response; do not build it from the title."),
+            body("content", "Updated post body.", required=True),
+            body("title", "Updated title."),
+            body("topic", "Updated topic."),
+            body("poll_question", "Optional poll question. Adds a poll to a post that does not already have one."),
+            body("poll_options", "Poll options as a JSON array of strings, or one option per line, or comma separated. At least two are required for the poll to be created."),
+        ),
+    ),
+    Action(
+        name="delete_post",
+        method="POST",
+        path="/posts/delete/{post_slug}",
+        summary="Delete a post",
+        params=(path("post_slug", "Exact post slug copied from a /posts/... link in a feed or listing response; do not build it from the title."),),
+    ),
+    Action(
+        name="create_comment",
+        method="POST",
+        path="/comments/create",
+        summary="Create a comment on a post or other target",
+        params=(
+            body("content", "Comment body.", required=True),
+            body("post_uid", "Uid of the post being commented on."),
+            body("target_uid", "Uid of the target when not a post."),
+            body("target_type", TARGET_TYPE),
+            body("parent_uid", "Parent comment uid for replies."),
+            body("attachment_uids", ATTACHMENTS),
+        ),
+    ),
+    Action(
+        name="delete_comment",
+        method="POST",
+        path="/comments/delete/{comment_uid}",
+        summary="Delete a comment",
+        params=(path("comment_uid", "Uid of the comment."),),
+    ),
+    Action(
+        name="list_projects",
+        method="GET",
+        path="/projects",
+        summary="List projects",
+        params=(
+            query("tab", "Projects tab."),
+            query("search", "Search query."),
+            query("user_uid", "Filter by owner uid."),
+            query("project_type", "Filter by project type."),
+            query("before", "Pagination cursor."),
+        ),
+    ),
+    Action(
+        name="view_project",
+        method="GET",
+        path="/projects/{project_slug}",
+        summary="View a project by slug",
+        params=(path("project_slug", "Exact project slug copied from a /projects/... link in a listing response; do not build it from the title."),),
+    ),
+    Action(
+        name="create_project",
+        method="POST",
+        path="/projects/create",
+        summary="Create a new project",
+        params=(
+            body("title", "Project title.", required=True),
+            body("description", "Project description.", required=True),
+            body("release_date", "Release date."),
+            body("demo_date", "Demo date."),
+            body("project_type", "Project type."),
+            body("platforms", "Supported platforms."),
+            body("status", "Project status."),
+            body("attachment_uids", ATTACHMENTS),
+        ),
+    ),
+    Action(
+        name="delete_project",
+        method="POST",
+        path="/projects/delete/{project_slug}",
+        summary="Delete a project",
+        params=(path("project_slug", "Exact project slug copied from a /projects/... link in a listing response; do not build it from the title."),),
+    ),
+    Action(
+        name="project_set_private",
+        method="POST",
+        path="/projects/{project_slug}/private",
+        summary="Mark a project private (owner-only) or public",
+        description=(
+            "Set value=true to hide the project from everyone except its owner (and administrators), "
+            "or value=false to make it public again. This is a significant change: you MUST ask the "
+            "user for explicit confirmation BEFORE calling it, and only pass confirm=true once they "
+            "have agreed. Only the project owner may change this."
+        ),
+        params=(
+            path("project_slug", "Project slug or uid."),
+            body("value", "true to make the project private, false to make it public.", required=True),
+            body("confirm", "Must be true, set only after the user has explicitly confirmed.", required=True),
+        ),
+    ),
+    Action(
+        name="project_set_readonly",
+        method="POST",
+        path="/projects/{project_slug}/readonly",
+        summary="Mark a project read-only (immutable files) or writable again",
+        description=(
+            "Set value=true to make every file in the project immutable - no writes, edits, line "
+            "edits, moves, deletes, or uploads succeed afterwards, from anyone including you - or "
+            "value=false to allow changes again. This is a significant change: you MUST ask the user "
+            "for explicit confirmation BEFORE calling it, and only pass confirm=true once they have "
+            "agreed. Only the project owner may change this."
+        ),
+        params=(
+            path("project_slug", "Project slug or uid."),
+            body("value", "true to make the project read-only, false to make it writable.", required=True),
+            body("confirm", "Must be true, set only after the user has explicitly confirmed.", required=True),
+        ),
+    ),
+    Action(
+        name="project_list_files",
+        method="GET",
+        path="/projects/{project_slug}/files",
+        summary="List every file and directory in a project filesystem",
+        description="Returns the flat list of nodes (path, type, size, mime). Use it to inspect the project tree before reading or writing files.",
+        params=(path("project_slug", "Project slug or uid."),),
+        requires_auth=False,
+    ),
+    Action(
+        name="project_read_file",
+        method="GET",
+        path="/projects/{project_slug}/files/raw",
+        summary="Read one file from a project filesystem",
+        description="Returns the file metadata plus the text content. Binary files return a url instead of content.",
+        params=(
+            path("project_slug", "Project slug or uid."),
+            query("path", "Relative file path inside the project, e.g. src/main.py.", required=True),
+        ),
+        requires_auth=False,
+    ),
+    Action(
+        name="project_write_file",
+        method="POST",
+        path="/projects/{project_slug}/files/write",
+        summary="Create or overwrite a whole text file in a project (parent directories are created automatically)",
+        description=(
+            "Replaces the ENTIRE file with the content you send. Creating a new file needs no prior "
+            "read, but overwriting an existing one requires reading it first (project_read_file). "
+            "For an existing file prefer the surgical line tools (project_replace_lines, "
+            "project_insert_lines, project_delete_lines, project_append_file) instead of rewriting it, "
+            "and never batch several writes in one turn. Use this only to create a new file or fully "
+            "rewrite a small one. Missing parent directories are created recursively."
+        ),
+        params=(
+            path("project_slug", "Project slug or uid."),
+            body("path", "Relative file path, e.g. src/app/main.py.", required=True),
+            body("content", "Full file content.", required=True),
+        ),
+    ),
+    Action(
+        name="project_read_lines",
+        method="GET",
+        path="/projects/{project_slug}/files/lines",
+        summary="Read a 1-indexed line range of a text file in a project",
+        description=(
+            "Returns {path, start, end, total_lines, lines, content} for the requested range. Use it "
+            "to inspect part of a large file before editing, and to learn total_lines so you can target "
+            "the right range with the line-edit tools."
+        ),
+        params=(
+            path("project_slug", "Project slug or uid."),
+            query("path", "Relative file path inside the project.", required=True),
+            query("start", "First line to read (1-indexed, default 1)."),
+            query("end", "Last line to read (inclusive); omit for end of file."),
+        ),
+        requires_auth=False,
+    ),
+    Action(
+        name="project_replace_lines",
+        method="POST",
+        path="/projects/{project_slug}/files/replace-lines",
+        summary="Replace an inclusive 1-indexed line range of a text file with new content",
+        description=(
+            "Surgically rewrites lines start..end (inclusive) with content (which may be any number of "
+            "lines, or empty to delete the range). The preferred way to edit an existing file: it leaves "
+            "the rest of the file untouched and never overflows the model output limit."
+        ),
+        params=(
+            path("project_slug", "Project slug or uid."),
+            body("path", "Relative file path.", required=True),
+            body("start", "First line to replace (1-indexed).", required=True),
+            body("end", "Last line to replace (inclusive).", required=True),
+            body("content", "Replacement text for those lines (empty deletes them)."),
+        ),
+    ),
+    Action(
+        name="project_insert_lines",
+        method="POST",
+        path="/projects/{project_slug}/files/insert-lines",
+        summary="Insert content before a 1-indexed line of a text file",
+        description=(
+            "Inserts content before line 'at' without touching existing lines. Use at=1 to prepend and "
+            "at=total_lines+1 to insert at the end."
+        ),
+        params=(
+            path("project_slug", "Project slug or uid."),
+            body("path", "Relative file path.", required=True),
+            body("at", "Insert before this 1-indexed line (1 prepends, total+1 appends).", required=True),
+            body("content", "Text to insert.", required=True),
+        ),
+    ),
+    Action(
+        name="project_delete_lines",
+        method="POST",
+        path="/projects/{project_slug}/files/delete-lines",
+        summary="Delete an inclusive 1-indexed line range from a text file",
+        params=(
+            path("project_slug", "Project slug or uid."),
+            body("path", "Relative file path.", required=True),
+            body("start", "First line to delete (1-indexed).", required=True),
+            body("end", "Last line to delete (inclusive).", required=True),
+        ),
+    ),
+    Action(
+        name="project_append_file",
+        method="POST",
+        path="/projects/{project_slug}/files/append",
+        summary="Append content to the end of a text file in a project",
+        description="Adds content as new lines at the end of the file. Use it to grow a large file across turns without resending the whole thing.",
+        params=(
+            path("project_slug", "Project slug or uid."),
+            body("path", "Relative file path.", required=True),
+            body("content", "Text to append.", required=True),
+        ),
+    ),
+    Action(
+        name="project_upload_file",
+        method="POST",
+        path="/projects/{project_slug}/files/upload",
+        summary="Upload a local file into a project directory (parents created automatically)",
+        params=(
+            path("project_slug", "Project slug or uid."),
+            upload("file", "Local filesystem path of the file to upload."),
+            body("path", "Target directory inside the project, empty for the root."),
+        ),
+    ),
+    Action(
+        name="project_make_dir",
+        method="POST",
+        path="/projects/{project_slug}/files/mkdir",
+        summary="Create a directory (and parents) in a project filesystem",
+        params=(
+            path("project_slug", "Project slug or uid."),
+            body("path", "Relative directory path, e.g. src/components.", required=True),
+        ),
+    ),
+    Action(
+        name="project_move_file",
+        method="POST",
+        path="/projects/{project_slug}/files/move",
+        summary="Move or rename a file or directory within a project",
+        params=(
+            path("project_slug", "Project slug or uid."),
+            body("from_path", "Existing path.", required=True),
+            body("to_path", "New path.", required=True),
+        ),
+    ),
+    Action(
+        name="project_delete_file",
+        method="POST",
+        path="/projects/{project_slug}/files/delete",
+        summary="Delete a file or directory (recursive) from a project filesystem",
+        params=(
+            path("project_slug", "Project slug or uid."),
+            body("path", "Relative path to delete.", required=True),
+        ),
+    ),
+    Action(
+        name="zip_project",
+        method="POST",
+        path="/projects/{project_slug}/zip",
+        summary="Build a downloadable zip archive of a whole project",
+        description=(
+            "Queues a background zip job and returns {uid, status_url}. Poll the status_url with "
+            "zip_status until status is 'done', then give the user the download_url."
+        ),
+        params=(path("project_slug", "Project slug or uid."),),
+        requires_auth=False,
+    ),
+    Action(
+        name="zip_status",
+        method="GET",
+        path="/zips/{uid}",
+        summary="Check a zip job and obtain its download link once finished",
+        description=(
+            "Returns the job status and stats. When status is 'done', download_url points at the "
+            "ready archive; while 'pending' or 'running', poll again shortly."
+        ),
+        params=(path("uid", "Zip job uid returned by zip_project."),),
+        requires_auth=False,
+    ),
+    Action(
+        name="fork_project",
+        method="POST",
+        path="/projects/{project_slug}/fork",
+        summary="Fork a project into a new project owned by the current user",
+        description=(
+            "Queues a background fork job and returns {uid, status_url}. Poll the status_url with "
+            "fork_status until status is 'done', then give the user the project_url of the new fork."
+        ),
+        params=(
+            path("project_slug", "Slug or uid of the project to fork."),
+            body("title", "Title for the new forked project.", required=True),
+        ),
+        requires_auth=True,
+    ),
+    Action(
+        name="fork_status",
+        method="GET",
+        path="/forks/{uid}",
+        summary="Check a fork job and obtain the new project once finished",
+        description=(
+            "Returns the job status. When status is 'done', project_url points at the new forked "
+            "project; while 'pending' or 'running', poll again shortly."
+        ),
+        params=(path("uid", "Fork job uid returned by fork_project."),),
+        requires_auth=False,
+    ),
+    Action(
+        name="search_users",
+        method="GET",
+        path="/profile/search",
+        summary="Search for users by name",
+        params=(query("q", "Search query."),),
+    ),
+    Action(
+        name="view_profile",
+        method="GET",
+        path="/profile/{username}",
+        summary="View a user profile",
+        params=(
+            path("username", "Username to view."),
+            query("tab", "Profile tab."),
+        ),
+    ),
+    Action(
+        name="update_profile",
+        method="POST",
+        path="/profile/update",
+        summary="Update the current user's profile",
+        params=(
+            body("bio", "Profile biography."),
+            body("location", "Location."),
+            body("git_link", "Git profile link."),
+            body("website", "Personal website."),
+        ),
+    ),
+    Action(
+        name="regenerate_api_key",
+        method="POST",
+        path="/profile/regenerate-api-key",
+        summary="Issue a new API key and invalidate the current one",
+        description=(
+            "Returns the new api_key. Warning: this immediately invalidates any key currently "
+            "used for authentication, so confirm with the user before calling it."
+        ),
+    ),
+    Action(
+        name="list_messages",
+        method="GET",
+        path="/messages",
+        summary="View direct message conversations",
+        params=(
+            query("with_uid", "Open a conversation with a user uid."),
+            query("search", "Search conversations."),
+        ),
+    ),
+    Action(
+        name="search_message_users",
+        method="GET",
+        path="/messages/search",
+        summary="Search users to message",
+        params=(query("q", "Search query."),),
+    ),
+    Action(
+        name="send_message",
+        method="POST",
+        path="/messages/send",
+        summary="Send a direct message",
+        params=(
+            body("content", "Message body.", required=True),
+            body("receiver_uid", "Recipient user uid.", required=True),
+            body("attachment_uids", ATTACHMENTS),
+        ),
+    ),
+    Action(
+        name="list_notifications",
+        method="GET",
+        path="/notifications",
+        summary="List notifications",
+        params=(query("before", "Pagination cursor."),),
+    ),
+    Action(
+        name="notification_counts",
+        method="GET",
+        path="/notifications/counts",
+        summary="Get unread notification and message counts",
+    ),
+    Action(
+        name="open_notification",
+        method="GET",
+        path="/notifications/open/{notification_uid}",
+        summary="Open a notification and follow it",
+        params=(path("notification_uid", "Notification uid."),),
+    ),
+    Action(
+        name="mark_notification_read",
+        method="POST",
+        path="/notifications/mark-read/{notification_uid}",
+        summary="Mark a single notification read",
+        params=(path("notification_uid", "Notification uid."),),
+    ),
+    Action(
+        name="mark_all_notifications_read",
+        method="POST",
+        path="/notifications/mark-all-read",
+        summary="Mark all notifications read",
+    ),
+    Action(
+        name="vote",
+        method="POST",
+        path="/votes/{target_type}/{target_uid}",
+        summary="Cast or toggle a vote on a target",
+        description="Re-sending the same value removes the vote. Returns the net/up/down tally.",
+        ajax=True,
+        params=(
+            path("target_type", TARGET_TYPE),
+            path("target_uid", "Uid of the target, copied from a listing response; do not invent it."),
+            body("value", "Vote value: 1 to upvote, -1 to downvote (re-send to remove).", required=True),
+        ),
+    ),
+    Action(
+        name="react",
+        method="POST",
+        path="/reactions/{target_type}/{target_uid}",
+        summary="Toggle an emoji reaction on a target",
+        description="Re-sending the same emoji removes it. Returns reaction counts.",
+        ajax=True,
+        params=(
+            path("target_type", TARGET_TYPE),
+            path("target_uid", "Uid of the target, copied from a listing response; do not invent it."),
+            body("emoji", "One of the allowed reaction emoji.", required=True),
+        ),
+    ),
+    Action(
+        name="list_bookmarks",
+        method="GET",
+        path="/bookmarks/saved",
+        summary="List saved bookmarks",
+        params=(query("before", "Pagination cursor."),),
+    ),
+    Action(
+        name="toggle_bookmark",
+        method="POST",
+        path="/bookmarks/{target_type}/{target_uid}",
+        summary="Toggle a bookmark on a target",
+        description="Re-sending removes the bookmark. Returns {saved: true|false}.",
+        ajax=True,
+        params=(
+            path("target_type", TARGET_TYPE),
+            path("target_uid", "Uid of the target, copied from a listing response; do not invent it."),
+        ),
+    ),
+    Action(
+        name="vote_poll",
+        method="POST",
+        path="/polls/{poll_uid}/vote",
+        summary="Vote in a poll",
+        description="Casts or changes your vote on a poll option. Returns the option tallies.",
+        ajax=True,
+        params=(
+            path("poll_uid", "Poll uid."),
+            body("option_uid", "Chosen option uid.", required=True),
+        ),
+    ),
+    Action(
+        name="follow_user",
+        method="POST",
+        path="/follow/{username}",
+        summary="Follow a user",
+        params=(path("username", "Username to follow."),),
+    ),
+    Action(
+        name="unfollow_user",
+        method="POST",
+        path="/follow/unfollow/{username}",
+        summary="Unfollow a user",
+        params=(path("username", "Username to unfollow."),),
+    ),
+    Action(
+        name="list_followers",
+        method="GET",
+        path="/profile/{username}/followers",
+        summary="List the users who follow a profile",
+        requires_auth=False,
+        params=(
+            path("username", "Username whose followers to list."),
+            query("page", "Page number, 25 per page."),
+        ),
+    ),
+    Action(
+        name="list_following",
+        method="GET",
+        path="/profile/{username}/following",
+        summary="List the users a profile follows",
+        requires_auth=False,
+        params=(
+            path("username", "Username whose following to list."),
+            query("page", "Page number, 25 per page."),
+        ),
+    ),
+    Action(
+        name="view_leaderboard",
+        method="GET",
+        path="/leaderboard",
+        summary="View the leaderboard",
+    ),
+    Action(
+        name="list_bugs",
+        method="GET",
+        path="/bugs",
+        summary="List reported bugs",
+    ),
+    Action(
+        name="create_bug",
+        method="POST",
+        path="/bugs/create",
+        summary="Report a bug",
+        params=(
+            body("title", "Bug title.", required=True),
+            body("description", "Bug description.", required=True),
+            body("attachment_uids", ATTACHMENTS),
+        ),
+    ),
+    Action(
+        name="list_gists",
+        method="GET",
+        path="/gists",
+        summary="List gists",
+        params=(
+            query("language", "Filter by language."),
+            query("user_uid", "Filter by owner uid."),
+            query("before", "Pagination cursor."),
+        ),
+    ),
+    Action(
+        name="view_gist",
+        method="GET",
+        path="/gists/{gist_slug}",
+        summary="View a gist by slug",
+        params=(path("gist_slug", "Exact gist slug copied from a /gists/... link in a listing response; do not build it from the title."),),
+    ),
+    Action(
+        name="create_gist",
+        method="POST",
+        path="/gists/create",
+        summary="Create a gist",
+        params=(
+            body("title", "Gist title.", required=True),
+            body("source_code", "Gist source code.", required=True),
+            body("description", "Gist description."),
+            body("language", "Programming language."),
+            body("attachment_uids", ATTACHMENTS),
+        ),
+    ),
+    Action(
+        name="edit_gist",
+        method="POST",
+        path="/gists/edit/{gist_slug}",
+        summary="Edit a gist",
+        params=(
+            path("gist_slug", "Exact gist slug copied from a /gists/... link in a listing response; do not build it from the title."),
+            body("title", "Gist title.", required=True),
+            body("source_code", "Gist source code.", required=True),
+            body("description", "Gist description."),
+            body("language", "Programming language."),
+        ),
+    ),
+    Action(
+        name="delete_gist",
+        method="POST",
+        path="/gists/delete/{gist_slug}",
+        summary="Delete a gist",
+        params=(path("gist_slug", "Exact gist slug copied from a /gists/... link in a listing response; do not build it from the title."),),
+    ),
+    Action(
+        name="list_news",
+        method="GET",
+        path="/news",
+        summary="List news articles",
+        params=(query("before", "Pagination cursor."),),
+    ),
+    Action(
+        name="view_news",
+        method="GET",
+        path="/news/{news_slug}",
+        summary="View a news article by slug",
+        params=(path("news_slug", "Exact news slug copied from a /news/... link in a listing response; do not build it from the title."),),
+    ),
+    Action(
+        name="upload_file",
+        method="POST",
+        path="/uploads/upload",
+        summary="Upload a local file and obtain its attachment uid",
+        params=(upload("file", "Local filesystem path of the file to upload."),),
+    ),
+    Action(
+        name="delete_attachment",
+        method="DELETE",
+        path="/uploads/delete/{attachment_uid}",
+        summary="Delete an uploaded attachment",
+        params=(path("attachment_uid", "Uid of the attachment."),),
+    ),
+    Action(
+        name="admin_overview",
+        method="GET",
+        path="/admin",
+        summary="View the admin overview",
+    ),
+    Action(
+        name="site_analytics",
+        method="GET",
+        path="/admin/analytics",
+        summary="Site-wide aggregate analytics in one call (admin only)",
+        description=(
+            "Returns JSON: total members, active users in the last 24h/7d/30d, users signed in "
+            "now, new signups (24h/7d/30d), content totals (posts, comments, gists, projects, "
+            "news), and top authors. Use this for any 'how many'/'how active'/count question "
+            "instead of paging through admin_list_users."
+        ),
+        params=(query("top_n", "How many top authors to include (1-50)."),),
+        requires_admin=True,
+    ),
+    Action(
+        name="ai_usage",
+        method="GET",
+        path="/admin/ai-usage/data",
+        summary="AI gateway usage, cost, latency, and reliability metrics (admin only)",
+        description=(
+            "Returns JSON for a bounded window: request volume and throughput, token usage with "
+            "averages and percentiles, latency (upstream, gateway overhead, queue wait, connect), "
+            "error rates by category, cost in USD (per model, per caller, input vs output, projected "
+            "monthly burn, caching savings), caller behavior, and an hourly breakdown. Use this for "
+            "any question about AI spend, token consumption, gateway performance, or errors."
+        ),
+        params=(
+            query("hours", "Lookback window in hours (1-168, default 48)."),
+            query("top_n", "How many rows in each top-N breakdown (default 10)."),
+        ),
+        requires_admin=True,
+    ),
+    Action(
+        name="admin_list_users",
+        method="GET",
+        path="/admin/users",
+        summary="List users for administration",
+        params=(query("page", "Page number."),),
+    ),
+    Action(
+        name="admin_set_user_role",
+        method="POST",
+        path="/admin/users/{uid}/role",
+        summary="Set a user's role",
+        params=(
+            path("uid", "User uid."),
+            body("role", "New role.", required=True),
+        ),
+    ),
+    Action(
+        name="admin_set_user_password",
+        method="POST",
+        path="/admin/users/{uid}/password",
+        summary="Set a user's password",
+        params=(
+            path("uid", "User uid."),
+            body("password", "New password.", required=True),
+        ),
+    ),
+    Action(
+        name="admin_toggle_user",
+        method="POST",
+        path="/admin/users/{uid}/toggle",
+        summary="Toggle a user's active state",
+        params=(path("uid", "User uid."),),
+    ),
+    Action(
+        name="admin_get_settings",
+        method="GET",
+        path="/admin/settings",
+        summary="View site settings",
+    ),
+    Action(
+        name="admin_save_settings",
+        method="POST",
+        path="/admin/settings",
+        summary="Save site settings",
+        params=(
+            body("site_name", "Site name."),
+            body("site_description", "Site description."),
+            body("site_tagline", "Site tagline."),
+            body("max_upload_size_mb", "Maximum upload size in megabytes."),
+            body("allowed_file_types", "Allowed file types."),
+            body("max_attachments_per_resource", "Maximum attachments per resource."),
+            body("rate_limit_per_minute", "Rate limit per minute."),
+            body("rate_limit_window_seconds", "Rate limit window in seconds."),
+            body("session_max_age_days", "Session maximum age in days."),
+            body("session_remember_days", "Remember-me duration in days."),
+            body("registration_open", "Whether registration is open."),
+            body("maintenance_mode", "Whether maintenance mode is on."),
+            body("maintenance_message", "Maintenance message."),
+        ),
+    ),
+    Action(
+        name="admin_list_news",
+        method="GET",
+        path="/admin/news",
+        summary="List news for administration",
+        params=(query("page", "Page number."),),
+    ),
+    Action(
+        name="admin_toggle_news",
+        method="POST",
+        path="/admin/news/{uid}/toggle",
+        summary="Toggle a news article",
+        params=(path("uid", "News uid."),),
+    ),
+    Action(
+        name="admin_publish_news",
+        method="POST",
+        path="/admin/news/{uid}/publish",
+        summary="Publish a news article",
+        params=(path("uid", "News uid."),),
+    ),
+    Action(
+        name="admin_landing_news",
+        method="POST",
+        path="/admin/news/{uid}/landing",
+        summary="Set a news article as landing content",
+        params=(path("uid", "News uid."),),
+    ),
+    Action(
+        name="admin_delete_news",
+        method="POST",
+        path="/admin/news/{uid}/delete",
+        summary="Delete a news article",
+        params=(path("uid", "News uid."),),
+    ),
+    Action(
+        name="admin_list_services",
+        method="GET",
+        path="/admin/services",
+        summary="View managed services",
+    ),
+    Action(
+        name="admin_services_data",
+        method="GET",
+        path="/admin/services/data",
+        summary="Get live status, metrics, and log tail for every background service",
+    ),
+    Action(
+        name="admin_service_status",
+        method="GET",
+        path="/admin/services/{name}/data",
+        summary="Get live status, metrics, and log tail for one background service",
+        params=(path("name", "Service name (e.g. news, bots, openai)."),),
+    ),
+    Action(
+        name="admin_start_service",
+        method="POST",
+        path="/admin/services/{name}/start",
+        summary="Start a managed service",
+        params=(path("name", "Service name."),),
+    ),
+    Action(
+        name="admin_stop_service",
+        method="POST",
+        path="/admin/services/{name}/stop",
+        summary="Stop a managed service",
+        params=(path("name", "Service name."),),
+    ),
+    Action(
+        name="admin_run_service",
+        method="POST",
+        path="/admin/services/{name}/run",
+        summary="Run a managed service once",
+        params=(path("name", "Service name."),),
+    ),
+    Action(
+        name="admin_clear_service_logs",
+        method="POST",
+        path="/admin/services/{name}/clear-logs",
+        summary="Clear a managed service's logs",
+        params=(path("name", "Service name."),),
+    ),
+    Action(
+        name="admin_config_service",
+        method="POST",
+        path="/admin/services/{name}/config",
+        summary="Update a managed service's configuration",
+        params=(path("name", "Service name."),),
+        freeform_body=True,
+    ),
+)
+
+PLATFORM_CATALOG = Catalog(actions=ACTIONS)

devplacepy/services/devii/actions/chunk_actions.py

@@ -0,0 +1,42 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from .spec import Action, Param
+
+CHUNK_ACTIONS: tuple[Action, ...] = (
+    Action(
+        name="read_more",
+        method="LOCAL",
+        path="",
+        summary="Read the next part of a previously truncated tool result",
+        description=(
+            "When any tool result is truncated it includes a chunk_id, total_chars, "
+            "remaining_chars, and next_offset. Call read_more with that chunk_id and offset to "
+            "page through the rest until remaining_chars is 0, so you can read the whole content."
+        ),
+        handler="chunks",
+        requires_auth=False,
+        read_only=True,
+        params=(
+            Param(
+                name="chunk_id",
+                location="body",
+                description="The chunk_id from a truncated result.",
+                required=True,
+            ),
+            Param(
+                name="offset",
+                location="body",
+                description="Character offset to read from (use next_offset from the prior part).",
+                type="integer",
+            ),
+            Param(
+                name="length",
+                location="body",
+                description="Optional number of characters to return (capped to the response limit).",
+                type="integer",
+            ),
+        ),
+    ),
+)

devplacepy/services/devii/actions/client_actions.py

@@ -0,0 +1,131 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from .spec import Action, Param
+
+
+def arg(name: str, description: str, required: bool = False, kind: str = "string") -> Param:
+    return Param(name=name, location="body", description=description, required=required, type=kind)
+
+
+CLIENT = (
+    "Runs in the user's own browser through the web terminal. Only effective in the web "
+    "interface with a live connection; elsewhere it reports that no browser is attached."
+)
+
+CLIENT_ACTIONS: tuple[Action, ...] = (
+    Action(
+        name="get_page_context",
+        method="LOCAL",
+        path="",
+        summary="Read the user's current page: URL, title, viewport, scroll, selected text, visible headings, and whether they are signed in",
+        description=CLIENT + " Use this to understand where the user is and what they are looking at before acting or guiding them.",
+        handler="client",
+        requires_auth=False,
+        read_only=True,
+    ),
+    Action(
+        name="run_js",
+        method="LOCAL",
+        path="",
+        summary="Execute JavaScript in the user's browser and return its result",
+        description=(
+            CLIENT + " The code is the body of an async function; use 'return value' to return a "
+            "JSON-serializable result. You have full access to window and document. Use this for "
+            "anything not covered by the dedicated tools: read or change the DOM, drive a live "
+            "demo, inspect state, or update the screen. Prefer the dedicated tools "
+            "(highlight_element, show_toast, scroll_to_element, navigate_to, reload_page) when they fit."
+        ),
+        handler="client",
+        requires_auth=False,
+        params=(
+            arg("code", "JavaScript to run as an async function body. Return a JSON-serializable value.", required=True),
+        ),
+    ),
+    Action(
+        name="highlight_element",
+        method="LOCAL",
+        path="",
+        summary="Highlight an element on screen with an outline and an optional callout label, for live tutorials",
+        description=CLIENT + " Scrolls the element into view and draws an attention outline. Call clear_highlights to remove it.",
+        handler="client",
+        requires_auth=False,
+        params=(
+            arg("selector", "CSS selector, or the element's exact visible text (e.g. a heading or link label).", required=True),
+            arg("label", "Optional callout text shown next to the element."),
+        ),
+    ),
+    Action(
+        name="clear_highlights",
+        method="LOCAL",
+        path="",
+        summary="Remove all highlights and callouts placed by highlight_element",
+        description=CLIENT,
+        handler="client",
+        requires_auth=False,
+    ),
+    Action(
+        name="show_toast",
+        method="LOCAL",
+        path="",
+        summary="Show a brief on-screen message (toast) to the user",
+        description=CLIENT,
+        handler="client",
+        requires_auth=False,
+        params=(
+            arg("text", "Message to display.", required=True),
+            arg("duration_ms", "How long to show it, in milliseconds (default 4000).", kind="integer"),
+        ),
+    ),
+    Action(
+        name="scroll_to_element",
+        method="LOCAL",
+        path="",
+        summary="Smoothly scroll an element into view",
+        description=CLIENT,
+        handler="client",
+        requires_auth=False,
+        params=(arg("selector", "CSS selector, or the element's exact visible text.", required=True),),
+    ),
+    Action(
+        name="navigate_to",
+        method="LOCAL",
+        path="",
+        summary="Send the user's browser to a URL",
+        description=(
+            CLIENT + " Use a same-origin path like /feed or /docs/index.html, or a full URL. The "
+            "page reloads; the user's Devii session and conversation persist and reconnect automatically."
+        ),
+        handler="client",
+        requires_auth=False,
+        params=(arg("url", "Path or URL to navigate to.", required=True),),
+    ),
+    Action(
+        name="reload_page",
+        method="LOCAL",
+        path="",
+        summary="Refresh the user's current page, e.g. after something changed",
+        description=CLIENT + " The Devii session and conversation persist and reconnect automatically.",
+        handler="client",
+        requires_auth=False,
+    ),
+    Action(
+        name="open_terminal",
+        method="LOCAL",
+        path="",
+        summary="Open a floating interactive terminal window attached to a running container instance",
+        description=(
+            CLIENT + " Opens a new xterm.js window in the user's browser connected to the container's "
+            "interactive shell (admin only). Resolve the instance first with container_list_instances, "
+            "then pass the project slug and the instance slug or uid."
+        ),
+        handler="client",
+        requires_admin=True,
+        params=(
+            arg("project_slug", "Project slug or uid that owns the container.", required=True),
+            arg("instance", "Container instance slug or uid.", required=True),
+            arg("label", "Optional window title (defaults to the instance name)."),
+        ),
+    ),
+)

devplacepy/services/devii/actions/container_actions.py

@@ -0,0 +1,88 @@
+# retoor <retoor@molodetz.nl>
+
+from .spec import Action, Param
+
+
+def arg(name: str, description: str, required: bool = False, kind: str = "string") -> Param:
+    return Param(name=name, location="body", description=description, required=required, type=kind)
+
+
+SLUG = arg("project_slug", "Project slug or uid that owns the container resources.", required=True)
+
+CONTAINER_ACTIONS: tuple[Action, ...] = (
+    Action(
+        name="container_list_instances",
+        method="LOCAL", path="", handler="container", requires_admin=True, read_only=True,
+        summary="List a project's container instances and their status",
+        params=(SLUG,),
+    ),
+    Action(
+        name="container_create_instance",
+        method="LOCAL", path="", handler="container", requires_admin=True,
+        summary="Create and start a container instance (runs the shared ppy image with the project files mounted at /app)",
+        params=(
+            SLUG,
+            arg("name", "Instance name.", required=True),
+            arg("boot_command", "Optional command to run on boot, e.g. 'python app.py'."),
+            arg("restart_policy", "never, always, on-failure, or unless-stopped."),
+            arg("env", "Optional env vars as KEY=VALUE lines."),
+            arg("ports", "Port maps per line or comma separated. Use a bare container port (e.g. '8899') to auto-assign a unique host port above 20000, or 'host:container' to pin one."),
+            arg("cpu_limit", "Optional CPU limit, e.g. 1 or 1.5."),
+            arg("mem_limit", "Optional memory limit, e.g. 512m or 1g."),
+            arg("autostart", "Start immediately ('true' or 'false', default true)."),
+            arg("ingress_slug", "Optional public ingress slug; the service is then reachable at /p/<slug>."),
+            arg("ingress_port", "Container port to publish at /p/<slug> (must be one of the mapped ports).", kind="integer"),
+        ),
+    ),
+    Action(
+        name="container_instance_action",
+        method="LOCAL", path="", handler="container", requires_admin=True,
+        summary="Control an instance: start, stop, restart, pause, resume, delete, or sync",
+        description="sync imports the container /app workspace back into the project files.",
+        params=(
+            SLUG,
+            arg("instance", "Instance name, slug, or uid.", required=True),
+            arg("action", "start, stop, restart, pause, resume, delete, or sync.", required=True),
+        ),
+    ),
+    Action(
+        name="container_logs",
+        method="LOCAL", path="", handler="container", requires_admin=True, read_only=True,
+        summary="Read the recent logs of a running instance",
+        params=(
+            SLUG,
+            arg("instance", "Instance name, slug, or uid.", required=True),
+            arg("tail", "Number of log lines (default 200).", kind="integer"),
+        ),
+    ),
+    Action(
+        name="container_exec",
+        method="LOCAL", path="", handler="container", requires_admin=True,
+        summary="Run a one-shot command inside a running instance and return its output",
+        params=(
+            SLUG,
+            arg("instance", "Instance name, slug, or uid.", required=True),
+            arg("command", "Command to run, e.g. 'pip list'.", required=True),
+        ),
+    ),
+    Action(
+        name="container_stats",
+        method="LOCAL", path="", handler="container", requires_admin=True, read_only=True,
+        summary="Get aggregated resource and runtime statistics for an instance",
+        params=(SLUG, arg("instance", "Instance name, slug, or uid.", required=True)),
+    ),
+    Action(
+        name="container_schedule",
+        method="LOCAL", path="", handler="container", requires_admin=True,
+        summary="Schedule a start or stop of an instance (cron, interval, or one-time)",
+        params=(
+            SLUG,
+            arg("instance", "Instance name, slug, or uid.", required=True),
+            arg("action", "start or stop.", required=True),
+            arg("kind", "once, interval, or cron.", required=True),
+            arg("cron", "Cron expression for kind=cron, e.g. '0 2 * * *'."),
+            arg("run_at", "ISO time for kind=once, e.g. 2026-06-15T02:00:00."),
+            arg("every_seconds", "Interval seconds for kind=interval.", kind="integer"),
+        ),
+    ),
+)

devplacepy/services/devii/actions/cost_actions.py

@@ -0,0 +1,39 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from .spec import Action
+
+COST_ACTIONS: tuple[Action, ...] = (
+    Action(
+        name="usage_quota",
+        method="LOCAL",
+        path="",
+        summary="Report the current user's AI usage as a percentage of their rolling 24h quota",
+        description=(
+            "Returns ONLY the percentage of the rolling 24-hour AI quota the current user has "
+            "used, the number of turns taken today, and whether the limit is reached. It never "
+            "returns any cost, dollar amount, pricing, or spend figure. Use this for any "
+            "'how much have I used' or 'what percent of resources' question."
+        ),
+        handler="cost",
+        requires_auth=False,
+        read_only=True,
+    ),
+    Action(
+        name="cost_stats",
+        method="LOCAL",
+        path="",
+        summary="Report token usage and full USD cost statistics for the current session (administrators only)",
+        description=(
+            "Administrators only. Returns this session's LLM token counts (prompt, completion, "
+            "total, cache hit/miss, reasoning), the cost in USD broken down by cache-hit input, "
+            "cache-miss input, and output, per-request averages, cache hit rate, and session "
+            "timing. Financial figures must never be shown to non-admin users."
+        ),
+        handler="cost",
+        requires_auth=True,
+        requires_admin=True,
+        read_only=True,
+    ),
+)

devplacepy/services/devii/actions/customization_actions.py

@@ -0,0 +1,100 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from .spec import Action, Param
+
+
+def arg(name: str, description: str, required: bool = False, kind: str = "string") -> Param:
+    return Param(name=name, location="body", description=description, required=required, type=kind)
+
+
+SCOPE = (
+    "Either 'global' (applies to every page of the site for this user) or a page-type string "
+    "as returned by get_page_context.pageType (e.g. '/posts/{slug}'), which applies to every "
+    "page of that type."
+)
+
+CUSTOMIZATION_ACTIONS: tuple[Action, ...] = (
+    Action(
+        name="customize_list",
+        method="LOCAL",
+        path="",
+        summary="List the user's saved CSS/JS customizations (scopes, languages, sizes)",
+        description=(
+            "Returns every stored customization for the current user (or guest session), with its "
+            "scope, language, enabled flag and a short preview of the code."
+        ),
+        handler="customization",
+        requires_auth=False,
+        read_only=True,
+    ),
+    Action(
+        name="customize_get",
+        method="LOCAL",
+        path="",
+        summary="Read the full saved CSS or JS code for one scope",
+        description="Use this before editing so you build on existing customization rather than overwriting it.",
+        handler="customization",
+        requires_auth=False,
+        read_only=True,
+        params=(
+            arg("scope", SCOPE, required=True),
+            arg("lang", "Which code to read: 'css' or 'js'.", required=True),
+        ),
+    ),
+    Action(
+        name="customize_set_css",
+        method="LOCAL",
+        path="",
+        summary="Permanently save custom CSS for a page type or the whole site",
+        description=(
+            "Persists custom CSS for the current user. Before calling, ask the user whether it should "
+            "apply to the current page TYPE or to the whole site (global). The CSS is the final, full "
+            "stylesheet for that scope and replaces any previously saved CSS for it."
+        ),
+        handler="customization",
+        requires_auth=False,
+        params=(
+            arg("scope", SCOPE, required=True),
+            arg("css", "The full CSS source to save for this scope.", required=True),
+            arg("confirm", "Must be true after the user has confirmed the scope.", required=True, kind="boolean"),
+        ),
+    ),
+    Action(
+        name="customize_set_js",
+        method="LOCAL",
+        path="",
+        summary="Permanently save custom JavaScript for a page type or the whole site",
+        description=(
+            "Persists custom JavaScript for the current user. Before calling, ask the user whether it "
+            "should apply to the current page TYPE or to the whole site (global). The code runs after "
+            "the application boots, with access to window, document and the global 'app'. It is the "
+            "final, full script for that scope and replaces any previously saved JS for it."
+        ),
+        handler="customization",
+        requires_auth=False,
+        params=(
+            arg("scope", SCOPE, required=True),
+            arg("js", "The full JavaScript source to save for this scope.", required=True),
+            arg("confirm", "Must be true after the user has confirmed the scope.", required=True, kind="boolean"),
+        ),
+    ),
+    Action(
+        name="customize_reset",
+        method="LOCAL",
+        path="",
+        summary="Delete saved customizations, restoring the default look and behaviour",
+        description=(
+            "Removes saved customizations. Omit lang to remove both CSS and JS for the scope. Pass "
+            "scope='all' to remove every customization for this user. This cannot be undone."
+        ),
+        handler="customization",
+        requires_auth=False,
+        params=(
+            arg("scope", SCOPE + " Use 'all' to remove every customization.", required=True),
+            arg("lang", "Optional: limit the reset to 'css' or 'js'. Omit to remove both."),
+            arg("confirm", "Must be true after the user has confirmed the deletion.", required=True, kind="boolean"),
+        ),
+    ),
+)

devplacepy/services/devii/actions/dispatcher.py

@@ -0,0 +1,347 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any
+from urllib.parse import quote, urlencode
+
+from ..config import Settings
+from ..errors import (
+    AuthRequiredError,
+    DeviiError,
+    ToolInputError,
+    error_result,
+    unexpected_result,
+)
+from ..agentic.controller import AgenticController
+from ..agentic.state import record_mutation
+from ..avatar import AvatarController
+from ..chunks import ChunkController, get_store, serve_resource, wrap_if_large
+from ..cost import CostController
+from ..docs import DocsController
+from ..fetch import FetchController
+from ..http_client import PlatformClient
+from ..rsearch import RsearchController
+from ..tasks.controller import TaskController
+from ..text import format_response
+from .spec import Action, Catalog
+
+MUTATING_METHODS = ("POST", "DELETE", "PUT", "PATCH")
+
+CONFIRM_REQUIRED = {"project_set_readonly", "project_set_private", "customize_set_css", "customize_set_js", "customize_reset"}
+
+logger = logging.getLogger("devii.dispatch")
+
+
+def _is_confirmed(arguments: dict[str, Any]) -> bool:
+    return str(arguments.get("confirm", "")).strip().lower() in ("true", "1", "yes", "on")
+
+
+def confirmation_error(name: str, arguments: dict[str, Any]) -> ToolInputError | None:
+    if name not in CONFIRM_REQUIRED or _is_confirmed(arguments):
+        return None
+    if name in ("customize_set_css", "customize_set_js"):
+        scope = str(arguments.get("scope", "")).strip() or "(unspecified)"
+        return ToolInputError(
+            "Before saving, confirm the scope with the user: should this apply to THIS page type "
+            f"('{scope}') or to the whole site ('global')? Ask them, then call again with the chosen "
+            "scope and confirm=true."
+        )
+    if name == "customize_reset":
+        return ToolInputError(
+            "Deleting customizations cannot be undone. Ask the user to confirm, then call again with "
+            "confirm=true."
+        )
+    if name == "project_set_private":
+        making_private = str(arguments.get("value", "")).strip().lower() in ("true", "1", "yes", "on")
+        change = (
+            "hide the project from everyone except its owner and administrators"
+            if making_private
+            else "make the project public so everyone can see it and all its files"
+        )
+        return ToolInputError(
+            f"Changing a project's visibility will {change}. Ask the user to confirm this explicitly "
+            "first, then call again with confirm=true."
+        )
+    return ToolInputError(
+        "Setting a project read-only makes every file immutable and blocks all further "
+        "edits. Ask the user to confirm this explicitly first, then call again with "
+        "confirm=true."
+    )
+
+
+class Dispatcher:
+    def __init__(
+        self,
+        catalog: Catalog,
+        client: PlatformClient,
+        settings: Settings,
+        tasks: TaskController,
+        agentic: AgenticController,
+        avatar: AvatarController | None = None,
+        browser: Any = None,
+        is_admin: bool = False,
+        quota_provider: Any = None,
+        owner_kind: str = "guest",
+        owner_id: str = "",
+        virtual_tools: Any = None,
+    ) -> None:
+        self._actions = catalog.by_name()
+        self._client = client
+        self._settings = settings
+        self._tasks = tasks
+        self._agentic = agentic
+        self._avatar = avatar
+        self._browser = browser
+        self._is_admin = is_admin
+        self._fetch = FetchController(settings)
+        self._docs = DocsController(settings)
+        self._cost = CostController(quota_provider=quota_provider)
+        self._chunks = ChunkController(settings)
+        self._rsearch = RsearchController(settings)
+        from ..container import ContainerController
+        self._container = ContainerController(client)
+        from ..customization import CustomizationController
+        self._customization = CustomizationController(owner_kind, owner_id)
+        self._virtual_tools = virtual_tools
+        self._read_files: set[tuple[str, str]] = set()
+
+    @staticmethod
+    def _file_key(arguments: dict[str, Any]) -> tuple[str, str] | None:
+        from devplacepy.project_files import normalize_path, ProjectFileError as _PFError
+        raw = arguments.get("path")
+        if not raw:
+            return None
+        try:
+            path = normalize_path(raw)
+        except _PFError:
+            return None
+        return str(arguments.get("project_slug", "")), path
+
+    def is_read_only(self, name: str) -> bool:
+        action = self._actions.get(name)
+        return bool(action and action.is_read_only)
+
+    async def dispatch(self, name: str, arguments: dict[str, Any]) -> str:
+        action = self._actions.get(name)
+        if action is None:
+            if self._virtual_tools is not None and self._virtual_tools.has(name):
+                logger.info("Dispatch virtual tool %s args=%s", name, list(arguments))
+                try:
+                    return await self._virtual_tools.run(name, arguments)
+                except DeviiError as exc:
+                    return error_result(exc)
+                except Exception as exc:  # noqa: BLE001 - surfaced to the model as data
+                    logger.exception("Virtual tool %s crashed", name)
+                    return unexpected_result(exc)
+            return error_result(ToolInputError(f"Unknown tool: {name}"))
+
+        logger.info("Dispatch %s args=%s", name, list(arguments))
+        try:
+            if action.requires_auth and not self._client.authenticated:
+                raise AuthRequiredError(
+                    "Not authenticated. Ask the user for credentials and call the login tool first.",
+                    tool=name,
+                )
+            if action.requires_admin and not self._is_admin:
+                raise AuthRequiredError(
+                    "This information is restricted to administrators.",
+                    tool=name,
+                )
+            guard = confirmation_error(name, arguments)
+            if guard is not None:
+                raise guard
+            resource_key = self._resource_key(action, arguments)
+            if resource_key:
+                cached = serve_resource(resource_key, self._settings.max_response_chars)
+                if cached is not None:
+                    logger.info("Resource cache hit for %s (%s)", name, resource_key)
+                    return cached
+            result = await self._run(action, arguments)
+            if action.handler == "chunks":
+                return result
+            return wrap_if_large(result, self._settings.max_response_chars, resource_key)
+        except DeviiError as exc:
+            logger.info("Dispatch %s failed: %s", name, exc.message)
+            return error_result(exc)
+        except Exception as exc:  # noqa: BLE001 - surfaced to the model as data
+            logger.exception("Dispatch %s crashed", name)
+            return unexpected_result(exc)
+
+    async def _run(self, action: Action, arguments: dict[str, Any]) -> str:
+        if action.handler == "status":
+            return json.dumps(
+                {
+                    "authenticated": self._client.authenticated,
+                    "user": self._client.username,
+                },
+                ensure_ascii=False,
+            )
+
+        if action.handler == "login":
+            result = await self._client.login(
+                email=self._require(arguments, "email"),
+                password=self._require(arguments, "password"),
+                remember_me=str(arguments.get("remember_me", "on")).lower() not in ("", "false", "off", "no"),
+            )
+            return json.dumps(result, ensure_ascii=False)
+
+        if action.handler == "logout":
+            return json.dumps(await self._client.logout(), ensure_ascii=False)
+
+        if action.handler == "task":
+            return await self._tasks.dispatch(action.name, arguments)
+
+        if action.handler == "agentic":
+            return await self._agentic.dispatch(action.name, arguments)
+
+        if action.handler == "fetch":
+            return await self._fetch.dispatch(action.name, arguments)
+
+        if action.handler == "docs":
+            return await self._docs.dispatch(action.name, arguments)
+
+        if action.handler == "cost":
+            return await self._cost.dispatch(action.name, arguments)
+
+        if action.handler == "chunks":
+            return await self._chunks.dispatch(action.name, arguments)
+
+        if action.handler == "rsearch":
+            return await self._rsearch.dispatch(action.name, arguments)
+
+        if action.handler == "container":
+            return await self._container.dispatch(action.name, arguments)
+
+        if action.handler == "customization":
+            return await self._customization.dispatch(action.name, arguments)
+
+        if action.handler == "virtual_tool":
+            if self._virtual_tools is None:
+                return error_result(
+                    ToolInputError("User-defined tools are not available in this context.")
+                )
+            return await self._virtual_tools.dispatch(action.name, arguments)
+
+        if action.handler == "avatar":
+            if self._avatar is None:
+                return error_result(
+                    ToolInputError(
+                        "No avatar is attached; devii's on-screen actions need the web interface."
+                    )
+                )
+            return await self._avatar.dispatch(action.name, arguments)
+
+        if action.handler == "client":
+            if self._browser is None:
+                return error_result(
+                    ToolInputError(
+                        "No browser is attached; client-side actions need the web interface."
+                    )
+                )
+            return await self._browser.dispatch(action.name, arguments)
+
+        return await self._run_http(action, arguments)
+
+    def _build_request(
+        self, action: Action, arguments: dict[str, Any]
+    ) -> tuple[str, dict[str, Any], dict[str, Any], tuple[str, str] | None]:
+        url_path = action.path
+        params: dict[str, Any] = {}
+        data: dict[str, Any] = {}
+        file_field: tuple[str, str] | None = None
+
+        for param in action.params:
+            if param.name not in arguments or arguments[param.name] is None:
+                if param.required:
+                    raise ToolInputError(
+                        f"Missing required parameter '{param.name}' for {action.name}.")
+                continue
+            value = arguments[param.name]
+            if param.location == "path":
+                url_path = url_path.replace("{" + param.name + "}", quote(str(value), safe=""))
+            elif param.location == "query":
+                params[param.name] = value
+            elif param.location == "body":
+                data[param.name] = value
+            elif param.location == "file":
+                file_field = (param.name, str(value))
+
+        if action.freeform_body:
+            extra = arguments.get("form_fields") or {}
+            if isinstance(extra, dict):
+                data.update({str(k): v for k, v in extra.items()})
+
+        return url_path, params, data, file_field
+
+    def _resource_key(self, action: Action, arguments: dict[str, Any]) -> str | None:
+        try:
+            if action.handler == "fetch" and action.name == "fetch_url":
+                url = str(arguments.get("url", "")).strip()
+                if not url:
+                    return None
+                if "://" not in url:
+                    url = "https://" + url
+                return f"fetch:{url}"
+            if action.handler == "http" and action.method == "GET":
+                url_path, params, _, _ = self._build_request(action, arguments)
+                query = urlencode(sorted((str(k), str(v)) for k, v in params.items()))
+                return f"http:GET {url_path}?{query}"
+        except ToolInputError:
+            return None
+        return None
+
+    async def _file_exists(self, arguments: dict[str, Any]) -> bool:
+        slug = str(arguments.get("project_slug", "")).strip()
+        path = str(arguments.get("path", "")).strip()
+        if not slug or not path:
+            return False
+        response = await self._client.call(
+            method="GET",
+            path=f"/projects/{quote(slug, safe='')}/files/raw",
+            params={"path": path},
+            headers={"X-Requested-With": "fetch"},
+        )
+        return response.status_code == 200
+
+    async def _run_http(self, action: Action, arguments: dict[str, Any]) -> str:
+        if action.name == "project_write_file":
+            key = self._file_key(arguments)
+            if key is not None and key not in self._read_files and await self._file_exists(arguments):
+                raise ToolInputError(
+                    f"Read '{key[1]}' before overwriting it. It already exists; call "
+                    "project_read_file first. For an existing file prefer the line tools "
+                    "(project_replace_lines, project_insert_lines, project_delete_lines, "
+                    "project_append_file); project_write_file replaces the entire file."
+                )
+
+        url_path, params, data, file_field = self._build_request(action, arguments)
+
+        headers = {"X-Requested-With": "fetch"} if action.ajax else None
+        response = await self._client.call(
+            method=action.method,
+            path=url_path,
+            params=params or None,
+            data=data or None,
+            file_field=file_field,
+            headers=headers,
+        )
+        if action.name in ("project_read_file", "project_read_lines", "project_write_file"):
+            key = self._file_key(arguments)
+            if key is not None:
+                self._read_files.add(key)
+        if action.method in MUTATING_METHODS:
+            record_mutation(action.name)
+            store = get_store()
+            if store is not None:
+                store.invalidate_resources(prefix="http:")
+        return format_response(response)
+
+    @staticmethod
+    def _require(arguments: dict[str, Any], key: str) -> str:
+        value = arguments.get(key)
+        if value is None or str(value).strip() == "":
+            raise ToolInputError(f"Missing required field '{key}'.")
+        return str(value)

devplacepy/services/devii/actions/docs_actions.py

@@ -0,0 +1,36 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from .spec import Action, Param
+
+DOCS_ACTIONS: tuple[Action, ...] = (
+    Action(
+        name="search_docs",
+        method="LOCAL",
+        path="",
+        summary="Full-text search the DevPlace API documentation",
+        description=(
+            "Searches the platform's developer documentation and returns the most relevant "
+            "sections (title and content). Use it to confirm how an endpoint, parameter, or "
+            "feature works before acting."
+        ),
+        handler="docs",
+        requires_auth=False,
+        read_only=True,
+        params=(
+            Param(
+                name="query",
+                location="body",
+                description="What to look for in the documentation.",
+                required=True,
+            ),
+            Param(
+                name="max_results",
+                location="body",
+                description="Maximum number of documentation sections to return (1-10).",
+                type="integer",
+            ),
+        ),
+    ),
+)

devplacepy/services/devii/actions/fetch_actions.py

@@ -0,0 +1,37 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from .spec import Action, Param
+
+FETCH_ACTIONS: tuple[Action, ...] = (
+    Action(
+        name="fetch_url",
+        method="LOCAL",
+        path="",
+        summary="Fetch a web page and return its readable text content",
+        description=(
+            "Retrieves any http(s) URL as a real browser would (stealth headers) and returns "
+            "the page title and readable text with links preserved, safely truncated to fit "
+            "the context window. Use it to read external pages, then summarize or create posts, "
+            "gists, or articles from the content. Private and loopback addresses are refused."
+        ),
+        handler="fetch",
+        requires_auth=False,
+        read_only=True,
+        params=(
+            Param(
+                name="url",
+                location="body",
+                description="The page URL to fetch (https is assumed if no scheme is given).",
+                required=True,
+            ),
+            Param(
+                name="max_chars",
+                location="body",
+                description="Optional cap on returned characters; clamped to a context-safe maximum.",
+                type="integer",
+            ),
+        ),
+    ),
+)