From 9a8046ab2ad62fe787b86607b09d5aaff4cc103b Mon Sep 17 00:00:00 2001 From: retoor Date: Mon, 6 Jul 2026 03:57:47 +0000 Subject: [PATCH] feat: add ISSLOP AI usage analysis CLI commands, game router, and politics topic - Add `cmd_isslop_prune`, `cmd_isslop_clear`, `cmd_isslop_analyze` CLI commands for AI usage analysis job management - Register `/game` router with `index` and `farm` endpoints for Code Farm idle game - Add `politics` to allowed TOPICS constant replacing `signals` - Introduce `ISSLOP_DIR`, `ISSLOP_WORKSPACES_DIR`, `ISSLOP_RUNS_DIR`, `ISSLOP_MEDIA_DIR` config paths - Add `clear_user_stars` and `clear_user_projects_cache` calls on vote and project create/delete - Update `make prod` to use `nproc` workers via `DEVPLACE_WEB_WORKERS` env var - Convert `database.py` and `utils.py` to packages for modular structure - Add `devplace apikey` and `devplace token` CLI subcommands for API key and access token management --- AGENTS.md | 64 +++++- CLAUDE.md | 64 ++++-- README.md | 45 +++-- devplacepy/cli/__init__.py | 6 + devplacepy/cli/jobs.py | 116 +++++++++++ devplacepy/config.py | 8 + devplacepy/constants.py | 2 +- devplacepy/content.py | 9 + devplacepy/database/__init__.py | 3 +- devplacepy/database/ranking.py | 20 +- devplacepy/database/schema.py | 120 +++++++++++ devplacepy/database/soft_delete.py | 1 + devplacepy/docs_api/groups/admin.py | 2 +- devplacepy/docs_api/groups/auth.py | 14 +- devplacepy/docs_api/groups/containers.py | 11 +- devplacepy/docs_api/groups/profiles.py | 4 +- devplacepy/docs_api/groups/tools.py | 191 +++++++++++++++++- devplacepy/docs_prose.py | 29 ++- devplacepy/main.py | 15 +- devplacepy/models.py | 22 ++ devplacepy/routers/docs/pages.py | 12 ++ devplacepy/routers/issues/index.py | 29 ++- devplacepy/routers/messages.py | 89 ++++---- devplacepy/routers/profile/index.py | 24 ++- devplacepy/routers/tools/__init__.py | 3 +- devplacepy/routers/tools/index.py | 11 + devplacepy/schemas/__init__.py | 4 + devplacepy/schemas/jobs.py | 72 +++++++ devplacepy/services/audit/categories.py | 1 + devplacepy/services/bot/config.py | 8 +- devplacepy/services/bot/llm.py | 1 + devplacepy/services/containers/api.py | 14 +- devplacepy/services/containers/files/.vimrc | 6 +- devplacepy/services/containers/files/bot.py | 8 +- devplacepy/services/containers/files/d.py | 4 +- devplacepy/services/containers/files/dpc | Bin 3399192 -> 3578664 bytes devplacepy/services/containers/files/pagent | 10 +- .../services/devii/actions/catalog/tools.py | 54 +++++ .../devii/actions/container_actions.py | 4 +- .../services/devii/actions/dispatcher.py | 2 +- .../services/devii/container/controller.py | 17 +- devplacepy/services/game/store/farm.py | 8 + devplacepy/static/css/base.css | 2 +- devplacepy/static/css/docs.css | 91 +++++++++ devplacepy/static/css/variables.css | 2 +- devplacepy/static/js/ApiTester.js | 4 +- devplacepy/static/js/CommentManager.js | 1 + devplacepy/static/js/ContentEnhancer.js | 14 +- devplacepy/static/js/ContentRenderer.js | 13 +- devplacepy/static/js/EmojiPicker.js | 12 ++ devplacepy/static/js/Http.js | 4 +- devplacepy/static/js/MessagesLayout.js | 10 +- devplacepy/static/js/components/AppContent.js | 4 + devplacepy/static/js/components/AppTitle.js | 4 + devplacepy/static/js/components/index.js | 2 + devplacepy/templates/base.html | 5 +- .../templates/docs/architecture-backend.html | 21 +- .../docs/architecture-conventions.html | 4 +- .../templates/docs/architecture-frontend.html | 2 + .../templates/docs/architecture-jobs.html | 4 +- .../templates/docs/architecture-styling.html | 2 +- .../templates/docs/architecture-workflow.html | 8 +- devplacepy/templates/docs/architecture.html | 8 +- devplacepy/templates/docs/audit-log.html | 7 +- devplacepy/templates/docs/backups.html | 6 + .../templates/docs/bots-architecture.html | 2 +- devplacepy/templates/docs/bots-config.html | 15 +- devplacepy/templates/docs/bots-content.html | 2 +- devplacepy/templates/docs/bots-personas.html | 6 +- devplacepy/templates/docs/claude-agents.html | 37 +++- .../templates/docs/claude-commands.html | 6 + devplacepy/templates/docs/claude.html | 2 +- devplacepy/templates/docs/code-farm.html | 28 ++- .../templates/docs/component-dp-content.html | 3 +- .../templates/docs/component-dp-toast.html | 2 +- devplacepy/templates/docs/components.html | 6 +- devplacepy/templates/docs/dashboard.html | 4 +- .../templates/docs/devii-architecture.html | 8 +- devplacepy/templates/docs/devii-config.html | 12 +- devplacepy/templates/docs/devii-data.html | 10 +- .../templates/docs/devii-internals.html | 9 +- devplacepy/templates/docs/devii-security.html | 9 + devplacepy/templates/docs/devii-tools.html | 71 ++++--- devplacepy/templates/docs/gamification.html | 29 ++- .../docs/getting-started-vibing.html | 30 +-- .../templates/docs/getting-started.html | 6 +- .../templates/docs/notification-settings.html | 26 ++- .../docs/production-concurrency.html | 2 +- .../templates/docs/production-deploy.html | 2 +- .../templates/docs/services-containers.html | 8 +- devplacepy/templates/docs/services-data.html | 6 +- .../templates/docs/services-framework.html | 2 +- devplacepy/templates/docs/services-news.html | 9 +- .../templates/docs/services-overview.html | 4 +- devplacepy/templates/docs/services-zip.html | 2 +- devplacepy/templates/docs/soft-delete.html | 2 +- devplacepy/templates/docs/styles-colors.html | 46 ++--- devplacepy/templates/docs/styles-layout.html | 8 +- .../templates/docs/styles-responsiveness.html | 2 +- devplacepy/templates/docs/testing-make.html | 5 +- devplacepy/templates/docs/testing.html | 2 +- devplacepy/templates/feed.html | 6 +- devplacepy/templating.py | 12 +- devplacepy/utils/badges.py | 1 + devplacepy/utils/rewards.py | 1 + pyproject.toml | 1 + tests/e2e/feed.py | 6 +- tests/e2e/post.py | 2 +- tests/unit/models.py | 27 +++ tests/unit/services/containers.py | 4 +- 110 files changed, 1466 insertions(+), 374 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 84bda4ec..114832c6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,7 +5,7 @@ ```bash make install # pip install -e . make dev # uvicorn --reload on port 10500, backlog 4096 -make prod # uvicorn with 2 workers, backlog 8192 (production) +make prod # uvicorn with nproc workers (DEVPLACE_WEB_WORKERS), backlog 8192 (production) make test # Playwright integration + unit tests, serial (one at a time), fail-fast -x make test-headed # same tests in a visible browser (single window) make locust # Locust load test (interactive web UI) @@ -67,6 +67,7 @@ Routers in `devplacepy/routers/` form a **directory tree that mirrors the endpoi | `/api` | `routers/devrant/` package (`auth`, `rants`, `comments`, `notifs`) | | `/dbapi` | `routers/dbapi/` package (`tables`, `query`, `nl`, `crud`) - primary administrator only (no internal-key path) | | `/pubsub` | `routers/pubsub.py` | +| `/game` | `routers/game/` package (`index`, `farm`) - the Code Farm idle game (state/leaderboard, plant/harvest/upgrade/prestige, and social farm water/steal) | | `(none)` | `routers/seo.py` (`/robots.txt`, `/sitemap.xml`) | ## Content Rendering Pipeline @@ -102,7 +103,7 @@ Elements with `data-render` attribute are auto-rendered by `Application.js` on p - It walks the `emoji` library's `EMOJI_DATA`, taking every `en` and `alias` name of a fully-qualified emoji (`status <= emoji.STATUS["fully_qualified"]`), then merges `CUSTOM_SHORTCODES` (legacy extras the library lacks, currently `party`/`tools`). First name wins (`setdefault`). - **Backend:** `EMOJI_MAP = build_emoji_shortcodes()` is computed ONCE at import; `_replace_shortcodes` (`_SHORTCODE_RE = :([A-Za-z0-9_+\-]+):`, case-insensitive) is the first step of `render_content`/`render_title`. An unknown name is returned verbatim. -- **Frontend:** `write_emoji_module()` serializes the SAME map to `static/js/emoji-shortcodes.js` as `export const EMOJI_SHORTCODES = {...}`. `ContentRenderer.js` imports it (`this.emojiMap = EMOJI_SHORTCODES`), so `dp-content`, `dp-title`, and every live renderer share the identical list with the backend. It is a plain ES6 module (synchronous static import - deliberately NOT a JSON-import-attribute or runtime `fetch`, to avoid a module-load failure on older engines and any first-render race). +- **Frontend:** `write_emoji_module()` serializes the SAME map to `static/js/emoji-shortcodes.js` as `export const EMOJI_SHORTCODES = {...}`. `ContentRenderer.js` loads it via a top-level dynamic `import("./emoji-shortcodes.js")` started at module evaluation (fire-and-forget, exposed as the `contentRenderer.ready` promise + `contentRenderer.emojiLoaded` flag), so the ~190KB map is OFF the boot-critical module graph on every page while `dp-content`, `dp-title`, and every live renderer still share the identical list with the backend. It stays a plain ES6 module (NOT a JSON-import-attribute or runtime `fetch`); the relative dynamic import inherits the versioned static path. **The first-render race is prevented by the ready gate, not by a static import:** every boot-time render entry awaits `contentRenderer.ready` (`ContentEnhancer.initContentRenderer`, the `AppContent`/`AppTitle` `connectedCallback` re-entry guard beside their existing DOMPurify/marked guard, `CommentManager.submitEdit`, `MessagesLayout.renderBody`, `ApiTester.buildNotes`), so no client-rendered content ever paints with unexpanded `:name:` codes. Any NEW call site that renders user content through `contentRenderer` MUST gate on `contentRenderer.ready` the same way. `replaceShortcodes` passes text through verbatim until the map is loaded (fail-soft on import error). - **The JS file is a committed build artifact, never hand-edited.** Regenerate it after bumping the `emoji` dependency with **`devplace emoji-sync`** (`cmd_emoji_sync` -> `write_emoji_module`, audited `cli.emoji.sync`). Backend and frontend stay byte-identical because both derive from the same generator at the same library version. - It is served under `static/js/`, so it inherits the versioned `immutable, max-age=31536000` static path (busted on deploy, cached for a year; the relative ES6 import inherits the version segment). User docs: `/docs/emoji-shortcodes`. @@ -251,7 +252,7 @@ The public **Tools -> SEO Diagnostics** auditor crawls a URL or sitemap with a h A separate async subservice (`SeoMetaService`, kind `seo_meta`) that generates clean SEO title/description/keywords for every published content item and meters its own AI spend. It is a distinct concern from the public **Tools -> SEO Diagnostics** auditor (kind `seo`) - do NOT conflate the two kinds; they share only the `jobs` queue table. It is the **constructive counterpart** to the diagnostics tool: diagnostics audits a URL, this one populates the on-page metadata. - **Tables.** `seo_metadata` is polymorphic and soft-deletable (in `SOFT_DELETE_TABLES`, born-live `deleted_at`/`deleted_by`): `uid, target_type, target_uid, seo_title, seo_description, seo_keywords, status (ready|pending|failed), source (ai|plain), generated_at, created_at, updated_at`, keyed UNIQUE on `(target_type, target_uid)`. `init_db()` ensures every column, the UNIQUE `idx_seo_metadata_target` and the live `idx_seo_metadata_status (status, deleted_at)` index. `seo_usage` is a single-row config-like usage table (NOT soft-delete) mirroring `news_usage`, keyed `SEO_USAGE_KEY="seo_meta"`. Helpers in `database.py`: `get_seo_metadata`/`get_seo_metadata_batch`/`has_fresh_seo_metadata`/`upsert_seo_metadata`/`mark_seo_metadata_stale` (every read filters `deleted_at IS NULL`; `get_seo_metadata` returns only `status="ready"` live rows) and `add_seo_usage`/`get_seo_usage`. -- **Choke helper.** `services/seo_meta.py` `schedule_seo_meta(target_type, uid, regenerate=False)` and `schedule_seo_meta_for_table(table, uid, ...)` are import-cycle-free (only `database` + `queue`). They no-op for unknown types, missing uid, or (without `regenerate`) when a fresh `ready` row exists; `regenerate=True` marks the row stale first; both skip a target with an existing pending/running `seo_meta` job; otherwise `queue.enqueue("seo_meta", {...}, "system", "seo_meta")`. Hooked at `content.create_content_item` (create, no-op guard) and `content.edit_content_item` (regenerate), the news publish sites in `services/news.py` (`status=="published"` only; existing-row update path uses `regenerate=True`), and `IssueCreateService` after the Gitea ticket is recorded. Because the work is async via the queue (NOT `run_in_executor`), the helper only enqueues. +- **Choke helper.** `services/seo_meta.py` `schedule_seo_meta(target_type, uid, regenerate=False)` and `schedule_seo_meta_for_table(table, uid, ...)` are import-cycle-free (only `database` + `queue`). They no-op for unknown types, missing uid, or (without `regenerate`) when a fresh `ready` row exists; `regenerate=True` marks the row stale first; both skip a target with an existing pending/running `seo_meta` job; otherwise `queue.enqueue("seo_meta", {...}, "system", "seo_meta")`. Hooked at `content.create_content_item` (create, no-op guard) and `content.edit_content_item` (regenerate), the news publish sites in `services/news/service.py` (`status=="published"` only; existing-row update path uses `regenerate=True`), and `IssueCreateService` after the Gitea ticket is recorded. Because the work is async via the queue (NOT `run_in_executor`), the helper only enqueues. - **`process`** loads the row (posts/projects/gists/news via `get_table`, issues via `gitea.store.get_ticket`), grounds via `services/ai_context.build_context` (fail-soft for news/empty `user_uid`), and calls `correction.gateway_complete(internal_gateway_key(), system, source_text, timeout)` **via `asyncio.to_thread`** (the gateway call is synchronous and posts to the in-process gateway on localhost, so it must run off the loop thread or it self-deadlocks the single worker - the correction.py sync-mode lesson). It demands strict JSON `{seo_title, seo_description, seo_keywords}`, parses fail-soft, re-clamps every field server-side (`seo_meta_text.clamp_generated`), and falls back to `seo_meta_text.plain_seo_defaults` (status `failed`, source `plain`) on any failure - **fields are never empty**. Usage accumulates via `correction.new_usage_totals` and flushes once with `database.add_seo_usage(totals)` when `calls>0`. Audit `seo.meta.generate`/`seo.meta.failed` (`record_system`, category `tools`). `cleanup()` is a no-op (the metadata is permanent, like a fork); only the job tracking row is swept. - **Backfill.** `run_once` calls `super().run_once()` then a bounded backfill sweep (gated by `seo_meta_backfill_enabled`, `seo_meta_backfill_batch` per type per tick) over published content lacking a fresh `ready` row, so pre-existing items get metadata with no one-shot migration. - **Admin surface.** `collect_metrics()` merges the `JobService` job-pipeline stats with `usage_metric_cards(get_seo_usage())`, so the **SEO Metadata** card on `/admin/services` shows the live task pipeline AND the AI cost/averages. The existing `admin.services.{name}` pub/sub topic + `live_view_relay` row push it live with NO new VIEWS row. A standard-paginated task list reads `queue.list_jobs(kind="seo_meta")` with `database.build_pagination` + `_pagination.html` when a dedicated page is desired. @@ -278,6 +279,32 @@ The public **Tools -> DeepSearch** researcher is a multi-agent deep web research - **RAG-audit hardening (engine correctness, no route/schema change):** (1) **Embedding-dimension consistency** - gateway embeddings carry provider-native dims while `local_embed` is fixed 256-dim; the worker `_index_chunks` now decides the backend ONCE per job (the first gateway failure or non-gateway result forces local for ALL remaining batches), and `VectorStore.add` drops any vector whose length differs from the collection's established dim, so one collection never mixes dims (cosine search across mixed dims is corrupt). `embeddings.EmbedResult.dims` and `VectorStore.dims` (lazily probed from the collection) expose the dimension; `chat.retrieve` re-embeds the query locally and skips retrieval if it still cannot match the stored dim. (2) **Citation grounding** - `orchestrate` drops any finding with no citations, and the summarizer/critic prompts forbid uncited claims and treat the QUESTION as data not an instruction (prompt-injection reduction via `_sanitize_question`); `chat._strip_unmatched_markers` removes any inline `[n]` marker that does not map to an emitted citation. (3) **Confidence calibration** - when only a single domain was crawled, `orchestrate` caps confidence by the source-diversity-derived ceiling (overconfidence guard). (4) `EmbeddingCache` is bounded at `EMBED_CACHE_MAX` to cap in-memory growth. - Devii tools `deepsearch`/`deepsearch_status`/`deepsearch_session` (public); docs `tools-deepsearch`; CLI `devplace deepsearch prune|clear`; audit `deepsearch.run.request|complete|failed` + `deepsearch.chat` (category `tools`). **New dependencies:** `chromadb`, `weasyprint`, `pypdf` (all unpinned). New runtime dirs `config.DEEPSEARCH_DIR`/`DEEPSEARCH_CHROMA_DIR` are registered in `DATA_PATHS`. **Add a dedicated nginx WS `location` for `/tools/deepsearch/{uid}/ws` and `/chat`** above `location /` for production, like the SEO and Devii sockets. +## AI Usage Analyzer tool (`services/jobs/isslop/`, `routers/tools/isslop.py`) + +The public **Tools -> AI Usage Analyzer** classifies a git repository or website as AI slop, sophisticated AI-assisted work or genuine human work. It is built on the standard async-job pattern (a `JobService` running a subprocess worker), but its live channel is **pub/sub, not a dedicated WS route**: every worker event is published to `public.isslop.{uid}` AND persisted to `isslop_events`, and the frontend pairs the pub/sub subscription with an incremental `GET /tools/isslop/{uid}/events?after=SEQ` poll, so guests (who cannot subscribe to `public.*` unless `pubsub_allow_guests` is on) and reconnecting tabs replay from the durable trail. Never rely on pub/sub alone for this tool: the DB event trail is the source of truth, pub/sub is the fast path. + +- **Engine layout:** `services/jobs/isslop/` holds `acquisition/` (git probe via `git ls-remote`, depth-1 clone with size preflight + live 3 GB kill guard, stealth Playwright website crawler with HTTP fallback, path-traversal-safe workspace helpers), `analysis/` (exclusion rules, stylometric metrics, language detection, per-repo baselines, `signals/` with one detector family per file, two-axis scoring), `agent/` (gateway LLM client, per-file classifier, vision reviewer, report writer with deterministic fallback), plus `pipeline.py` (the event-yielding run), `worker.py` (subprocess entry), `events.py` (frame protocol), `persistence.py` (`EventPersister` writes events/file results/image results/report and stamps the analysis row), `store.py` (all DB access), `badge.py` (SVG), `service.py` (`IsslopService`), `config.py` (all constants + `WorkerSettings`). +- **Worker contract:** `IsslopService.process` writes the worker payload (url + admin toggles + gateway endpoint/model/key) to `config.ISSLOP_RUNS_DIR/{uid}/payload.json`, resolves the workspace under `config.ISSLOP_WORKSPACES_DIR` (`workspace_for` rejects any path escaping the root), launches `python -m devplacepy.services.jobs.isslop.worker `, and relays each NDJSON stdout line through `EventPersister.apply` (SQLite) then `pubsub.publish`. The workspace and run dir are removed in a `finally`; the pipeline also deletes the workspace itself as its final act, so **no acquired source survives an analysis** - only the report and its evidence rows. +- **AI through the gateway only:** `agent/llm.py` talks solely to `config.INTERNAL_GATEWAY_URL` with model `molodetz` and `database.internal_gateway_key()` (vision uses the same model - the gateway handles image parts). `review_available`/`vision_available` gate the AI and image stages; on any gateway failure the static engine remains authoritative and the report falls back to the deterministic composer. All HTTP (gateway, git size preflight, website fallback crawl) goes through `stealth_async_client`. +- **Artifacts are permanent, the job row is not.** Like `ForkService`, `cleanup()` never touches `isslop_analyses`/`isslop_events`/`isslop_file_results`/`isslop_image_results`/`isslop_reports` - the report and badge are public capability URLs meant to outlive the run; the retention sweep removes only the `jobs` tracking row. `devplace isslop clear` is the only bulk hard-delete (plus per-analysis `store.purge_analysis`). +- **Ownership and guest history sync:** the owner is `("user", uid)` or `("guest", DEVII_GUEST_COOKIE)` - NOT the tools `_shared.owner_for` IP fallback, because history must survive IP changes and be claimable. The page/list/run handlers mint the guest cookie when absent (same cookie as Devii/customization, one guest identity platform-wide). `_sync_guest_history` runs on page and list requests: when a signed-in user still carries a guest cookie, `store.claim_guest_analyses` re-owns those rows via UPDATE (a move, never a copy - no duplicate data). One active analysis per owner (`429` otherwise, audited `denied`). +- **Tables:** `isslop_analyses` is soft-deletable (in `SOFT_DELETE_TABLES`, born-live inserts, reads filter `deleted_at IS NULL`, indexed on `(owner_kind, owner_id, created_at)`/`status`/`content_hash`); the evidence tables (`isslop_events` keyed `(analysis_uid, seq)`, `isslop_file_results`, `isslop_image_results`, `isslop_reports` UNIQUE on `analysis_uid`) are GC-only evidence purged with their analysis. All ensured in `init_db()`. +- **Routes** (all under `/tools/isslop`, capability URLs): `GET ""` page, `POST /run` (`IsslopRunForm`, http/git/ssh URL pattern), `GET /list` (owner history), `GET /{uid}` (`IsslopAnalysisOut`), `GET /{uid}/events` (ordered replay), `GET /{uid}/report` (`respond(..., IsslopReportOut)` - HTML shows the live `` while running and the server-rendered report when completed; the markdown body goes through `render_content`), `GET /{uid}/report.md`, `GET /{uid}/badge.svg` (self-contained SVG, hardcoded colors by design - it must render on external sites). Badge/report URLs are absolute via `seo.site_url`. +- **Frontend:** two site-wide web components (`static/js/components/AppIsslop.js` `` = submit form + history list; `AppIsslopRun.js` `` = live progress feed), registered in `components/index.js`, light DOM, reusing `Http`/`Poller` and `app.pubsub`. Page CSS `static/css/isslop.css` (design tokens). On `done` the run component reloads the page so the report is the server-rendered (SEO/`render_content`) version, never a client re-render. +- **Verdict blending is per-file, never a global mean (load-bearing).** The AI review pass runs its per-file gateway calls concurrently (`pipeline.AI_REVIEW_CONCURRENCY`, semaphore-bounded like the image pass) and adjusts ONLY the files it actually reviewed: `pipeline.apply_ai_verdicts` blends each sampled file's static origin/quality with its own verdict (0.6/0.4), then the WHOLE repo is re-aggregated with the normal SLOC/criticality weights, and `scoring.ai_fraction` maps per-file origin scores through a smooth 35-65 ramp (never the old hard 45/55 buckets). Never reintroduce a repo-level mean of the 12 sampled verdicts - it hands a tiny sample a fixed 40% of the verdict, so the LLM's clustered hedging values (30/40/50) drown thousands of files of static evidence and unrelated projects converge on identical percentages (the real-world twin-84%-human defect). **Single source of truth for the final verdict:** the SCORE event, the DONE payload and `generate_report` all consume the SAME final `RepoScores` (image influence applied via `scoring.adjust_for_images`, which recomputes slop/grade/human together) - the summary grade and the report body grade can therefore never disagree; `tests/unit/services/jobs/isslop/pipeline.py` guards both invariants. +- **Image evidence thumbnails + retry-safe evidence.** Workspaces die with the run, so the vision stage persists an aspect-preserving WebP thumbnail per reviewed image (`vision.make_thumbnail`, sha1-of-relative-path name) into `config.ISSLOP_MEDIA_DIR/{uid}` (the dir comes to the worker via the payload `media_dir`); the `thumb` name rides the `image` event, is stored on `isslop_image_results`, and is served by `GET /tools/isslop/{uid}/media/{name}` (strict `^[a-f0-9]{16}\.webp$` name pattern + `is_relative_to` root check - never loosen either). The report page renders the thumbnails with `data-lightbox` (the shared `app.lightbox` opens them full-size) and the live feed shows a tiny inline preview per image event. **`store.reset_evidence(uid)` runs at the top of every `IsslopService.process`** - a retried job (orphan recovery) previously re-inserted its events/file/image rows, duplicating every image and file in the report; any new evidence table MUST be added to `reset_evidence` AND `purge_analysis`. +- **Template provenance (the "ships defaults" detector).** `analysis/templates.py` `detect_template(workspace)` scores starter-template evidence repo-wide (it reads files the inventory excludes, like `package.json`): known template slugs/authors in the manifest (+ `ct3aMetadata`), README template marketing (weights capped so a wordy README cannot alone confirm), and the kitchen-sink scaffold constellation (count of standard scaffold artifacts past 3 freebies). The saturating score feeds `scoring.adjust_for_template` - a no-op below 35, a 0.7x floor on ai_percent/origin when confident, 0.85x when >= 70 - applied LAST in the pipeline scoring stage, after the AI and image blends. Near-certain evidence (>= 70) also FORCES category `ai-slop` (defaults shipped as-is are slop by the canonical definition - clean scaffold code never earns an untouched template `sophisticated-ai`, whose meaning is 'the presenter decided'); the confident band caps a `human-*` category at `uncertain`. Calibration truth set (guarded by tests): the six stock boilerplates (ixartz x2, create-t3-app, vercel ai-chatbot x2, fullstack-nextjs-app-template) grade C-D with markers listed, while devplacepy itself scores 0.0 with zero markers - tune weights against BOTH sides, never only the slop set. The evidence rides the `signal` inventory event, the SCORE payload (`template_score`/`template_markers`) and the report's Template Provenance section; admin toggle `isslop_template_detection`. +- **Rendered-DOM signal family (`analysis/domsignals/`).** A homepage-only, live-browser companion to the text-based `signals/webtells.py` checks: `acquisition/browser.py`'s `StealthBrowser.capture()` loads the page and returns the actual rendered DOM (computed styles, a class-name census, meta tags, headings/landmarks, console warnings, network response hosts/headers, a screenshot) via `DOM_EXTRACT_SCRIPT` in `acquisition/domcapture.py`. `domsignals/base.py` defines its own `@dom_check`/`@dom_site_check` decorator registry, mirroring the SEO job's `checks/` `@page_check`/`@site_check` mechanics, but emitting isslop's own `Signal` type (not a new one). Eight category modules (`builders`, `color`, `typography`, `layout`, `copy`, `metaseo`, `accessibility`, `buildsignals`) contribute 49 distinct signal codes; `domsignals/aggregate.py` `aggregate_dom_evidence` runs every registered check over the captured page(s) and saturates the weighted total into a `DomEvidence` (score/bucket/signals/detected_builder/builder_confidence). This brings the engine's total to twenty-one detector families (13 text-based in `signals/`, 8 rendered-DOM in `domsignals/`) and 126 signal codes (77 text-based, 49 rendered-DOM) - update these counts again the next time a family is added or removed, never leave them stale. +- **Homepage-only by design (`config.DOM_ANALYSIS_MAX_PAGES = 1`).** Capturing a full DOM, console, network trail and screenshot on every crawled page would multiply the browser cost of a run, so the pass runs ONLY against the first page at crawl depth 0 (`acquisition/website.py` gates `depth == 0 and index < DOM_ANALYSIS_MAX_PAGES`). Bump the constant later if the tool needs multi-page DOM coverage - `DomSiteContext`/`dom_site_check` (e.g. `detect_duplicate_meta_description`) already support more than one page. Git-repository sources never populate `dom_snapshots` (`dom_sink` is threaded only through `crawl_website`, never `clone_repository`), and a run where the browser could not be launched simply yields an empty page list, so `aggregate_dom_evidence([])` is a clean no-op (`DomEvidence(score=0.0, bucket="none")`). +- **Scoring order is load-bearing: images -> DOM -> template, never reorder.** `pipeline.py` applies `scoring.adjust_for_images`, then `adjust_for_dom_signals`, then `adjust_for_template`, in that exact sequence, and `adjust_for_template` MUST stay last. `adjust_for_dom_signals` blends `DomEvidence.score` into `ai_percent` at a small, deliberately cautious weight (`config.DOM_AI_WEIGHT = 0.12`, since this whole signal family is new and uncalibrated) UNLESS a confident builder match (`builder_confidence >= DOM_BUILDER_CONFIDENT_THRESHOLD`) forces the category to `ai-slop` via the same `_force_slop_category` helper the template detector uses. Because `adjust_for_template` runs after it, a template match can still floor/force the category further; nothing may run after `adjust_for_template`, since a later step would silently undo a forced `ai-slop` category. +- **DOM evidence never attaches to a per-file `FileScore`/`FileContext` (do not bolt it on).** `DomEvidence` is repo/page-level evidence blended once into the final `RepoScores`, exactly like template provenance and the image mean - never distributed across individual files. A rendered page has no SLOC and no line numbers to weight against, and its evidence (computed styles, a screenshot, console/network output) is not textual, so it cannot be scored, sampled or displayed through the SLOC-weighted per-file model the rest of the engine uses. Any new DOM check emits page-level `Signal`s into `DomEvidence.signals`, never into a `FileScore.signals` list. +- **`isslop_dom_results` follows the same evidence-table obligation as every other isslop table.** It mirrors `isslop_image_results` (`store.py` `TABLE_DOM_RESULTS`) and is already wired into both `store.reset_evidence(uid)` and `store.purge_analysis(uid)`; any future evidence table added under `domsignals/` must be added to both the same way. +- **DEP_UNRESOLVED is alias-aware (do not regress).** The JS/TS unresolved-import detector (`signals/hallucination.py`) flags imports that match no package.json dependency (all four sections), Node builtin or local path - i.e. phantom/hallucinated dependencies. It MUST skip everything that is not an npm specifier: relative/absolute/URL imports, `node:` and any `scheme:` specifier, the non-package prefixes `@/`, `~`, `#`, `$` (tsconfig/subpath/Svelte aliases - none are valid npm names), and every prefix parsed from `tsconfig.json`/`jsconfig.json` `compilerOptions.paths` (`engine._javascript_alias_prefixes`, carried on `RepoContext.javascript_alias_prefixes`). tsconfig is JSONC and alias keys contain `/*`, so comments are stripped with the string-aware scanner `_strip_jsonc_comments` - NEVER a comment regex (a regex eats the `"@/*"` alias strings themselves; this was a real defect). Before the alias awareness the detector flagged nearly every file of a standard Next.js app; after, only genuine phantom deps remain. `tests/unit/services/jobs/isslop/hallucination.py` guards it. +- **Clickable source references (annotated source viewer).** The static stage persists the FULL source of every signal-bearing file (cap `SOURCE_CAP_FILES`=60 files / 200KB each) into the same per-analysis media dir (`_persist_source`, `s.txt`); the name rides the `file` event and the `isslop_file_results.source` column. `GET /tools/isslop/{uid}/source?path=...&line=N` renders `isslop_source.html`: server-rendered line table (Jinja autoescape covers XSS) with line-number anchors `#LN`, signal lines highlighted with inline annotation chips, a focused line, and a findings nav in the sidebar; the strict `^s[a-f0-9]{16}\.txt$` + `is_relative_to` checks mirror the media route. Everything referencing a file links there: the file-results table path, each signal chip (`&line=N#LN`), and the report prose - `_linkify_sources` rewrites backticked paths in the report markdown into links BEFORE `render_content`, and the reporter system prompt requires the model to backtick every path it mentions. `reset_evidence`/`purge_analysis` already sweep the media dir, so sources share the thumbnail lifecycle. +- **No absolute paths ever leave the engine:** every user-facing path is workspace-relative (`relative_to(workspace)`); keep it that way in new detectors/events. +- **Docs heading anchors + `.docs-toc` (platform-wide):** `docs_prose._render_markdown` post-processes every prose page, stamping a slugified `id` on each `h2`/`h3` (`heading_slug`, GFM-style, deduplicated with `-N` suffixes) and appending a hover-visible `.docs-heading-anchor` permalink; `scroll-margin-top` keeps targets below the topnav. The `isslop-checks` page uses this for its clickable Contents grid: a `.docs-toc` nav placed OUTSIDE the `data-render` block (raw HTML passes through untouched) whose `href="#slug"` values are computed with the SAME `heading_slug` function at generation time - reuse `.docs-toc`/`.docs-toc-item`/`.docs-toc-count` (styled in `docs.css`) for any other long docs page, and never hand-write a slug that `heading_slug` would not produce. `tests/unit/docs_prose.py` guards the slugging and injection. +- Devii tools `isslop`/`isslop_status`/`isslop_report`/`isslop_list` (member-only, `requires_auth=True` per policy - the HTTP surface stays public); docs `tools-isslop` + `isslop-checks` (the full plain-language check catalog); CLI `devplace isslop analyze|prune|clear`; audit `isslop.run.request|complete|failed` (category `tools`); achievement key `isslop` ("Slop Hunter"). New unpinned dep `playwright-stealth`; runtime dirs `config.ISSLOP_DIR`/`ISSLOP_WORKSPACES_DIR`/`ISSLOP_RUNS_DIR` in `DATA_PATHS`. + ## Backup service (`services/backup/`, `services/jobs/backup_worker.py`, `routers/admin/backups.py`) Admin-only, enterprise-grade backups built on the **same async-job pattern as zip/fork**, so nothing runs on the request path. `BackupService(JobService)` (kind `backup`) is registered in `main.py` and overrides `run_once` to call `super().run_once()` (reap/refill/sweep) and then `_fire_due_schedules()`. @@ -304,8 +331,8 @@ Admin-only. Supervises container instances, all running one shared prebuilt imag - **Data model** (`store.py`, indexed in `init_db`): `instances`, `instance_events` (audit), `instance_metrics` (ring buffer, sweep keeps `METRICS_RING`), `instance_schedules`. - **Reconciler** (`service.py`, `BaseService`): the model is desired-vs-actual, NOT a task per container. Each tick: `backend.ps(label=devplace.instance)` -> converge each instance to `desired_state`, apply restart policy, reap orphan containers (labeled but no DB row), fire due `instance_schedules` (reuse `devii/tasks/schedule.py` `cron_next`/`next_run`), sample `docker stats`. Only the lock owner reconciles; HTTP routes only flip `desired_state`/write events. `docker run --name ` collision is the double-launch guard. - **Workspace** = `/app`: `project_files.export_to_dir` materializes the project to `config.CONTAINER_WORKSPACES_DIR/` once, bind-mounted RW; `project_files.import_from_dir` (the inverse) syncs it back. Both run via `asyncio.to_thread`. **All runtime data lives OUTSIDE the `devplacepy/` package and NOT under `/static`**: workspaces + zips in `config.DATA_DIR` (`data/`, configurable via `DEVPLACE_DATA_DIR`). Never put generated data under `STATIC_DIR` - it is served publicly AND watched by dev `--reload` (scoped to `--reload-dir devplacepy`). The docker daemon must be able to bind-mount `DATA_DIR` for `/app`. The mount point is the single constant `backend/base.py` `WORKSPACE_MOUNT` (`/app`); the `ppy` image sets `WORKDIR /app` and **every exec passes `-w /app` explicitly** (`DockerCliBackend.exec` and the PTY exec in `routers/projects/containers/instances.py`), so one-shot `container_exec`, the interactive shell, and tmux sessions all start in the project workspace. The agent is told this (system prompt + `container_exec` tool summary) so it never prefixes a command with `cd /app`. -- **Pravda image (load-bearing, workspace ownership).** The hotpatch that used to run per build is now baked into `ppy.Dockerfile` once. The final stage `COPY`s the **sudo superclone** (`files/sudo`) over `/usr/local/bin/sudo` (+ symlink `/usr/bin/sudo`; the real `sudo` package is not installed) and **`pagent`** (`files/pagent`, the stdlib AI agent; reads `PRAVDA_OPENAI_URL`+`PRAVDA_API_KEY`, falling back to its public endpoint + `DEEPSEEK_API_KEY`) to `/usr/bin/pagent.py` (and `files/.vimrc` to `/home/pravda/.vimrc`, whose `AiEditSelection` helper hits the same gateway as pagent via `PRAVDA_OPENAI_URL`/`PRAVDA_API_KEY` with a public fallback, not `api.openai.com`), evicts any pre-existing uid-1000 user, creates user **`pravda` at `1000:1000`**, hands pravda **ownership of the toolchain and the OS package trees** (`chown -R pravda` over `/usr/local/lib`, `/usr/local/bin`, `/usr/lib/python3`, `/opt`, `/app`, `/home/pravda`, plus `/usr/lib`, `/usr/bin`, `/usr/sbin`, `/usr/share`, `/usr/include`, `/etc`, `/var/lib`, `/var/cache`, `/var/log`, `/srv`; `~/.local/bin` on `PATH`), and ends on `USER pravda`. **Why uid 1000:** `/app` is bind-mounted from the host, and under DooD a container's UID maps 1:1 to the host. A container running as **root** wrote root-owned files into the workspace; the app process (host `retoor`, uid 1000) then hit `[Errno 13] Permission denied` in `export_to_dir` on the next instance create - a permanent per-project brick. Pinning the container UID to `1000` makes every write land as `retoor`, and pravda owning the global site-packages means runtime `pip install` just succeeds as pravda with no elevation. **The sudo superclone** (`files/sudo`, POSIX `sh`) is sudo-CLI-compatible but **never swaps user**: it parses the full flag interface (`-u/-g/-E/-H/-n/-S/-i/-s/--`, leading `VAR=value` env assignments, `-V/-l/-v/-k/-K` short-circuits) and `exec`s the command **as the current user (pravda)**, exporting `SUDO_USER`/`SUDO_UID`/`SUDO_GID`/`SUDO_COMMAND`. So even a blind `sudo apt ...` / `sudo -u root ...` runs as uid 1000 and **cannot create a root-owned file** - the residual is gone by construction. **Rootless apt (`files/aptroot`):** pravda installs system packages directly (`apt install `, no sudo). `aptroot` is symlinked over `apt`/`apt-get`/`dpkg` in `/usr/local/bin` (ahead of `/usr/bin`) and execs the real tool under **`fakeroot`** with `APT::Sandbox::User=root`; dpkg's chown-to-root is virtualized while files actually land owned by pravda (uid 1000), so installs succeed with no real euid 0 and still cannot brick the bind mount. Pravda owning the system trees makes the writes themselves succeed. Run `apt update` first (the image clears `/var/lib/apt/lists`). **Trade-off (intentional):** the only genuinely-root op left that does NOT escalate is binding a port < 1024 - use a high port + `/p/` ingress. Packages whose maintainer scripts need real privileged syscalls still belong in `ppy.Dockerfile` (its `RUN apt-get` runs as root before `USER pravda`) then `make ppy`. Enforcement lives in the Dockerfile (no `--user` on `docker run`). The `export_to_dir` unlink-before-write fix remains as belt-and-suspenders (the app owns the workspace dir, so it may delete any stale file in it regardless of owner before rewriting it). -- **`PRAVDA_*` runtime env injection.** `api.run_spec_for` merges `api.pravda_env(instance)` over the instance's own `env_json` (PRAVDA keys win) so every running container gets six platform vars: `PRAVDA_BASE_URL` (the `site_url` setting via `seo.public_base_url()`), `PRAVDA_OPENAI_URL` (`{base}/openai/v1`, the OpenAI-compatible gateway base), `PRAVDA_API_KEY` (the **creating** user's `api_key`, resolved live), `PRAVDA_USER_UID` (the project **owner**'s uid), `PRAVDA_CONTAINER_NAME` (instance name), `PRAVDA_CONTAINER_UID` (instance uid), `PRAVDA_INGRESS_URL` (the instance's absolute public ingress URL `{base}/p/{ingress_slug}` when published and `site_url` is set, relative `/p/{slug}` if not, empty if the instance has no `ingress_slug`). They let code inside a container call back into the platform and the AI gateway authenticated as the user. **Injected at run (docker run `-e`), not baked into the image:** the image is shared by every instance, but name/uid/api-key are per-instance and base-url/key must stay fresh, so they are resolved each launch and never stored as secrets in the DB. Resolution needs two new instance columns set at `create_instance`: `created_by` (= `actor[1]` when the actor is a user) feeds `PRAVDA_API_KEY`, and `owner_uid` (= `project["user_uid"]`) feeds `PRAVDA_USER_UID`; instances created before this change have neither and degrade to an empty key/uid (base-url and container name/uid still resolve) until recreated. `PRAVDA_BASE_URL`/`PRAVDA_OPENAI_URL` are empty when `site_url` is unset, so set the admin `site_url` for container-to-platform calls to work (same requirement as ingress URLs). +- **Pravda image (load-bearing, workspace ownership).** The hotpatch that used to run per build is now baked into `ppy.Dockerfile` once. The final stage `COPY`s the **sudo superclone** (`files/sudo`) over `/usr/local/bin/sudo` (+ symlink `/usr/bin/sudo`; the real `sudo` package is not installed) and **`pagent`** (`files/pagent`, the stdlib AI agent; reads `DEVPLACE_OPENAI_URL`+`DEVPLACE_API_KEY`, falling back to its public endpoint + `DEEPSEEK_API_KEY`) to `/usr/bin/pagent.py` (and `files/.vimrc` to `/home/pravda/.vimrc`, whose `AiEditSelection` helper hits the same gateway as pagent via `DEVPLACE_OPENAI_URL`/`DEVPLACE_API_KEY` with a public fallback, not `api.openai.com`), evicts any pre-existing uid-1000 user, creates user **`pravda` at `1000:1000`**, hands pravda **ownership of the toolchain and the OS package trees** (`chown -R pravda` over `/usr/local/lib`, `/usr/local/bin`, `/usr/lib/python3`, `/opt`, `/app`, `/home/pravda`, plus `/usr/lib`, `/usr/bin`, `/usr/sbin`, `/usr/share`, `/usr/include`, `/etc`, `/var/lib`, `/var/cache`, `/var/log`, `/srv`; `~/.local/bin` on `PATH`), and ends on `USER pravda`. **Why uid 1000:** `/app` is bind-mounted from the host, and under DooD a container's UID maps 1:1 to the host. A container running as **root** wrote root-owned files into the workspace; the app process (host `retoor`, uid 1000) then hit `[Errno 13] Permission denied` in `export_to_dir` on the next instance create - a permanent per-project brick. Pinning the container UID to `1000` makes every write land as `retoor`, and pravda owning the global site-packages means runtime `pip install` just succeeds as pravda with no elevation. **The sudo superclone** (`files/sudo`, POSIX `sh`) is sudo-CLI-compatible but **never swaps user**: it parses the full flag interface (`-u/-g/-E/-H/-n/-S/-i/-s/--`, leading `VAR=value` env assignments, `-V/-l/-v/-k/-K` short-circuits) and `exec`s the command **as the current user (pravda)**, exporting `SUDO_USER`/`SUDO_UID`/`SUDO_GID`/`SUDO_COMMAND`. So even a blind `sudo apt ...` / `sudo -u root ...` runs as uid 1000 and **cannot create a root-owned file** - the residual is gone by construction. **Rootless apt (`files/aptroot`):** pravda installs system packages directly (`apt install `, no sudo). `aptroot` is symlinked over `apt`/`apt-get`/`dpkg` in `/usr/local/bin` (ahead of `/usr/bin`) and execs the real tool under **`fakeroot`** with `APT::Sandbox::User=root`; dpkg's chown-to-root is virtualized while files actually land owned by pravda (uid 1000), so installs succeed with no real euid 0 and still cannot brick the bind mount. Pravda owning the system trees makes the writes themselves succeed. Run `apt update` first (the image clears `/var/lib/apt/lists`). **Trade-off (intentional):** the only genuinely-root op left that does NOT escalate is binding a port < 1024 - use a high port + `/p/` ingress. Packages whose maintainer scripts need real privileged syscalls still belong in `ppy.Dockerfile` (its `RUN apt-get` runs as root before `USER pravda`) then `make ppy`. Enforcement lives in the Dockerfile (no `--user` on `docker run`). The `export_to_dir` unlink-before-write fix remains as belt-and-suspenders (the app owns the workspace dir, so it may delete any stale file in it regardless of owner before rewriting it). +- **`PRAVDA_*` runtime env injection.** `api.run_spec_for` merges `api.pravda_env(instance)` over the instance's own `env_json` (PRAVDA keys win) so every running container gets six platform vars: `DEVPLACE_BASE_URL` (the `site_url` setting via `seo.public_base_url()`), `DEVPLACE_OPENAI_URL` (`{base}/openai/v1`, the OpenAI-compatible gateway base), `DEVPLACE_API_KEY` (the **creating** user's `api_key`, resolved live), `DEVPLACE_USER_UID` (the project **owner**'s uid), `DEVPLACE_CONTAINER_NAME` (instance name), `DEVPLACE_CONTAINER_UID` (instance uid), `DEVPLACE_INGRESS_URL` (the instance's absolute public ingress URL `{base}/p/{ingress_slug}` when published and `site_url` is set, relative `/p/{slug}` if not, empty if the instance has no `ingress_slug`). They let code inside a container call back into the platform and the AI gateway authenticated as the user. **Injected at run (docker run `-e`), not baked into the image:** the image is shared by every instance, but name/uid/api-key are per-instance and base-url/key must stay fresh, so they are resolved each launch and never stored as secrets in the DB. Resolution needs two new instance columns set at `create_instance`: `created_by` (= `actor[1]` when the actor is a user) feeds `DEVPLACE_API_KEY`, and `owner_uid` (= `project["user_uid"]`) feeds `DEVPLACE_USER_UID`; instances created before this change have neither and degrade to an empty key/uid (base-url and container name/uid still resolve) until recreated. `DEVPLACE_BASE_URL`/`DEVPLACE_OPENAI_URL` are empty when `site_url` is unset, so set the admin `site_url` for container-to-platform calls to work (same requirement as ingress URLs). - **Shared ops** in `api.py` back BOTH `routers/projects/containers/instances.py` and the Devii `ContainerController` (`services/devii/container/`, `handler="container"`, 7 instance tools). The reconciler service defaults disabled (needs docker). - **UI is two surfaces over one API set, both discoverable** (the old `templates/containers.html` page had zero links to it - that issue is fixed). (1) Per-project manager: `templates/containers.html` + `static/js/ContainerManager.js` handles instance creation through an app **modal form** (the `_macros.html` `modal()` macro + `ModalManager` `.visible` toggle); its instance list links out to the shared detail page. Reached from the project detail page's admin-only **Containers** button (`is_admin(user)` gate) and now passes breadcrumbs so content clears the fixed nav. (2) Admin **Containers** section: `routers/admin/containers.py` (mounted `/admin/containers`, sidebar link in `admin_base.html`, `admin_section="containers"`) - `GET /admin/containers` lists every instance via `store.all_instances()` (decorated with project title/slug from one `projects` lookup) in an `.admin-table`; `GET /admin/containers/data` is the poll JSON; `GET /admin/containers/{uid}` renders `templates/containers_instance.html` + `static/js/ContainerInstance.js`, a dedicated detail page (lifecycle, poll logs/metrics, schedules add/delete, ingress, sync, interactive exec over a PTY WebSocket gated on the lock owner). **Key reuse rule:** `containers_admin.py` adds NO lifecycle/logs/exec endpoints - the instance carries its `project_uid`, so the detail route resolves the project and `ContainerInstance.js` targets the existing `/projects/{slug}/containers/instances/{uid}/...` routes. Add new instance operations to `routers/projects/containers/instances.py` only; the admin detail page picks them up for free. All container CSS (`static/css/containers.css`) uses app design tokens (`--bg-card`, `--text-primary`, `--success`/`--danger`/`--warning`, `--radius`) and the shared `.card` recipe. - **Interactive shell = floating xterm.js window.** An instance's shell is NOT inline; it is a floating `` (`static/js/components/ContainerTerminal.js`) over the existing exec WS (`/projects/{slug}/containers/instances/{uid}/exec/ws`, `pty.openpty()` + `docker exec -it`, admin + `service_manager.owns_lock()` gated). xterm + fit addon are vendored under `static/vendor/xterm/` and lazy-loaded. It extends the generic **`FloatingWindow`** base (`static/js/components/FloatingWindow.js` + `static/css/floating-window.css`) which owns the Devii-style chrome (drag, native `resize:both`, maximize/fullscreen, geometry persistence). **`app.windows`** (`WindowManager`, `components/WindowManager.js`) assigns z-index on `pointerdown` so the last-clicked window is on top. **The Devii terminal (`devii/devii-terminal.js`) now also `extends FloatingWindow`** and reuses the base drag/geometry/preset/resize/persist plus `_window`/`_registerWindow` (z-order + context-menu) machinery; it overrides only the parts that genuinely differ - `_setState` (Devii adds a `closed` state + launcher FAB), `_defaultGeometry` (bottom-right 760x600), `_changeFont` (per-user `--devii-font-size` CSS var), `_contextItems`, and `_persistGeometry` (delegates to its richer `{state,geometry,focused}` blob under `devii-terminal-state`). Devii keeps native `resize:both` (no `.fw-resize` grip) and **`devii.css` is unchanged**: the only base change required was a `get _dragClass()` getter (default `fw-dragging`) that Devii overrides to `devii-dragging`, so the base drag handlers toggle Devii's own class and its existing CSS drives everything. Open it two ways: the instance-page **Open terminal** button (`app.containerTerminals.open(slug, uid, name)`, `ContainerTerminalManager`) or Devii "attach <container>" -> the admin-only `open_terminal` **client** action (`client_actions.py`, `requires_admin=True`) -> `DeviiClient._openTerminal`. **Resize protocol:** the exec WS treats a brace-leading JSON frame `{"type":"resize","cols","rows"}` as a control message - it `TIOCSWINSZ`-es the host pty (`fcntl.ioctl`) **and** `proc.send_signal(SIGWINCH)` to the `docker exec -it` client so the resize forwards into the container tty (the SIGWINCH is required: with `start_new_session=True` the host slave is not the client's controlling terminal, so the kernel does not auto-deliver it; the pair shares its winsize, so the client reads the new size off its slave stdio and calls Docker's exec-resize API). Everything else is raw stdin, so keystrokes are untouched. Client-side the **fit addon** drives it and the window has a dedicated `.fw-resize` grip (xterm covers the native CSS resize corner, so the base `FloatingWindow` adds a manual bottom-right grip instead of relying on `resize:both`). **Persistence (tmux):** with `?session=` (validated `^[A-Za-z0-9_-]{1,64}$`) the WS runs `tmux new-session -A -s ` (falls back to bash if tmux missing), so the shell/server survives WS death - `proc.kill` on disconnect kills the tmux *client*, the server daemon + session live on (proven: a detached `devplace` session keeps its process running across exec clients). The frontend keeps exactly ONE persistent terminal (the last opened, `ContainerTerminalManager`): it attaches to session `devplace`, **auto-reconnects** with capped backoff on unexpected WS close (not on code 1008), and is remembered per user in `localStorage` (`ct-persistent:`); `app.containerTerminals.restore()` (one call in `Application.js` after `window.app`) reopens it on the next page load. Opening another terminal calls `setPersistent(false)` on the previous one (its tmux session lingers in the container, but the frontend stops auto-reconnecting/remembering it); explicit window-close (`_onClose`) detaches + forgets (session still survives, reopen re-attaches). **Context menu:** every window wires `app.contextMenu.attach(this.win, () => this._contextItems())` (right-click + long-press). The base `FloatingWindow._contextItems` is Minimize/Normalize/Close; `ContainerTerminal` overrides it with Copy / Paste / Sync files / Restart / Terminate (`dp-dialog` confirm, then `delete` + close) / Minimize / Normalize / Project / Files / Close - the lifecycle items POST to the existing `instances/{uid}/{sync,restart,delete}` routes and Project/Files `window.location.assign` to `/projects/{slug}` and `/projects/{slug}/files`. Devii's `_contextItems` override returns Copy / Paste / Minimize/Normalize/Close (no container/project actions, since it is bound to neither). **Copy/Paste:** both use the async Clipboard API with a hidden-`textarea` + `execCommand("copy")` write fallback. Container Copy reads `term.getSelection()` (the item is `disabled` when `!term.hasSelection()`) and Paste sends `navigator.clipboard.readText()` straight down the exec WS as stdin. Devii Copy takes the live `window.getSelection()` or, when empty, the current input line; Paste inserts clipboard text into the `.devii-input` at the caret (`selectionStart/End` splice) and refocuses. **Minimize/Normalize** are `_presetGeometry(w,h)` size presets (smallest-usable / comfortable), available as both titlebar buttons (`data-win="minimize|normalize"`) and menu items. xterm renders ANSI natively (no `cleanTerm` stripping for the interactive shell; the one-shot exec box still strips, since `docker exec` without `-t` is non-TTY). @@ -318,7 +345,7 @@ Admin-only. Supervises container instances, all running one shared prebuilt imag - **Run-as user, boot source, start_on_boot columns.** `run_as_uid` (identity + `api_key` only; the container OS user stays `pravda` uid 1000 for the bind mount - `api.pravda_env` resolves it ahead of `created_by`/`owner_uid`), `boot_language`/`boot_script` (precedence `boot_script` by language > `boot_command` > image CMD in `api.run_spec_for`; `api.materialize_boot_script` writes `.devplace_boot.{py,sh}` into the workspace before launch; `api.validate_boot` enforces the language set + 100k cap), and `start_on_boot` (per-container; `service._boot_pass` forces `desired_state=running` for flagged instances once on first `run_once`). All four are ensured in `init_db`'s `instances` block and defaulted in `store.create_instance`. - **Status-change logging choke point.** `service._set_status(inst, changes, reason=)` diffs old vs new `status`, and on change writes an `instance_events` `status_change` row plus an audit `container.instance.status` event (`record_system`, `old_value`/`new_value`). Every reconciler status mutation (`_reconcile`, `_launch`, `_handle_exit` terminal + policy_restart) flows through it, so previously-silent transitions are now logged and visible on the detail page's Status history. - **Testing without Docker:** `FakeBackend` (its `image_exists` returns True) + `runtime.set_backend`, monkeypatch `config.CONTAINER_WORKSPACES_DIR` to tmp. See `tests/unit/services/containers.py` and `tests/api/containers.py` (argv, instance creation on `config.CONTAINER_IMAGE`, image-not-built guard, reconcile matrices, schedule firing, ingress validation + live HTTP proxy, HTTP admin gate). -- **Vibe coding on-ramp (user-facing doc).** The container runtime is also the basis of "vibe coding": the public prose page `templates/docs/getting-started-vibing.html` (slug `getting-started-vibing`, `SECTION_GENERAL`, not admin-gated so the docs stay public while the feature is admin-only/Alpha) is the canonical user-facing guide. It teaches the Devii-driven flow project -> container -> start -> terminal (`create_project`, `container_create_instance`, `container_instance_action`, the `open_terminal` client action), documents the three baked-in agents (`dpc` = DevPlace Code at `/usr/bin/dpc`, the Claude-Code-class coding agent; `botje.py` = the copy of `services/containers/files/bot.py` at `/usr/bin/botje.py`; `pagent`), all metered through the container's own `PRAVDA_API_KEY`, the full `PRAVDA_*` env table (see `api.pravda_env`), and ingress at `/p/` via `ingress_slug`/`ingress_port`. When the runtime, the agent binaries, or the `PRAVDA_*`/ingress contract change, update this page alongside the source. **The agents are gateway-only:** `dpc`/`d.py` and `botje.py`/`bot.py` use a single `molodetz` backend pointed at `PRAVDA_OPENAI_URL` (the gateway); the former direct `api.deepseek.com` fallback backend was removed so every in-container AI call is ledgered under the run-as user and nothing bypasses `gateway_usage_ledger`. `pagent`/`.vimrc` already posted to the gateway URL (using `DEEPSEEK_API_KEY` only as a key fallback, never the DeepSeek endpoint). Rebuild the image (`make ppy`) for the change to reach running containers. +- **Vibe coding on-ramp (user-facing doc).** The container runtime is also the basis of "vibe coding": the public prose page `templates/docs/getting-started-vibing.html` (slug `getting-started-vibing`, `SECTION_GENERAL`, not admin-gated so the docs stay public while the feature is admin-only/Alpha) is the canonical user-facing guide. It teaches the Devii-driven flow project -> container -> start -> terminal (`create_project`, `container_create_instance`, `container_instance_action`, the `open_terminal` client action), documents the three baked-in agents (`dpc` = DevPlace Code at `/usr/bin/dpc`, the Claude-Code-class coding agent; `botje.py` = the copy of `services/containers/files/bot.py` at `/usr/bin/botje.py`; `pagent`), all metered through the container's own `DEVPLACE_API_KEY`, the full `PRAVDA_*` env table (see `api.pravda_env`), and ingress at `/p/` via `ingress_slug`/`ingress_port`. When the runtime, the agent binaries, or the `PRAVDA_*`/ingress contract change, update this page alongside the source. **The agents are gateway-only:** `dpc`/`d.py` and `botje.py`/`bot.py` use a single `molodetz` backend pointed at `DEVPLACE_OPENAI_URL` (the gateway); the former direct `api.deepseek.com` fallback backend was removed so every in-container AI call is ledgered under the run-as user and nothing bypasses `gateway_usage_ledger`. `pagent`/`.vimrc` already posted to the gateway URL (using `DEEPSEEK_API_KEY` only as a key fallback, never the DeepSeek endpoint). Rebuild the image (`make ppy`) for the change to reach running containers. ## Admin seniority guard (`routers/admin/users.py`) @@ -729,6 +756,8 @@ Do this with `get_table(name)` (NOT `if name in db.tables`): `get_table` + `crea - **For "live, newest-first" listings add a composite or live-partial index that includes the sort column.** A `WHERE deleted_at IS NULL ORDER BY created_at` query needs the ordering in the index or it filesorts. Posts use a partial `idx_posts_live_created ON (created_at) WHERE deleted_at IS NULL` (feed) plus `idx_posts_user_created (user_uid, created_at)` (profile); comments use `idx_comments_target_created (target_type, target_uid, created_at)`; votes use `idx_votes_user_target (user_uid, target_uid)` (the per-user "my_vote" check on every card); notifications/gists/projects use `(user_uid, created_at)`. All were verified to drop the `USE TEMP B-TREE FOR ORDER BY` step. +- **Index the non-`uid` lookup keys too, not just the sort/owner columns.** A demand-vs-supply audit added the last missing single-key lookups: the `resolve_by_slug` hot path filters `slug` on content detail pages, so posts/gists/news/projects each get `idx__slug (slug)`; `get_setting`/`set_setting` filter `key`, so `idx_site_settings_key (key)`; the container store's `find_one(slug=)`/`find_one(name=)` fallbacks get `idx_instances_slug`/`idx_instances_name`. The DM thread load `find(sender_uid=, receiver_uid=)` gets the covering composites `idx_messages_conversation (sender_uid, receiver_uid)` + `idx_messages_conversation_rev (receiver_uid, sender_uid)` (the read-flag `UPDATE` uses the reverse); the badge-has check gets `idx_badges_user_name (user_uid, badge_name)`; the admin user list `ORDER BY -created_at` gets `idx_users_created_at (created_at)` (the existing `(role, created_at)` cannot serve a full-table created_at sort). All are non-unique so `_index` always creates them even if legacy duplicate data exists. Column sets already resolved to ~1 row by an existing prefix index (votes `+target_type`, game_quests `+kind`, poll_options `position`) are intentionally left uncovered - a trailing column there only adds write cost. + `init_db()` ends with `_refresh_query_planner_stats()`: a one-time `ANALYZE` when `sqlite_stat1` is absent, then `PRAGMA optimize` every boot (both inside `with db:`). Validate any index change with `EXPLAIN QUERY PLAN` against `data/devplace.db` (or a copy) - a correct plan reads `SEARCH ... USING INDEX ` with no temp b-tree. ## HTML/JSON content negotiation @@ -869,10 +898,25 @@ credentialed URL `http://username:password@host/xmlrpc`. Public endpoints need n - **Post deletion must cascade:** delete comments and votes first, then the post. Always check ownership: `post["user_uid"] == user["uid"]`. - **Message deduplication needed** when `sender_uid == receiver_uid` (messaging yourself): `seen = set()` of message UIDs before appending to result list. +## Performance caches (request-path) + +Read-mostly aggregates that were recomputed per request or per vote sit behind short per-worker `TTLCache`s (`devplacepy/cache.py`). These are DISPLAY caches: staleness up to the TTL is accepted by design, so none of them uses the cross-worker `cache_state` versioning (that is reserved for correctness-critical caches like auth/settings/admins). The registry: + +| Cache | Where | Key | TTL | Invalidation | +|---|---|---|---|---| +| `_authors_cache` | `database/ranking.py` | `ranked`/`rank_map` | 15s | TTL only - `update_target_stars` deliberately does NOT clear it per vote anymore (the old per-vote `clear()` forced a full UNION-JOIN ranking recompute on the next feed/leaderboard/landing request and never propagated cross-worker anyway) | +| `_stars_cache` | `database/ranking.py` | user uid | 15s | TTL + `clear_user_stars(owner_uid)` from `content.apply_vote`, so a user's own total updates immediately on the voting worker | +| `_leaderboard_cache` | `services/game/store/farm.py` | `top:{limit}` | 15s | TTL only (the farm scan + Python `farm_score` sort runs at most once per 15s per worker) | +| `_projects_cache` | `templating.py` | user uid | 10s | TTL + `clear_user_projects_cache(uid)` from the `content.py` project create/delete choke points (in-function import - `templating` imports `content` at module level). `jinja_user_projects` also filters `deleted_at=None` now (the composer dropdown previously listed soft-deleted projects) | + +Rules: a new hot read-path aggregate follows this exact pattern (module-level `TTLCache`, 10-15s, `clear_*` helper only when a same-worker write must be visible immediately); never reintroduce a whole-cache `clear()` on a per-event write path; profile `projects`/`gists` loads are tab-gated in `routers/profile/index.py` (`tab == X or wants_json(request)` - the JSON `ProfileOut` keeps serializing both, HTML tabs that do not render them get `[]`). + ## Real-time messaging (`devplacepy/services/messaging/`) The `/messages` feature is a live WebSocket chat layered over the SAME `messages` table and audit/notification cores as before; there is no parallel chat store. The WS only adds live delivery, presence, typing, and read receipts on top of the existing persistence. +- **Bounded page queries (load-bearing performance rule).** `get_conversations` is ONE window-function query (`ROW_NUMBER() OVER (PARTITION BY other_uid ORDER BY created_at DESC, id DESC)` over `sender_uid = :me OR receiver_uid = :me`, the `get_recent_comments_by_target_uids` precedent) that returns only each partner's latest row - never load the user's full message history into Python to build the sidebar. `get_conversation_messages` loads at most `CONVERSATION_MESSAGE_LIMIT` (500) most-recent rows (`ORDER BY created_at DESC, id DESC LIMIT`, reversed to ascending in Python); older history stays in the DB and is simply not rendered. Both are covered by `idx_messages_conversation`/`_rev` (verified `EXPLAIN QUERY PLAN`, MULTI-INDEX OR, no table scan). `messages` is NOT in `SOFT_DELETE_TABLES`, so neither query filters `deleted_at`. Any new message listing must stay bounded and indexed the same way. + - **WS endpoint `/messages/ws`** lives on the existing messages router (no new router/prefix; already mounted at `/messages`). It is **NOT lock-owner gated** (deliberately - unlike `/devii/ws`): `await websocket.accept()`, resolve the owner via session cookie / api-key (`_resolve_ws_user` reuses `utils._user_from_session`/`_user_from_api_key`), and accept on EVERY worker. Gating on `service_manager.owns_lock()` was the original design and was wrong here: when no worker holds the service lock (services disabled, or the lock not yet acquired) every socket got closed `4013` and the client fast-retried every 200ms forever (a connection storm with no stream, and the send form fell back to a full-page POST reload). Chat must not depend on the background-service lock. Messaging requires an authenticated user - guests are closed with `1008`. The receive loop is wrapped in `try/except WebSocketDisconnect` + a broad `except` (never crash the worker) and ALWAYS unregisters the socket in `finally`. - **Connection registry** is the `ConnectionManager` singleton `message_hub` (`services/messaging/hub.py`), **one per worker**: `user_uid -> set[WebSocket]`, plus an in-process `last_seen` map and a bounded `delivered` LRU set (`mark_delivered`/`was_delivered`, cap 4000) used to dedupe direct vs relayed delivery. `register`/`unregister` return whether the user just came online / went offline (for presence announces). `send_to_user`/`send_to_users` fan a frame out to all of a user's sockets (multi-tab) and swallow dead-socket errors. Presence and last-seen are in-process per worker (no DB column). - **Cross-worker delivery (the relay).** Because the WS accepts on every worker, a message persisted on worker A must still reach a recipient whose socket lives on worker B. `services/messaging/relay.py` `message_relay` (a per-worker singleton asyncio loop, started on the first socket connect, self-stops when `message_hub.has_connections()` is false) polls `SELECT * FROM messages WHERE id > :watermark` (uuid7-backed autoincrement `id`) every ~1s and pushes any row whose sender/receiver has a LOCAL socket and that was not already `was_delivered` to those local sockets, advancing the watermark to the max row seen. Same-worker sends are delivered INSTANTLY by `broadcast_message` (which calls `message_hub.mark_delivered` so the relay skips them); the relay only fills the cross-worker gap, so same-worker latency is zero and cross-worker latency is bounded by the poll interval. The relay primes its watermark to the current `MAX(id)` on first start so it never replays history. **Trade-off:** typing/read/presence frames are in-process per worker only - across the two `make prod` workers those ephemeral signals reach only same-worker peers (message delivery is always correct). On a single dev worker everything is instant and complete. @@ -981,7 +1025,7 @@ Every directory is a package (`__init__.py`). `pyproject.toml` has `testpaths = ### General -- **932 tests across `tests/{unit,api,e2e}/`.** Playwright integration + HTTP + unit tests. All must pass before any merge. +- **Around 1959 tests across `tests/{unit,api,e2e}/`.** Playwright integration + HTTP + unit tests. All must pass before any merge. - **Tests use `-x` (fail-fast).** The suite stops at the first failure. Fix that test, then re-run. - **NEVER run tests unless specifically asked by user.** Not the full suite, not a single file - do not run any tests unless the user explicitly requests it. - **Validate each touched language manually:** Python (compile + import), JS (parse / bracket matching), CSS (brace matching), HTML (tag matching). Zero tolerance. @@ -1523,7 +1567,7 @@ Singleton that manages all registered services: - `supervise()` - start the reconciling task per service (lock worker only) - `shutdown_all()` - cancel tasks on server shutdown -### `NewsService` (`services/news.py`) +### `NewsService` (`services/news/service.py`) A fully automatic, zero-maintenance import pipeline implementing `BaseService` (`default_enabled = True`, registered unconditionally in `main.py`). The per-run flow per article is: clean text -> fetch and perceptually compare images -> AI-grade the cleaned text -> reliability gate -> compute an effective score -> AI-reformat the body into Markdown (valid articles only) -> publish/Feature decision -> store -> then a single post-loop Landing rotation. - Fetches `GET {news_api_url}` โ†’ `{"articles": [...]}`; already-synced `external_id`s (from `news_sync`) are skipped. - Every article is graded via AI on the CLEANED text: `POST {news_ai_url}` with model `{news_ai_model}`, temperature 0. Both default to the internal gateway (`INTERNAL_GATEWAY_URL` / `molodetz`); the key falls back to `internal_gateway_key()` when `news_ai_key`/`NEWS_AI_KEY` is unset. @@ -2136,9 +2180,9 @@ devplace devii reset-quota --guests # Reset every guest quota devplace devii reset-quota --all # Reset every quota (users and guests) ``` -## Signals Category +## Politics Category -The `signals` topic is available as a feed filter sidebar item and post topic. Added CSS variable `--topic-signals: #00bcd4` in `variables.css`, badge class in `base.css`, and dot color in `feed.css`. Allowed in `posts.py` topic validation. +The `politics` topic is available as a feed filter sidebar item (icon `🏛`) and post topic. It defines the CSS variable `--topic-politics: #00bcd4` in `variables.css` and the `.badge-politics` class in `base.css`. It is validated like every topic via the canonical `TOPICS` list in `constants.py` (consumed by `models.py` `valid_topic`, `templating.py` `TOPICS` global, `routers/posts.py`, `docs_api`). Unlike the former `signals` topic, `politics` is also part of the bot fleet's rotation - it is in `services/bot/config.py` `CATEGORIES`/`FEED_TOPICS`, carries a modest per-persona weight in `PERSONA_CATEGORY_WEIGHTS`, and has a writing instruction in `services/bot/llm.py` `category_extras`. ## Date Format diff --git a/CLAUDE.md b/CLAUDE.md index ae92bc09..06c9084c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -9,10 +9,10 @@ DevPlace is a server-rendered social network for developers. FastAPI backend ser ## Commands ```bash -make install # pip install -e . +make install # pip install -e . + playwright install chromium make ppy # build the single shared container image (ppy:latest); run once before launching instances make dev # uvicorn --reload on port 10500 -make prod # uvicorn 2 workers, port 10500 (backlog 8192) +make prod # uvicorn --workers $(WEB_WORKERS) (defaults to nproc), port 10500 (backlog 8192) make test # Playwright + unit tests, headless, serial (one at a time), -x fail-fast make test-headed # same tests in a visible Chromium window (single process) make locust # Locust load test, interactive web UI @@ -29,6 +29,14 @@ CLI (installed as `devplace`): ```bash devplace role get devplace role set +devplace apikey get # print a user's API key +devplace apikey reset # regenerate a user's API key +devplace apikey backfill # assign API keys to users that lack one +devplace token issue [--label L] # issue a DevPlace access token +devplace token list # list a user's active access tokens +devplace token revoke # revoke a single access token by uid +devplace token revoke-all # revoke all access tokens for a user +devplace token prune # soft-delete all expired access tokens devplace news clear # delete all news rows devplace news sanitize # strip HTML from news descriptions/content devplace attachments prune # remove orphan attachment records/files @@ -45,6 +53,9 @@ devplace seo-meta prune # delete expired SEO metadata job rows (generated me devplace seo-meta clear # delete every SEO metadata job row (generated metadata persists) devplace deepsearch prune # delete expired DeepSearch sessions + job rows + collections devplace deepsearch clear # delete every DeepSearch session + job row + collection +devplace isslop analyze # run a AI usage analysis from the terminal (report persists) +devplace isslop prune # delete expired AI usage analysis job rows (analyses + reports persist) +devplace isslop clear # delete every AI usage analysis, its report and job rows devplace backups list # list recorded backups devplace backups run # enqueue a backup (processed by the running server) devplace backups prune # remove backup records whose archive file is missing @@ -60,7 +71,7 @@ devplace migrate-data # relocate legacy runtime files into data/ (idempote ### Runtime data layout (single source of truth) -Every runtime/user-generated artifact lives under one root, `config.DATA_DIR` (`DEVPLACE_DATA_DIR`, default `/data`). `config.py` derives every runtime path from it and lists them in the `DATA_PATHS` registry; `ensure_data_dirs()` (called at the top of `startup()` and from `database.init_db`, plus a guard at `database.py` import before `dataset.connect`) creates the whole tree before anything is written. **No module computes a runtime path from scratch** - they import the constants (`UPLOADS_DIR`, `ATTACHMENTS_DIR`, `PROJECT_FILES_DIR`, `ZIPS_DIR`, `ZIP_STAGING_DIR`, `FORK_STAGING_DIR`, `KEYS_DIR`, `BOT_DIR`, `LOCKS_DIR`, `CONTAINER_WORKSPACES_DIR`, `DEVII_TASKS_DB`, `DEVII_LESSONS_DB`, `VAPID_*_FILE`, `SERVICE_LOCK_FILE`, `INIT_LOCK_FILE`). Uploads moved physically from `devplacepy/static/uploads/` to `data/uploads/` but are still **served at the unchanged `/static/uploads/` URL** (FastAPI `UploadStaticFiles` mount -> `config.UPLOADS_DIR`; nginx `alias /data/uploads/` from a mounted volume). Stored DB values for attachments/project files are relative (`directory` + `stored_name`), so **no DB rewrite is needed**. `devplace migrate-data` relocates an older scattered install (copy -> verify CRC -> atomic replace -> unlink; idempotent). +Every runtime/user-generated artifact lives under one root, `config.DATA_DIR` (`DEVPLACE_DATA_DIR`, default `/data`). `config.py` derives every runtime path from it and lists them in the `DATA_PATHS` registry; `ensure_data_dirs()` (called at the top of `startup()` and from `database.init_db`, plus a guard at `database/` import before `dataset.connect`) creates the whole tree before anything is written. **No module computes a runtime path from scratch** - they import the constants (`UPLOADS_DIR`, `ATTACHMENTS_DIR`, `PROJECT_FILES_DIR`, `ZIPS_DIR`, `ZIP_STAGING_DIR`, `FORK_STAGING_DIR`, `KEYS_DIR`, `BOT_DIR`, `LOCKS_DIR`, `CONTAINER_WORKSPACES_DIR`, `DEVII_TASKS_DB`, `DEVII_LESSONS_DB`, `VAPID_*_FILE`, `SERVICE_LOCK_FILE`, `INIT_LOCK_FILE`). Uploads moved physically from `devplacepy/static/uploads/` to `data/uploads/` but are still **served at the unchanged `/static/uploads/` URL** (FastAPI `UploadStaticFiles` mount -> `config.UPLOADS_DIR`; nginx `alias /data/uploads/` from a mounted volume). Stored DB values for attachments/project files are relative (`directory` + `stored_name`), so **no DB rewrite is needed**. `devplace migrate-data` relocates an older scattered install (copy -> verify CRC -> atomic replace -> unlink; idempotent). **Blob sharding (`attachments._directory_for`, the single shard function reused by `attachments`, `project_files`, and `zip_service`):** blobs are spread into a two-level `xx/yy` directory tree to keep any one directory small. The shard is taken from the **random tail** of the uuid7 (`tail = uid.replace("-", "")`, `f"{tail[-2:]}/{tail[-4:-2]}"`), NOT its leading bytes. This is load-bearing: a uuid7's first 48 bits are a millisecond timestamp, so the leading hex (`uid[:2]`/`uid[2:4]`) advances only every ~35 years / ~50 days respectively - prefix-sharding a time-ordered id funnels every contemporaneous write into ONE bucket (the legacy `data/uploads/attachments/01/9e/` pile is exactly this defect). The last 62 bits are random, so the tail gives a uniform 256x256 fan-out from the first write. Any new shard helper MUST shard on a high-entropy field (uuid tail or a hash), never the head. The change is forward-only: every consumer persists the computed `directory`/`local_path` and reads it back, so pre-existing rows keep resolving from their stored value and only new writes use the new layout (no migration required; `migrate-data` could rebalance the old pile if ever desired). @@ -88,7 +99,7 @@ Tests run on port 10501 with a tempfile SQLite DB and a dedicated `DEVPLACE_DATA `@app.on_event("startup")` calls `init_db()` and, unless `DEVPLACE_DISABLE_SERVICES` is set, registers `NewsService` with `service_manager` and kicks off `start_all()` as an asyncio task. `@app.on_event("shutdown")` calls `service_manager.stop_all()`. -`GET /` is the home page and never redirects: guests get the marketing splash, authenticated users get a personalized home (welcome card with their avatar/stats, a feed shortcut, quick links), both sharing the Latest Posts + Developer News sections. It pulls up to 6 articles where `show_on_landing=1` from the `news` table plus 6 recent posts (joined via the batch helpers in `database.py`). The Latest Posts section and the main `/feed` enforce **author diversity** by *interleaving* authors (never dropping posts) via `interleave_by_author` / `paginate_diverse` in `database.py` - the page set is the chronological window, then reordered so no two consecutive posts share an author while each author's own posts stay in date order (a run from one author is broken up by deferring to the next available author, recursively; an all-one-author page stays as-is). `paginate_diverse` mirrors `paginate` (same `(rows, next_cursor)` contract): the interleave is a pure permutation of the page, so the cursor (oldest `created_at` in the page) is unchanged. +`GET /` is the home page and never redirects: guests get the marketing splash, authenticated users get a personalized home (welcome card with their avatar/stats, a feed shortcut, quick links), both sharing the Latest Posts + Developer News sections. It pulls up to 6 articles where `show_on_landing=1` from the `news` table plus 6 recent posts (joined via the batch helpers in `database/`). The Latest Posts section and the main `/feed` enforce **author diversity** by *interleaving* authors (never dropping posts) via `interleave_by_author` / `paginate_diverse` in `database/` - the page set is the chronological window, then reordered so no two consecutive posts share an author while each author's own posts stay in date order (a run from one author is broken up by deferring to the next available author, recursively; an all-one-author page stays as-is). `paginate_diverse` mirrors `paginate` (same `(rows, next_cursor)` contract): the interleave is a pure permutation of the page, so the cursor (oldest `created_at` in the page) is unchanged. ### Routing layout Routers in `devplacepy/routers/` are organised as a **directory tree that mirrors the endpoint (URL) path**, exactly like `tests/`. A domain with a single resource stays one flat file (`feed.py`, `posts.py`, `gists.py`, ...); a domain with several sub-resources is a **package directory** split **one file per sub-resource** (a distinct noun under the domain), never one file per individual endpoint. Each leaf module declares its own `router = APIRouter()` keeping the exact path strings, and the package `__init__.py` aggregates them with `router.include_router(...)`. Because a package exposes `.router` like a module, `main.py` mounts each domain at its prefix unchanged. **Aggregation rules:** a leaf that owns the domain's collection-root (`""`) route must be the package's base router (FastAPI rejects an empty path included under an empty prefix), so the package `__init__` imports that leaf's `router` and includes the rest onto it (see `routers/issues`, `routers/admin`, `routers/projects`); leaves carved out of a former monolith keep their relative path strings and are included with no sub-prefix, while a router folded in from a deeper mount keeps its own paths and is included with a sub-prefix (`admin/__init__` includes `services.router` with `prefix="/services"`). Module-private helpers shared across a package's leaves live in its `_shared.py`. Prefixes are wired in `main.py`: @@ -102,32 +113,41 @@ Routers in `devplacepy/routers/` are organised as a **directory tree that mirror | `/projects` | projects/ package - `index.py` (listing/detail/create/delete plus owner-only visibility toggles `POST /{slug}/private` and `POST /{slug}/readonly`, and async `POST /{slug}/fork`), `files.py`, and the `containers/` subpackage (below). `main.py` mounts the whole `/projects` tree from this one package | | `/projects/{slug}/files` | projects/files.py - per-project virtual filesystem (CRUD dirs/files, upload, inline edit, plus line-range ops: `lines` read, `replace-lines`, `insert-lines`, `delete-lines`, `append` for large text files) | | `/profile` | profile/ package - `index.py` (page/search/followers/update/api-key; the **Posts** tab shows only the 10 most recent posts, while the sidebar `posts_count` stat always uses the real `count()` so it is never capped to the rendered 10), `customization.py`, `notifications.py`, `ai_correction.py` (`POST /{username}/ai-correction`, owner-or-admin AI content correction toggle + prompt), `ai_modifier.py` (`POST /{username}/ai-modifier`, owner-or-admin AI modifier toggle + prompt), `telegram.py` (`POST /{username}/telegram`, owner-or-admin Telegram pairing-code request/unpair), `usage.py` (`_ai_quota`, re-exported from the package) | -| `/messages` | messages.py - real-time DM chat. `GET ""` (page), `GET /search`, `POST /send` (no-JS fallback), and `WS /messages/ws` (NOT lock-owner gated - accepts on every worker; closes `1008` for guests). The WS adds live bidirectional delivery, typing, read receipts, and in-process presence on top of the existing `messages` table; both the WS `send` and HTTP `POST /send` route through the shared `services/messaging/persist.py` `persist_message` choke point (identical audit/notification/mention) and then through `_finalize_and_broadcast`, which awaits any pending SYNC AI correction/modifier futures (DM content runs through both, so typing `@ai ` executes live) and broadcasts the FINAL corrected/modified content (the WS path passes `request=websocket` and awaits directly since HTTP middleware does not run for websockets; the frontend swaps the sender's optimistic bubble to the authoritative content). Same-worker delivery is instant via the per-worker `message_hub`; cross-worker delivery is filled by `services/messaging/relay.py` `message_relay` (per-worker ~1s DB poll on the `messages.id` watermark, deduped against the hub's delivered set). See `AGENTS.md` -> "Real-time messaging" | +| `/messages` | messages.py - real-time DM chat. `GET ""` (page; the conversation sidebar is one windowed SQL aggregate and an opened thread loads only the newest `CONVERSATION_MESSAGE_LIMIT` = 500 messages - keep these queries bounded), `GET /search`, `POST /send` (no-JS fallback), and `WS /messages/ws` (NOT lock-owner gated - accepts on every worker; closes `1008` for guests). The WS adds live bidirectional delivery, typing, read receipts, and in-process presence on top of the existing `messages` table; both the WS `send` and HTTP `POST /send` route through the shared `services/messaging/persist.py` `persist_message` choke point (identical audit/notification/mention) and then through `_finalize_and_broadcast`, which awaits any pending SYNC AI correction/modifier futures (DM content runs through both, so typing `@ai ` executes live) and broadcasts the FINAL corrected/modified content (the WS path passes `request=websocket` and awaits directly since HTTP middleware does not run for websockets; the frontend swaps the sender's optimistic bubble to the authoritative content). Same-worker delivery is instant via the per-worker `message_hub`; cross-worker delivery is filled by `services/messaging/relay.py` `message_relay` (per-worker ~1s DB poll on the `messages.id` watermark, deduped against the hub's delivered set). See `AGENTS.md` -> "Real-time messaging" | | `/notifications` | notifications.py | | `/votes` | votes.py | +| `/reactions` | reactions.py - emoji reaction toggle on any target: `POST /reactions/{target_type}/{target_uid}` (optimistic `ReactionBar`) | +| `/bookmarks` | bookmarks.py - bookmark/favorite toggle: `GET /bookmarks/saved` (the viewer's saved list) and `POST /bookmarks/{target_type}/{target_uid}` (toggle a bookmark) | +| `/polls` | polls.py - poll voting: `POST /polls/{poll_uid}/vote` | | `/avatar` | avatar.py | | `/follow` | follow.py | +| (none) | relations.py - per-user block/mute relations: `POST /block/{username}`, `/block/unblock/{username}`, `/mute/{username}`, `/mute/unmute/{username}` (soft-deletable `user_relations` rows) | +| `/leaderboard` | leaderboard.py - `GET /leaderboard` XP/stars leaderboard page | | `/admin` | admin/ package - one leaf per sub-resource (`index`, `users`, `aiusage`, `aiquota`, `media`, `trash`, `settings`, `notifications`, `news`, `auditlog`, `backups`) plus the folded-in `services.py` and `containers.py` (mounted with `/services` and `/containers` sub-prefixes). `main.py` mounts the whole `/admin` tree from this one package. The `backups` leaf is the admin **Backups** dashboard (`BackupService`, kind `backup`): storage usage, backup archives, and interval/cron backup schedules (CRUD + rotation). **Archive download is restricted to the primary administrator** (the earliest-created Admin, resolved by `database.get_primary_admin_uid` / `utils.is_primary_admin`): `GET /admin/backups/{uid}/download` 403s every other admin, the `download_url` field is withheld from them at every endpoint (`can_download = is_primary_admin(admin)`, surfaced as `BackupDashboardOut.can_download_backups`), and `BackupMonitor.js` renders their Download control as a disabled button tooltipped `Not available`. See AGENTS.md -> "Backup service" | | `/admin/services` | admin/services.py | | `/issues` | issues/ package - issue tracker backed by Gitea (no local issue store): `index.py` (list `?state=`/`?page=`, detail `/{number}` with comments), `create.py` (async AI-enhanced filing `/create` enqueues a `issue_create` job, status at `/jobs/{uid}`), `comment.py` (synchronous, pushes to Gitea + notifies admins), `status.py` (admin open/closed), `attachments.py` (file attachments on open issues + comments, mirrored to Gitea native assets; add/list/delete with owner-or-admin + open-state guards) | | `/gists` | gists.py | | `/news` | news.py | | `/uploads` | uploads.py | +| `/media` | media.py - profile media gallery item soft delete/restore: `POST /media/{uid}/delete` and `POST /media/{uid}/restore` (owner or admin) | | `/openai` | openai_gateway.py | | `/devii` | devii.py - WebSocket terminal (`/devii/ws`), page, `/devii/usage`, `/devii/session` | | `/zips` | zips.py - generic zip job status (`/zips/{uid}`) and archive download (`/zips/{uid}/download`); enqueued from `/projects/{slug}/zip` and `/projects/{slug}/files/zip` | | `/forks` | forks.py - fork job status (`/forks/{uid}`); enqueued from `/projects/{slug}/fork`. When done its `project_url` points at the new forked project | -| `/tools` | tools/ package - public developer tools surface. `index.py` (`/tools` landing) plus `seo.py` (**SEO Diagnostics**): `GET /tools/seo` page, `POST /tools/seo/run` (enqueue `seo` job, per-owner one-active-job cap), `GET /tools/seo/{uid}` (`SeoJobOut`), `GET /tools/seo/{uid}/report` (HTML+JSON `SeoReportOut`), `WS /tools/seo/{uid}/ws` (live progress, lock-owner gated, close `4013` retry), `GET /tools/seo/{uid}/screenshot/{n}`. Also `deepsearch.py` (**DeepSearch**): `GET /tools/deepsearch` page, `POST /tools/deepsearch/run` (enqueue `deepsearch` job, per-owner one-active-job cap; resolves the user `api_key` into the payload, guests use the internal key), `GET /tools/deepsearch/{uid}` (`DeepsearchJobOut`), `GET /tools/deepsearch/{uid}/session` (HTML+JSON `DeepsearchSessionOut`), `WS /tools/deepsearch/{uid}/ws` (live progress, lock-owner gated, `4013` retry), `WS /tools/deepsearch/{uid}/chat` (grounded RAG chat over the session collection), `POST /tools/deepsearch/{uid}/{pause|resume|cancel}`, `GET /tools/deepsearch/{uid}/export.{md,json,pdf}`. The shared owner helper is `routers/tools/_shared.py` `owner_for`. Surfaced by a collapsible **Tools** header dropdown (`base.html`, visible to all) | +| `/tools` | tools/ package - public developer tools surface. `index.py` (`/tools` landing) plus `seo.py` (**SEO Diagnostics**): `GET /tools/seo` page, `POST /tools/seo/run` (enqueue `seo` job, per-owner one-active-job cap), `GET /tools/seo/{uid}` (`SeoJobOut`), `GET /tools/seo/{uid}/report` (HTML+JSON `SeoReportOut`), `WS /tools/seo/{uid}/ws` (live progress, lock-owner gated, close `4013` retry), `GET /tools/seo/{uid}/screenshot/{n}`. Also `deepsearch.py` (**DeepSearch**): `GET /tools/deepsearch` page, `POST /tools/deepsearch/run` (enqueue `deepsearch` job, per-owner one-active-job cap; resolves the user `api_key` into the payload, guests use the internal key), `GET /tools/deepsearch/{uid}` (`DeepsearchJobOut`), `GET /tools/deepsearch/{uid}/session` (HTML+JSON `DeepsearchSessionOut`), `WS /tools/deepsearch/{uid}/ws` (live progress, lock-owner gated, `4013` retry), `WS /tools/deepsearch/{uid}/chat` (grounded RAG chat over the session collection), `POST /tools/deepsearch/{uid}/{pause|resume|cancel}`, `GET /tools/deepsearch/{uid}/export.{md,json,pdf}`. The shared owner helper is `routers/tools/_shared.py` `owner_for`. Also `isslop.py` (**AI Usage Analyzer**): `GET /tools/isslop` page, `POST /tools/isslop/run` (enqueue `isslop` job, per-owner one-active-job cap; owner = user uid or the shared `DEVII_GUEST_COOKIE` guest identity, minted when absent), `GET /tools/isslop/list` (owner history; a signed-in request first claims any guest-cookie analyses via `store.claim_guest_analyses` - a move, never a copy), `GET /tools/isslop/{uid}` (`IsslopAnalysisOut`), `GET /tools/isslop/{uid}/events` (persisted ordered event trail, `?after=SEQ` incremental poll; live frames also publish on pub/sub `public.isslop.{uid}` - the DB trail is the source of truth, pub/sub the fast path), `GET /tools/isslop/{uid}/report` (HTML+JSON `IsslopReportOut`; live `` while running, server-rendered report via `render_content` when done), `GET /tools/isslop/{uid}/report.md`, `GET /tools/isslop/{uid}/badge.svg` (embeddable SVG authenticity badge). Analyses/reports/badges are permanent public capability URLs (`IsslopService.cleanup` never deletes them; only the job row is swept). Surfaced by a collapsible **Tools** header dropdown (`base.html`, visible to all) | | `/projects/{slug}/containers` | projects/containers/ subpackage - admin per-project container manager (`instances.py` for creation/lifecycle/exec/logs/metrics/sync plus the exec websocket, `schedules.py` for cron/interval/once schedules, shared helpers in `_shared.py`). Every instance runs the shared `ppy` image. Discoverable from the project detail page (admin-only **Containers** button) and from the admin index | | `/admin/containers` | admin/containers.py - admin **Containers** manager: `/admin/containers` lists every instance across all projects with inline actions (start/stop/restart/terminal/edit/delete) plus a create modal (project search-select, run-as user search-select, boot language + source editor, restart policy, start-on-boot, env/ports/limits/ingress); `/admin/containers/{uid}` is the per-instance detail page (lifecycle, live logs/metrics, PTY terminal, schedules, ingress, sync, status history) and `/admin/containers/{uid}/edit` edits run-as user, boot language/script/command, restart policy, start-on-boot, and limits. `POST /admin/containers/create`, `/{uid}/edit`, `/{uid}/{start,stop,restart,pause,resume}`, `/{uid}/sync`, `/{uid}/delete` call `api.*` directly under `require_admin` (no docker/exec backend duplicated); `GET /admin/containers/projects/search` and `/users/search` back the create/edit search-selects. The lifecycle and detail views stay layered over the per-project `/projects/{slug}/containers/instances/{uid}/...` endpoints (the instance carries its `project_uid`) | | `/p/{slug}` | proxy.py - public ingress reverse proxy (HTTP + WebSocket) to a running container instance's published host port, opt-in per instance via `ingress_slug` | | `/xmlrpc` | xmlrpc.py - reverse-proxies XML-RPC calls to the forking XML-RPC bridge (`services/xmlrpc/`, supervised by `XmlrpcService` on loopback `config.XMLRPC_PORT`), which generates one XML-RPC method per documented REST endpoint from `docs_api.API_GROUPS` (`posts.create`, `feed.list`, ...). One struct of named params per call; auth via in-band `api_key`, `X-API-KEY`/`Bearer` header, or `http://user:pass@host/xmlrpc` Basic. Full introspection + `system.multicall`; REST errors become XML-RPC faults. Exempt from the rate limiter (enforced on the forwarded internal hop). See AGENTS.md -> "XML-RPC bridge" | | `/api` | devrant/ package - devRant-compatible REST protocol (`auth.py`, `rants.py`, `comments.py`, `notifs.py`). Translates devRant requests onto DevPlace data: rants<->posts, comments/votes onto the native engagement layer, devRant integer ids onto each table's auto-increment `id`. Token auth via `devrant_tokens`. See `### devRant compatibility API` and AGENTS.md -> "devRant compatibility API" | | `/dbapi` | dbapi/ package - **primary-administrator-only, strictly READ-ONLY** generic database API over `dataset` (never inserts/updates/deletes/restores; the write surface was removed because it bypassed every per-route admin safeguard). `tables.py` (`GET /tables`, `GET /{table}/schema`), `crud.py` (read only: `GET /{table}` + `GET /{table}/{key}/{value}`, `?include_deleted`), `query.py` (`POST /query` hard SELECT-only validated read-only run; `POST /query/async` + `GET /query/{uid}` + `WS /query/{uid}/ws` via `DbApiJobService` kind `dbquery`), `nl.py` (`POST /nl` natural-language->SQL via the AI gateway, returns validated SELECT, `execute=true` runs it read-only). **Auth = the PRIMARY administrator only** (the earliest-created Admin, `utils.is_primary_admin` / `database.get_primary_admin_uid` - the same identity that gates backup downloads) by session/api_key (`services/dbapi/policy.py`); every other administrator is refused like a member, and there is **no internal-key path** - the gateway `internal_gateway_key()` is NOT accepted (no service-to-service access). SQL validated by `services/dbapi/validate.py` (sqlglot classify + suspicious-flag + read-only `EXPLAIN` dry-run). Devii tools `db_*` are read-only (list/get/query/nl; no write tools) and flagged `requires_primary_admin=True`, so they are added to the LLM tool list ONLY for the primary administrator (every other session never sees them and Devii is unaware they exist). `services/pubsub/policy.py` resolves its own admin/internal actor and does NOT reuse `dbapi.policy.caller_for`, so the primary-admin restriction does not leak onto the pub/sub bus. See AGENTS.md -> "Database API service" | +| `/game` | game/ package - the **Code Farm** idle game (`index.py` + `farm.py`): `GET /game` (page), `GET /game/state`, `GET /game/leaderboard`, `POST /game/{plant,harvest,buy-plot,upgrade,fertilize,daily,perk,prestige,legacy,quests/claim}`, plus social `GET /game/farm/{username}` and `POST /game/farm/{username}/{water,steal}` | +| (none) | push.py - web push + PWA: `GET`/`POST /push.json` (VAPID public key / subscription register), `GET /service-worker.js`, `GET /manifest.json` | +| (none) | docs.py (docs/ package) - the documentation site (prose pages + API reference); see the paragraph below and `DOCS_PAGES` | | `/pubsub` | pubsub.py - **database-free** publish/subscribe bus. `WS /pubsub/ws` (subscribe/unsubscribe/publish frames, `foo.*` wildcards), `POST /pubsub/publish` + `GET /pubsub/topics` (admin/internal). Lock-owner-gated WS (`4013` retry) so subscribers converge on one worker; topic authz in `services/pubsub/policy.py` (`user.{uid}.*` private, `public.*` shared, admin/internal anywhere, guests opt-in). In-process `services.pubsub.publish(topic, data)` for backends; frontend `app.pubsub`. See AGENTS.md -> "Pub/Sub service" | | (none) | seo.py - `/robots.txt`, `/sitemap.xml` | -Docs (`routers/docs/pages.py` `DOCS_PAGES`, re-exported from the `routers/docs` package; handlers in `routers/docs/views.py`) entries take optional `admin: True` (hidden + 404 for non-admins, but still indexed and surfaced only to admins by `docs_search`) and `section: "..."` (a nested sidebar group rendered by `docs_base.html`). Sidebar `section`s are grouped under four ordered **audience tiers** (`AUDIENCES` in `pages.py`: `Start here`, `Build with the API`, `Contribute and internals`, `Operate`); `nav_groups(visible_pages)` builds the `[(audience, [(section, [pages])])]` tree and `views.py` passes it as `nav`, so `docs_base.html` renders an audience super-header (`.sidebar-tier`) above each section. The tiering is sidebar-only - `DOCS_PAGES` stays canonical for search/export/routing. The public `getting-started` page (`SECTION_GENERAL`) is the new-contributor on-ramp; gate its deep-internals links with `{% if is_admin(user) %}` so guests get no 404s. Keep one canonical home per concept (the `auth` API group defers auth-method detail to the `authentication` prose page). The member-facing `devii` prose page is functional; admins also get a `Devii internals` section of `devii-*` technical subpages. Prose docs pages are rendered to HTML **server-side** (`docs_prose.py`, mistune GFM): the handler converts the `data-render` markdown block to HTML before sending and drops the `data-render` attribute, so the client never re-renders it (no markdown-to-HTML flicker, faster first paint). Authored example markup inside that block is therefore still HTML-escaped (`<dp-...>`); content outside the block (component live-demo blocks and their `` breakout. -**Emoji shortcodes (single source of truth).** `:name:` shortcodes are the **full GitHub/Discord set** (~4869 names, e.g. `:rocket:`->๐Ÿš€, `:smiling_imp:`->๐Ÿ˜ˆ), NOT a curated allowlist. `rendering.py` `build_emoji_shortcodes()` is the ONE generator: it walks the `emoji` library's `EMOJI_DATA` (every `en`/`alias` name of a fully-qualified emoji) and merges `CUSTOM_SHORTCODES` (the handful of legacy extras the library lacks, e.g. `party`/`tools`). The backend `EMOJI_MAP = build_emoji_shortcodes()` is built ONCE at import and used by `_replace_shortcodes` (first step of `render_content`/`render_title`). The frontend gets the **same** map as a generated plain-ES6 module `static/js/emoji-shortcodes.js` (`export const EMOJI_SHORTCODES`), written by `write_emoji_module()` and imported by `ContentRenderer.js` (so `dp-content`/`dp-title` and every live renderer share it). The module is a committed build artifact - regenerate it with `devplace emoji-sync` after bumping the `emoji` dependency, never hand-edit it. Unknown shortcodes pass through verbatim; shortcodes inside code spans/blocks are never expanded. The separate visual `emoji-picker-element` composer button is unrelated (it inserts the literal emoji char). User docs: `/docs/emoji-shortcodes`. +**Emoji shortcodes (single source of truth).** `:name:` shortcodes are the **full GitHub/Discord set** (~4869 names, e.g. `:rocket:`->๐Ÿš€, `:smiling_imp:`->๐Ÿ˜ˆ), NOT a curated allowlist. `rendering.py` `build_emoji_shortcodes()` is the ONE generator: it walks the `emoji` library's `EMOJI_DATA` (every `en`/`alias` name of a fully-qualified emoji) and merges `CUSTOM_SHORTCODES` (the handful of legacy extras the library lacks, e.g. `party`/`tools`). The backend `EMOJI_MAP = build_emoji_shortcodes()` is built ONCE at import and used by `_replace_shortcodes` (first step of `render_content`/`render_title`). The frontend gets the **same** map as a generated plain-ES6 module `static/js/emoji-shortcodes.js` (`export const EMOJI_SHORTCODES`), written by `write_emoji_module()` and loaded by `ContentRenderer.js` via a top-level dynamic `import()` started at module evaluation (off the boot-critical graph; every boot-time render entry gates on `contentRenderer.ready` so client-rendered content never paints with unexpanded shortcodes - so `dp-content`/`dp-title` and every live renderer share it). The module is a committed build artifact - regenerate it with `devplace emoji-sync` after bumping the `emoji` dependency, never hand-edit it. Unknown shortcodes pass through verbatim; shortcodes inside code spans/blocks are never expanded. The separate visual `emoji-picker-element` composer button is unrelated (it inserts the literal emoji char). User docs: `/docs/emoji-shortcodes`. **Template em-dash normalization.** The shared `templates.env.template_class` is overridden (`templating.py` `_Template`) so EVERY rendered template runs its final HTML through `normalize_em_dash` - the em-dash character (U+2014) and its HTML entity forms (the named entity plus decimal 8212 / hex 2014) all become a plain `-`, application-wide, covering literal template text, `{{ variables }}`, includes/macros, and injected `render_content` HTML. One hook; never re-strip em-dash per template. (`render_content`/`render_title` also normalize em-dash in their own pipeline.) ### Auth Session cookies named `session`, value is a 64-char hex token from `secrets.token_hex(32)`. Passwords hashed with `pbkdf2_sha256` via passlib. `get_current_user(request)` returns user dict or `None`, with a per-process TTL cache (`_user_cache`, 300s) keyed by token. `require_user(request)` raises a 303 redirect to `/` for guests; `require_admin(request)` redirects to `/feed` for non-admins. Public pages (feed, news detail, project detail, gists) use `get_current_user` so guests can browse - POSTs are all guarded by `require_user`. -### Database (`devplacepy/database.py`) +### Database (`devplacepy/database/`) SQLite via `dataset.connect` with WAL + 30s busy timeout + 256MB mmap. `init_db()` is idempotent - every index is created via `CREATE INDEX IF NOT EXISTS` wrapped in try/except, and seed defaults for `site_settings` only insert when the key doesn't already exist. **`init_db()` also ensures the full column set of any table that code filters on** (`news`, `news_sync`, `attachments` use `get_table(name)` + `create_column_by_example` before their indexes). This is mandatory, not optional: dataset gives a lazily-created table ONLY the columns of its first insert, so a partial insert from a CLI tool/test/maintenance script would otherwise create the table with a reduced schema, and the long-running server caches that stale schema - making later queries on the missing column 500 with `no such column`. When adding a new filtered/indexed column, add it to the matching `init_db()` ensure-block (see `AGENTS.md` -> "Dataset rules -> `init_db()` MUST create every queried column"). The `_index(...)` helper supports `where=` (partial) and `unique=` indexes; every table with a `uid` column gets a UNIQUE `idx_
_uid`, soft-delete tables get a PARTIAL `idx_
_trash` (`WHERE deleted_at IS NOT NULL`) and NEVER a bare `deleted_at` index (it mis-steers the planner on live reads), and "live newest-first" listings need a composite/live-partial index that carries the sort column. `init_db()` finishes with `ANALYZE`/`PRAGMA optimize`. Verify any index change with `EXPLAIN QUERY PLAN` (see `AGENTS.md` -> "Indexing conventions (the soft-delete planner trap)"). **SQLite is synchronous by design and will never be made async (hard rule).** The `dataset`/SQLite layer is synchronous and is called directly inside the async route handlers. This is intentional and more than fast enough for this platform: the database is a local file tuned with WAL, `synchronous=NORMAL`, a 30s busy timeout, an 8MB page cache, and a 256MB mmap, so queries are sub-millisecond and never meaningfully stall the event loop. Do NOT convert the database layer to async, do NOT wrap DB calls in a threadpool/`run_in_executor`/`to_thread`, and do NOT raise the synchronous DB access as a performance, blocking-I/O, or "requests aren't fully async" concern. This is a settled, sophisticated design choice and is not open for revisiting. @@ -200,12 +220,12 @@ The **standard** way to run heavy, blocking work off the request path and return The `/issues` feature is a **full Gitea integration with no local issue store** - the listing and detail views read issues straight from Gitea (default repo `retoor/devplacepy`) with live status. `services/gitea/` holds `config.py` (the admin-editable `CONFIG_FIELDS` for base URL/owner/repo/token + AI model, surfaced on the `IssueTrackerService` config at `/admin/services`; `gitea_config()` -> `GiteaConfig`), `client.py` (async `GiteaClient` over the REST API, `Authorization: token`, `GiteaError`), `fake.py` (`FakeGiteaClient` test double), `runtime.py` (`get_client`/`set_client` swap), `enhance.py` (`enhance_ticket` rewrites a raw report into a consistent markdown ticket via the **internal AI gateway**, fail-soft to a deterministic template), `store.py` (the only DB state: `issue_tickets` and `issue_comment_authors`, both in `SOFT_DELETE_TABLES` - thin mappings of Gitea number/comment-id -> DevPlace author for attribution + cached `last_status`/`last_comment_count` for update detection), and `service.py` (`IssueTrackerService`, `default_enabled=False`, polls tracked tickets and notifies the **reporter** on developer replies / status changes). Filing is an **async job** (`IssueCreateService`, kind `issue_create`): enhance -> create Gitea issue -> record mapping -> notify reporter; `POST /issues/create` enqueues, frontend `app.issueReporter` polls `/issues/jobs/{uid}`. Comments (`POST /issues/{number}/comment`, notifies **all admins**) and admin status changes (`POST /issues/{number}/status`, open/closed) are synchronous single Gitea calls. All Gitea actions use one shared bot token; per-user identity is shown by storing DevPlace author names locally, never per-user Gitea accounts. ### Audit log (`devplacepy/services/audit/`) -**Admin-only, append-only** record of every state-changing action; the authoritative event catalogue (201 keys across 35 domains) is `events.md`. Package: `store.py` (tables `audit_log` + `audit_log_links`, `ensure_tables`/`insert_event`/`insert_links`/`get_event`/`get_links`/`sweep`), `categories.py` (`category_for(event_key)`), `record.py` (the recorder + link builders), `query.py` (admin list/detail), `service.py` (`AuditService` retention). `ensure_tables()` + 11 indexes are wired into `init_db()` via a local import (audit modules import `database`/`utils`, so the import is deferred to avoid the cycle); `AuditService` is registered in `main.py`. **Two recorder entrypoints (the reuse keystone):** `record(request, event_key, *, user=_UNSET, actor_kind, target_*, old_value, new_value, summary, metadata, result, origin, via_agent, links, category)` for HTTP/WebSocket handlers (`request` may be a `Request` OR a `WebSocket`; auto-derives actor from `get_current_user`, request fields, the `X-Devii-Agent` header -> `origin=devii`/`via_agent=1`, and auto-appends the `actor` link), and `record_system(event_key, *, actor_kind, actor_uid, actor_username, actor_role, origin, via_agent, ...)` for request-less contexts (rewards, notifications, the AI gateway ledger, news/container services, Devii turns/tasks, job services, CLI). **Both are best-effort: wrapped in try/except, they NEVER raise into the caller** - the audited action is never blocked by a logging failure. `summary` is HTML-stripped + 140-char capped; `metadata` is JSON. **Emission is explicit per mutation** (`audit.record(...)`/`record_system(...)` after success, or on the guard branch with `result="denied"`/`"failure"`), with DRY choke points: posts/projects/gists record inside `content.py`; project file ops via the `project_files.py` helpers (read-only guard -> `denied`); container ops in `routers/containers.py` (HTTP path) and in the Devii `actions/dispatcher.py` `_audit_mechanic` hook (agent path; the two paths are disjoint so there is no double-count), which also emits the `devii.*` self-config mechanics. Auth failures, authz denials (in `require_user`/`require_admin`), rate-limit/maintenance blocks, self-role-change/self-disable denials, and quota blocks are recorded with `result` set. **Admin UI:** `GET /admin/audit-log` (paginated, filterable list) + `GET /admin/audit-log/{uid}` (detail + links), both `require_admin`, both negotiate via `respond(..., model=AuditLogOut|AuditEventOut)`; sidebar link sits last (before Settings). `_pagination.html` gained an optional backward-compatible `pagination_query` prefix so filters survive paging. **Devii (read-only):** admins query the same two routes conversationally via the `audit_log`/`audit_event` admin actions (`services/devii/actions/catalog.py`, `handler="http"`, `requires_admin=True`) - no new endpoint, since the `respond(...)` routes already serve JSON to Devii's `Accept: application/json` client (like `admin_list_users`); `audit_log` forwards every filter as a query param and its `AuditLogOut.options` lets the agent discover valid filter values in one call. **Retention:** `AuditService` (daily) prunes rows older than `audit_log_retention_days` (default 90; `0` disables). To add an event: pick/extend a key in `events.md`, extend `category_for` for a new domain, and call the recorder at the mutation point - never gate the action on the recording. +**Admin-only, append-only** record of every state-changing action; the authoritative event catalogue (223 keys across 38 domains) is `events.md`. Package: `store.py` (tables `audit_log` + `audit_log_links`, `ensure_tables`/`insert_event`/`insert_links`/`get_event`/`get_links`/`sweep`), `categories.py` (`category_for(event_key)`), `record.py` (the recorder + link builders), `query.py` (admin list/detail), `service.py` (`AuditService` retention). `ensure_tables()` + 11 indexes are wired into `init_db()` via a local import (audit modules import `database`/`utils`, so the import is deferred to avoid the cycle); `AuditService` is registered in `main.py`. **Two recorder entrypoints (the reuse keystone):** `record(request, event_key, *, user=_UNSET, actor_kind, target_*, old_value, new_value, summary, metadata, result, origin, via_agent, links, category)` for HTTP/WebSocket handlers (`request` may be a `Request` OR a `WebSocket`; auto-derives actor from `get_current_user`, request fields, the `X-Devii-Agent` header -> `origin=devii`/`via_agent=1`, and auto-appends the `actor` link), and `record_system(event_key, *, actor_kind, actor_uid, actor_username, actor_role, origin, via_agent, ...)` for request-less contexts (rewards, notifications, the AI gateway ledger, news/container services, Devii turns/tasks, job services, CLI). **Both are best-effort: wrapped in try/except, they NEVER raise into the caller** - the audited action is never blocked by a logging failure. `summary` is HTML-stripped + 140-char capped; `metadata` is JSON. **Emission is explicit per mutation** (`audit.record(...)`/`record_system(...)` after success, or on the guard branch with `result="denied"`/`"failure"`), with DRY choke points: posts/projects/gists record inside `content.py`; project file ops via the `project_files.py` helpers (read-only guard -> `denied`); container ops in `routers/containers.py` (HTTP path) and in the Devii `actions/dispatcher.py` `_audit_mechanic` hook (agent path; the two paths are disjoint so there is no double-count), which also emits the `devii.*` self-config mechanics. Auth failures, authz denials (in `require_user`/`require_admin`), rate-limit/maintenance blocks, self-role-change/self-disable denials, and quota blocks are recorded with `result` set. **Admin UI:** `GET /admin/audit-log` (paginated, filterable list) + `GET /admin/audit-log/{uid}` (detail + links), both `require_admin`, both negotiate via `respond(..., model=AuditLogOut|AuditEventOut)`; sidebar link sits last (before Settings). `_pagination.html` gained an optional backward-compatible `pagination_query` prefix so filters survive paging. **Devii (read-only):** admins query the same two routes conversationally via the `audit_log`/`audit_event` admin actions (`services/devii/actions/catalog.py`, `handler="http"`, `requires_admin=True`) - no new endpoint, since the `respond(...)` routes already serve JSON to Devii's `Accept: application/json` client (like `admin_list_users`); `audit_log` forwards every filter as a query param and its `AuditLogOut.options` lets the agent discover valid filter values in one call. **Retention:** `AuditService` (daily) prunes rows older than `audit_log_retention_days` (default 90; `0` disables). To add an event: pick/extend a key in `events.md`, extend `category_for` for a new domain, and call the recorder at the mutation point - never gate the action on the recording. ### Container manager (`devplacepy/services/containers/`) -**Admin-only.** Runs supervised container instances, all on one shared prebuilt image. Driven through the `docker` CLI via `asyncio.create_subprocess_exec` behind a pluggable **`Backend` ABC** (`backend/base.py`); `DockerCliBackend` streams run logs line-by-line, `FakeBackend` backs tests; `runtime.get_backend()`/`set_backend()` selects it. **Security: this requires mounting the docker socket, which is root-equivalent on the host - every run/exec/lifecycle/schedule op is `require_admin` (HTTP) and `requires_admin=True` (Devii); never `--privileged`; all calls are arg-list subprocesses (never `shell=True`).** **There is NO in-app image building.** Every instance runs ONE shared prebuilt image, `config.CONTAINER_IMAGE` (default `ppy:latest`, override `DEVPLACE_CONTAINER_IMAGE`), built once from `ppy.Dockerfile` via `make ppy`. There are no per-project Dockerfiles, builds, or `ContainerBuildService`. Data model (`store.py`, indexed in `init_db`): `instances` + `instance_events` + `instance_metrics` (ring-buffered, METRICS_RING=720) + `instance_schedules`. `api.create_instance(project, *, name, ...)` fails fast with a `ContainerError` if `backend.image_exists(CONTAINER_IMAGE)` is false (run `make ppy`); it no longer takes a dockerfile/build. **`ContainerService`** (`service.py`, a reconciler `BaseService`) each tick snapshots `docker ps` (label `devplace.instance=` is the join key), converges each instance to its `desired_state`, applies restart policies, reaps orphan containers (labeled but no DB row -> no orphans/no lost state), fires due `instance_schedules` (reusing `devii/tasks/schedule.py` `cron_next`/`next_run`), and samples `docker stats`. Only the service lock owner reconciles; HTTP handlers only edit `desired_state`. The `/app` workspace is materialized once per project from the virtual FS (`project_files.export_to_dir`) under `config.CONTAINER_WORKSPACES_DIR` (`DATA_DIR/container_workspaces`, default `data/`), bind-mounted RW; the sync action imports it back (`project_files.import_from_dir`). **Runtime data must live OUTSIDE the `devplacepy/` package**: workspaces in `DATA_DIR` (configurable via `DEVPLACE_DATA_DIR`, NOT under `/static`). `api.py` holds the operations shared by `routers/containers.py` and the Devii `ContainerController` (`services/devii/container/`, `handler="container"`). **UI:** the per-project manager is `templates/containers.html` + `static/js/ContainerManager.js` (instance creation via app modal forms; its instance list links to the shared detail page). The admin **Containers** section (`routers/containers_admin.py`, `templates/containers_admin.html` + `static/js/ContainerList.js`, sidebar link in `admin_base.html`) lists all instances across projects and links each row to the dedicated detail page `templates/containers_instance.html` + `static/js/ContainerInstance.js` (poll-based logs/metrics, lifecycle, schedules, ingress, sync, a one-shot exec box, and an **Open terminal** button). The admin detail page does NOT add its own lifecycle endpoints - it reuses the per-project `/projects/{slug}/containers/instances/{uid}/...` routes (the instance carries its `project_uid`). All container UI styling uses the app design tokens in `static/css/containers.css`. **Floating terminals (interactive shell):** an instance's shell is a floating **xterm.js** window (`static/js/components/ContainerTerminal.js`; xterm + fit addon vendored under `static/vendor/xterm/`, lazy-loaded), opened by the instance-page **Open terminal** button (`app.containerTerminals.open(slug, uid, name)`) or by telling Devii to "attach <container>" (the admin-only `open_terminal` **client** action -> `DeviiClient._openTerminal` -> the same manager). It extends the generic **`FloatingWindow`** base (`static/js/components/FloatingWindow.js` + `static/css/floating-window.css`): draggable/resizable/maximize/fullscreen chrome identical to the Devii window. **The Devii terminal (`devii/devii-terminal.js`) also `extends FloatingWindow`** and reuses the same drag/geometry/preset/resize/persist plus `_window`/`_registerWindow` machinery, overriding only what differs (a `closed` state + launcher FAB, bottom-right default geometry, per-user `--devii-font-size` font, `_contextItems`, and the richer `{state,geometry,focused}` persistence); `devii.css` is unchanged, the only base change being a `get _dragClass()` getter Devii overrides to `devii-dragging`. `app.windows` (`WindowManager`) gives every floating window - Devii included - a shared **last-clicked-on-top** z-order (below the avatar/toasts). The exec WS (`/projects/{slug}/containers/instances/{uid}/exec/ws`, `pty.openpty()` + `docker exec -it`, admin + lock-owner gated) now also accepts a JSON `{"type":"resize","cols","rows"}` control frame: it sets the host pty winsize (`TIOCSWINSZ`) **and** signals the `docker exec` client (`proc.send_signal(SIGWINCH)`) so the new size forwards through to the container's exec tty (vim/top reflow) - the SIGWINCH is required because the host slave is not the client's controlling terminal. Any non-resize frame is raw stdin (backward compatible). The fit addon drives it client-side and the window has a dedicated bottom-right resize grip (`.fw-resize`, since xterm covers the native CSS resize corner). xterm renders ANSI natively, so the old `cleanTerm` stripping is gone for the interactive shell. **Persistence:** a `?session=` query param makes the WS run the shell inside **tmux** (`tmux new-session -A -s `, falling back to bash if tmux is absent) so the session survives WS disconnect/window close (a running server keeps running; closing the WS only detaches - `proc.kill` kills the tmux *client*, not the server). The frontend keeps **one persistent slot** at a time (`ContainerTerminalManager`, the last-opened terminal): it attaches to session `devplace`, **auto-reconnects** with backoff on unexpected close, and is remembered per-user (`ct-persistent:` in localStorage) so `app.containerTerminals.restore()` (called once in `Application.js`) reopens it on the next page load. Opening another terminal demotes the previous one to non-persistent (its tmux session lingers but the frontend stops auto-managing it). **Right-click** any window opens the shared `dp-context-menu` (`app.contextMenu.attach`): the container terminal menu has Copy (xterm `getSelection`) / Paste (clipboard -> WS stdin) / Sync files / Restart / Terminate (confirm via `dp-dialog`) / Minimize / Normalize / Project (-> `/projects/{slug}`) / Files (-> `/projects/{slug}/files`) / Close; the Devii menu has Copy (selection or input line) / Paste (clipboard -> input at caret) / Minimize / Normalize / Close (it is not bound to a container or project). **Minimize** (smallest usable size) and **Normalize** (comfortable size) are geometry presets (`_presetGeometry`), exposed both as titlebar buttons and menu items on every window. The reconciler defaults disabled (needs docker); enable on `/admin/services`. **The `ppy` image** (`ppy.Dockerfile`, built by `make ppy` with context `devplacepy/services/containers/files`) is a `python:3.13-slim-bookworm` base with Playwright + a broad set of common Python libraries preinstalled, plus CLI tools (`tmux`, `apache2-utils` for `ab`, `procps`/`htop`/`iftop`/`iotop`, `netcat-openbsd` for `nc`, `zip`/`unzip`, `fakeroot`, git/curl/wget/vim/ack), plus the security hotpatch baked in: it `COPY`s the **sudo superclone** (`files/sudo`) over `/usr/local/bin/sudo` (+ symlink `/usr/bin/sudo`; the real `sudo` package is not installed), the **`aptroot` fakeroot wrapper** (`files/aptroot`, symlinked over `apt`/`apt-get`/`dpkg` in `/usr/local/bin` so pravda installs system packages without root), and **`pagent`** (`files/pagent`, the stdlib AI agent) to `/usr/bin/pagent.py` (plus `files/.vimrc` -> `/home/pravda/.vimrc`, whose AI helper targets the same gateway as pagent via `PRAVDA_OPENAI_URL`/`PRAVDA_API_KEY`), evicts any pre-existing uid-1000 user, creates user **`pravda` at `1000:1000`**, hands pravda ownership of the toolchain AND the OS package trees (`chown -R pravda` over `/usr/local/lib`, `/usr/local/bin`, `/usr/lib/python3`, `/opt`, `/app`, `/home/pravda`, plus `/usr/lib`, `/usr/bin`, `/usr/sbin`, `/usr/share`, `/usr/include`, `/etc`, `/var/lib`, `/var/cache`, `/var/log`, `/srv` so `apt`/`dpkg` can write; `~/.local/bin` on `PATH`), and ends on `USER pravda`. **Why uid 1000:** `/app` is bind-mounted from the host and under DooD a container's UID maps 1:1 to the host, so a container running as **root** wrote root-owned files into the workspace, after which the app process (host `retoor`, uid 1000) hit `[Errno 13] Permission denied` in `export_to_dir` on the next instance create - a permanent per-project brick. Pinning the container UID to 1000 makes every write land as `retoor`, and pravda owning the global site-packages means runtime `pip install` succeeds with no elevation. **The sudo superclone** (`files/sudo`, POSIX `sh`) is sudo-CLI-compatible but **never swaps user**: it parses the full flag interface (`-u/-g/-E/-H/-n/-S/-i/-s/--`, leading `VAR=value` assignments, `-V/-l/-v/-k` short-circuits), exports `SUDO_USER`/`SUDO_UID`/`SUDO_GID`/`SUDO_COMMAND`, then `exec`s the command **as pravda (uid 1000)**, so even a blind `sudo ...` cannot create a root-owned file. **Rootless apt (the `aptroot` fakeroot wrapper):** pravda can run `apt`/`apt-get`/`dpkg` directly (no sudo) and install system packages. `files/aptroot` is symlinked over `apt`/`apt-get`/`dpkg` in `/usr/local/bin` (ahead of `/usr/bin` on `PATH`) and execs the real tool under **`fakeroot`** with `APT::Sandbox::User=root`, so dpkg's chown-to-root calls are virtualized while every file actually written lands owned by **pravda (uid 1000)**. Combined with pravda owning the system trees (`/usr`, `/etc`, `/var/lib`, `/var/cache`, `/var/log`, `/srv`, ...), `apt install ` just works with no escalation and still cannot create a root-owned file on the bind mount (there is no real euid 0). It runs `apt update` first (the image clears `/var/lib/apt/lists`); a package whose maintainer script needs genuine privileged syscalls may still fail - bake those into `ppy.Dockerfile` + `make ppy`. **Trade-off (intentional):** the only genuinely-root op that still does NOT escalate is binding a port < 1024 - use a high port + `/p/` ingress. The `export_to_dir` unlink-before-write fix remains as belt-and-suspenders. `pagent` reads `PRAVDA_OPENAI_URL`+`PRAVDA_API_KEY` (the gateway), falling back to its public endpoint + `DEEPSEEK_API_KEY`. **`PRAVDA_*` env injection:** `api.run_spec_for` merges `api.pravda_env(instance)` over the instance `env_json` (PRAVDA wins) so every container is launched with `PRAVDA_BASE_URL` (`site_url` via `seo.public_base_url()`), `PRAVDA_OPENAI_URL` (`{base}/openai/v1`), `PRAVDA_API_KEY` (the **creating** user's `api_key`, resolved live), `PRAVDA_USER_UID` (the project **owner** uid), `PRAVDA_CONTAINER_NAME`, `PRAVDA_CONTAINER_UID`, `PRAVDA_INGRESS_URL` (absolute `{base}/p/{ingress_slug}` when the instance is published and `site_url` is set, else relative `/p/{slug}`, else empty) - injected at run via `docker run -e` (never baked into the shared image, never stored as a DB secret). `create_instance` records `created_by` (actor user) and `owner_uid` (`project["user_uid"]`) to back the api-key/uid; pre-existing instances degrade to empty key/uid until recreated, and the base/openai URLs are empty unless `site_url` is set. **Ingress:** an instance can set `ingress_slug` + `ingress_port` (a mapped container port); `routers/proxy.py` (`/p/{slug}`, registered in `main.py`) then **publicly** reverse-proxies HTTP and WebSocket to the address from `api.proxy_target(instance)` (`store.find_instance_by_ingress`; target host/port are derived from the instance, not user input, so no SSRF). **`proxy_target` dials the container's docker bridge gateway + the published host port** (`gateway:host_port`), NOT loopback and NOT the container's own bridge IP. The reconciler records `container_ip`/`container_gateway` from `docker inspect` on every running tick. This is deliberate: `127.0.0.1` fails where docker's `nat OUTPUT` excludes `127.0.0.0/8` from DNAT and no `docker-proxy` binds loopback for a `0.0.0.0`-published port, and the container's own bridge IP can be dropped by docker's bridge-isolation rule (`DOCKER ! -i docker0 -o docker0 -j DROP`); the gateway+published-port path survives both. Setting `DEVPLACE_CONTAINER_PROXY_HOST` overrides the host (e.g. `host.docker.internal` for a containerized app) and still uses the published host port; before a gateway is recorded the proxy falls back to `127.0.0.1`. The Devii container tools return an absolute `ingress_url` built from `seo.public_base_url()` (the admin `site_url` setting), so the agent fetches the public production URL, not localhost (which web tools refuse); unset `site_url` yields a relative `/p/`. **Production** needs heavy wiring (the container manager drives the host docker daemon), all carried by the opt-in `docker-compose.containers.yml` overlay (docker socket mount, `INSTALL_DOCKER_CLI=true`, `group_add` the docker gid). **The `make docker-build`/`make docker-up` targets always apply this overlay** and self-derive its inputs - `DOCKER_GID` via `stat -c '%g' /var/run/docker.sock` (the socket's owning group) and `DEVPLACE_DATA_DIR` = `$(CURDIR)/data` (the project's own dir at its real host absolute path) - so the manager works out of the box with no `sudo`, no `/srv`, no manual `.env`. A plain `docker compose up -d` drops the overlay (no CLI, no socket) and silently re-breaks it; always update through the make targets. The **DooD bind-mount gotcha**: `docker run -v :/app` resolves `` on the HOST, so `DEVPLACE_DATA_DIR` must be mounted at an identical host+container path (the make targets use `$(CURDIR)/data` on both sides; manual `docker compose` users get the `/srv/devplace-data` default); build contexts go via the docker API tarball so temp is fine. The ingress proxy reaches published ports through each instance's recorded docker bridge gateway (`api.proxy_target` -> `gateway:host_port`); set `DEVPLACE_CONTAINER_PROXY_HOST=host.docker.internal` only when the containerized app cannot route to that gateway. nginx proxies `/p/` with WS upgrade + long timeouts. +**Admin-only.** Runs supervised container instances, all on one shared prebuilt image. Driven through the `docker` CLI via `asyncio.create_subprocess_exec` behind a pluggable **`Backend` ABC** (`backend/base.py`); `DockerCliBackend` streams run logs line-by-line, `FakeBackend` backs tests; `runtime.get_backend()`/`set_backend()` selects it. **Security: this requires mounting the docker socket, which is root-equivalent on the host - every run/exec/lifecycle/schedule op is `require_admin` (HTTP) and `requires_admin=True` (Devii); never `--privileged`; all calls are arg-list subprocesses (never `shell=True`).** **There is NO in-app image building.** Every instance runs ONE shared prebuilt image, `config.CONTAINER_IMAGE` (default `ppy:latest`, override `DEVPLACE_CONTAINER_IMAGE`), built once from `ppy.Dockerfile` via `make ppy`. There are no per-project Dockerfiles, builds, or `ContainerBuildService`. Data model (`store.py`, indexed in `init_db`): `instances` + `instance_events` + `instance_metrics` (ring-buffered, METRICS_RING=720) + `instance_schedules`. `api.create_instance(project, *, name, ...)` fails fast with a `ContainerError` if `backend.image_exists(CONTAINER_IMAGE)` is false (run `make ppy`); it no longer takes a dockerfile/build. **`ContainerService`** (`service.py`, a reconciler `BaseService`) each tick snapshots `docker ps` (label `devplace.instance=` is the join key), converges each instance to its `desired_state`, applies restart policies, reaps orphan containers (labeled but no DB row -> no orphans/no lost state), fires due `instance_schedules` (reusing `devii/tasks/schedule.py` `cron_next`/`next_run`), and samples `docker stats`. Only the service lock owner reconciles; HTTP handlers only edit `desired_state`. The `/app` workspace is materialized once per project from the virtual FS (`project_files.export_to_dir`) under `config.CONTAINER_WORKSPACES_DIR` (`DATA_DIR/container_workspaces`, default `data/`), bind-mounted RW; the sync action imports it back (`project_files.import_from_dir`). **Runtime data must live OUTSIDE the `devplacepy/` package**: workspaces in `DATA_DIR` (configurable via `DEVPLACE_DATA_DIR`, NOT under `/static`). `api.py` holds the operations shared by `routers/containers.py` and the Devii `ContainerController` (`services/devii/container/`, `handler="container"`). **UI:** the per-project manager is `templates/containers.html` + `static/js/ContainerManager.js` (instance creation via app modal forms; its instance list links to the shared detail page). The admin **Containers** section (`routers/containers_admin.py`, `templates/containers_admin.html` + `static/js/ContainerList.js`, sidebar link in `admin_base.html`) lists all instances across projects and links each row to the dedicated detail page `templates/containers_instance.html` + `static/js/ContainerInstance.js` (poll-based logs/metrics, lifecycle, schedules, ingress, sync, a one-shot exec box, and an **Open terminal** button). The admin detail page does NOT add its own lifecycle endpoints - it reuses the per-project `/projects/{slug}/containers/instances/{uid}/...` routes (the instance carries its `project_uid`). All container UI styling uses the app design tokens in `static/css/containers.css`. **Floating terminals (interactive shell):** an instance's shell is a floating **xterm.js** window (`static/js/components/ContainerTerminal.js`; xterm + fit addon vendored under `static/vendor/xterm/`, lazy-loaded), opened by the instance-page **Open terminal** button (`app.containerTerminals.open(slug, uid, name)`) or by telling Devii to "attach <container>" (the admin-only `open_terminal` **client** action -> `DeviiClient._openTerminal` -> the same manager). It extends the generic **`FloatingWindow`** base (`static/js/components/FloatingWindow.js` + `static/css/floating-window.css`): draggable/resizable/maximize/fullscreen chrome identical to the Devii window. **The Devii terminal (`devii/devii-terminal.js`) also `extends FloatingWindow`** and reuses the same drag/geometry/preset/resize/persist plus `_window`/`_registerWindow` machinery, overriding only what differs (a `closed` state + launcher FAB, bottom-right default geometry, per-user `--devii-font-size` font, `_contextItems`, and the richer `{state,geometry,focused}` persistence); `devii.css` is unchanged, the only base change being a `get _dragClass()` getter Devii overrides to `devii-dragging`. `app.windows` (`WindowManager`) gives every floating window - Devii included - a shared **last-clicked-on-top** z-order (below the avatar/toasts). The exec WS (`/projects/{slug}/containers/instances/{uid}/exec/ws`, `pty.openpty()` + `docker exec -it`, admin + lock-owner gated) now also accepts a JSON `{"type":"resize","cols","rows"}` control frame: it sets the host pty winsize (`TIOCSWINSZ`) **and** signals the `docker exec` client (`proc.send_signal(SIGWINCH)`) so the new size forwards through to the container's exec tty (vim/top reflow) - the SIGWINCH is required because the host slave is not the client's controlling terminal. Any non-resize frame is raw stdin (backward compatible). The fit addon drives it client-side and the window has a dedicated bottom-right resize grip (`.fw-resize`, since xterm covers the native CSS resize corner). xterm renders ANSI natively, so the old `cleanTerm` stripping is gone for the interactive shell. **Persistence:** a `?session=` query param makes the WS run the shell inside **tmux** (`tmux new-session -A -s `, falling back to bash if tmux is absent) so the session survives WS disconnect/window close (a running server keeps running; closing the WS only detaches - `proc.kill` kills the tmux *client*, not the server). The frontend keeps **one persistent slot** at a time (`ContainerTerminalManager`, the last-opened terminal): it attaches to session `devplace`, **auto-reconnects** with backoff on unexpected close, and is remembered per-user (`ct-persistent:` in localStorage) so `app.containerTerminals.restore()` (called once in `Application.js`) reopens it on the next page load. Opening another terminal demotes the previous one to non-persistent (its tmux session lingers but the frontend stops auto-managing it). **Right-click** any window opens the shared `dp-context-menu` (`app.contextMenu.attach`): the container terminal menu has Copy (xterm `getSelection`) / Paste (clipboard -> WS stdin) / Sync files / Restart / Terminate (confirm via `dp-dialog`) / Minimize / Normalize / Project (-> `/projects/{slug}`) / Files (-> `/projects/{slug}/files`) / Close; the Devii menu has Copy (selection or input line) / Paste (clipboard -> input at caret) / Minimize / Normalize / Close (it is not bound to a container or project). **Minimize** (smallest usable size) and **Normalize** (comfortable size) are geometry presets (`_presetGeometry`), exposed both as titlebar buttons and menu items on every window. The reconciler defaults disabled (needs docker); enable on `/admin/services`. **The `ppy` image** (`ppy.Dockerfile`, built by `make ppy` with context `devplacepy/services/containers/files`) is a `python:3.13-slim-bookworm` base with Playwright + a broad set of common Python libraries preinstalled, plus CLI tools (`tmux`, `apache2-utils` for `ab`, `procps`/`htop`/`iftop`/`iotop`, `netcat-openbsd` for `nc`, `zip`/`unzip`, `fakeroot`, git/curl/wget/vim/ack), plus the security hotpatch baked in: it `COPY`s the **sudo superclone** (`files/sudo`) over `/usr/local/bin/sudo` (+ symlink `/usr/bin/sudo`; the real `sudo` package is not installed), the **`aptroot` fakeroot wrapper** (`files/aptroot`, symlinked over `apt`/`apt-get`/`dpkg` in `/usr/local/bin` so pravda installs system packages without root), and **`pagent`** (`files/pagent`, the stdlib AI agent) to `/usr/bin/pagent.py` (plus `files/.vimrc` -> `/home/pravda/.vimrc`, whose AI helper targets the same gateway as pagent via `DEVPLACE_OPENAI_URL`/`DEVPLACE_API_KEY`), evicts any pre-existing uid-1000 user, creates user **`pravda` at `1000:1000`**, hands pravda ownership of the toolchain AND the OS package trees (`chown -R pravda` over `/usr/local/lib`, `/usr/local/bin`, `/usr/lib/python3`, `/opt`, `/app`, `/home/pravda`, plus `/usr/lib`, `/usr/bin`, `/usr/sbin`, `/usr/share`, `/usr/include`, `/etc`, `/var/lib`, `/var/cache`, `/var/log`, `/srv` so `apt`/`dpkg` can write; `~/.local/bin` on `PATH`), and ends on `USER pravda`. **Why uid 1000:** `/app` is bind-mounted from the host and under DooD a container's UID maps 1:1 to the host, so a container running as **root** wrote root-owned files into the workspace, after which the app process (host `retoor`, uid 1000) hit `[Errno 13] Permission denied` in `export_to_dir` on the next instance create - a permanent per-project brick. Pinning the container UID to 1000 makes every write land as `retoor`, and pravda owning the global site-packages means runtime `pip install` succeeds with no elevation. **The sudo superclone** (`files/sudo`, POSIX `sh`) is sudo-CLI-compatible but **never swaps user**: it parses the full flag interface (`-u/-g/-E/-H/-n/-S/-i/-s/--`, leading `VAR=value` assignments, `-V/-l/-v/-k` short-circuits), exports `SUDO_USER`/`SUDO_UID`/`SUDO_GID`/`SUDO_COMMAND`, then `exec`s the command **as pravda (uid 1000)**, so even a blind `sudo ...` cannot create a root-owned file. **Rootless apt (the `aptroot` fakeroot wrapper):** pravda can run `apt`/`apt-get`/`dpkg` directly (no sudo) and install system packages. `files/aptroot` is symlinked over `apt`/`apt-get`/`dpkg` in `/usr/local/bin` (ahead of `/usr/bin` on `PATH`) and execs the real tool under **`fakeroot`** with `APT::Sandbox::User=root`, so dpkg's chown-to-root calls are virtualized while every file actually written lands owned by **pravda (uid 1000)**. Combined with pravda owning the system trees (`/usr`, `/etc`, `/var/lib`, `/var/cache`, `/var/log`, `/srv`, ...), `apt install ` just works with no escalation and still cannot create a root-owned file on the bind mount (there is no real euid 0). It runs `apt update` first (the image clears `/var/lib/apt/lists`); a package whose maintainer script needs genuine privileged syscalls may still fail - bake those into `ppy.Dockerfile` + `make ppy`. **Trade-off (intentional):** the only genuinely-root op that still does NOT escalate is binding a port < 1024 - use a high port + `/p/` ingress. The `export_to_dir` unlink-before-write fix remains as belt-and-suspenders. `pagent` reads `DEVPLACE_OPENAI_URL`+`DEVPLACE_API_KEY` (the gateway), falling back to its public endpoint + `DEEPSEEK_API_KEY`. **`PRAVDA_*` env injection:** `api.run_spec_for` merges `api.pravda_env(instance)` over the instance `env_json` (PRAVDA wins) so every container is launched with `DEVPLACE_BASE_URL` (`site_url` via `seo.public_base_url()`), `DEVPLACE_OPENAI_URL` (`{base}/openai/v1`), `DEVPLACE_API_KEY` (the **creating** user's `api_key`, resolved live), `DEVPLACE_USER_UID` (the project **owner** uid), `DEVPLACE_CONTAINER_NAME`, `DEVPLACE_CONTAINER_UID`, `DEVPLACE_INGRESS_URL` (absolute `{base}/p/{ingress_slug}` when the instance is published and `site_url` is set, else relative `/p/{slug}`, else empty) - injected at run via `docker run -e` (never baked into the shared image, never stored as a DB secret). `create_instance` records `created_by` (actor user) and `owner_uid` (`project["user_uid"]`) to back the api-key/uid; pre-existing instances degrade to empty key/uid until recreated, and the base/openai URLs are empty unless `site_url` is set. **Ingress:** an instance can set `ingress_slug` + `ingress_port` (a mapped container port); `routers/proxy.py` (`/p/{slug}`, registered in `main.py`) then **publicly** reverse-proxies HTTP and WebSocket to the address from `api.proxy_target(instance)` (`store.find_instance_by_ingress`; target host/port are derived from the instance, not user input, so no SSRF). **`proxy_target` dials the container's docker bridge gateway + the published host port** (`gateway:host_port`), NOT loopback and NOT the container's own bridge IP. The reconciler records `container_ip`/`container_gateway` from `docker inspect` on every running tick. This is deliberate: `127.0.0.1` fails where docker's `nat OUTPUT` excludes `127.0.0.0/8` from DNAT and no `docker-proxy` binds loopback for a `0.0.0.0`-published port, and the container's own bridge IP can be dropped by docker's bridge-isolation rule (`DOCKER ! -i docker0 -o docker0 -j DROP`); the gateway+published-port path survives both. Setting `DEVPLACE_CONTAINER_PROXY_HOST` overrides the host (e.g. `host.docker.internal` for a containerized app) and still uses the published host port; before a gateway is recorded the proxy falls back to `127.0.0.1`. The Devii container tools return an absolute `ingress_url` built from `seo.public_base_url()` (the admin `site_url` setting), so the agent fetches the public production URL, not localhost (which web tools refuse); unset `site_url` yields a relative `/p/`. **Production** needs heavy wiring (the container manager drives the host docker daemon), all carried by the opt-in `docker-compose.containers.yml` overlay (docker socket mount, `INSTALL_DOCKER_CLI=true`, `group_add` the docker gid). **The `make docker-build`/`make docker-up` targets always apply this overlay** and self-derive its inputs - `DOCKER_GID` via `stat -c '%g' /var/run/docker.sock` (the socket's owning group) and `DEVPLACE_DATA_DIR` = `$(CURDIR)/data` (the project's own dir at its real host absolute path) - so the manager works out of the box with no `sudo`, no `/srv`, no manual `.env`. A plain `docker compose up -d` drops the overlay (no CLI, no socket) and silently re-breaks it; always update through the make targets. The **DooD bind-mount gotcha**: `docker run -v :/app` resolves `` on the HOST, so `DEVPLACE_DATA_DIR` must be mounted at an identical host+container path (the make targets use `$(CURDIR)/data` on both sides; manual `docker compose` users get the `/srv/devplace-data` default); build contexts go via the docker API tarball so temp is fine. The ingress proxy reaches published ports through each instance's recorded docker bridge gateway (`api.proxy_target` -> `gateway:host_port`); set `DEVPLACE_CONTAINER_PROXY_HOST=host.docker.internal` only when the containerized app cannot route to that gateway. nginx proxies `/p/` with WS upgrade + long timeouts. - **Bidirectional newer-wins sync (load-bearing direction rule).** Sync is NOT one-directional import. `project_files.sync_dir_bidirectional(project_uid, workspace, user) -> {"exported", "imported"}` is the one helper that reconciles a project's virtual FS against an instance `workspace_dir`: per file the side with the newer timestamp wins (project `updated_at` epoch vs filesystem `st_mtime`, with a 1s skew tolerance favouring export on ties), and a file present on only one side propagates to the other. It NEVER deletes a file - only creates/overwrites the older side. A **read-only** project (`is_readonly`) exports only, never imports (the read-only guard direction). The reconciler calls it (via `api.sync_bidirectional_sync`, a non-blocking `record_event` system actor) before every `_launch` AND on a ~60s wall-clock cadence over running instances (gated by `SYNC_EVERY_SECONDS`, independent of the 5s reconcile tick). Both `api.sync_workspace` (HTTP/Devii) and the reconciler reuse the same helper; the per-instance sync helper files (`.devplace_boot.py`/`.devplace_boot.sh`) are in `SYNC_SKIP_NAMES` so they never round-trip into the project. -- **Run-as user = identity + API key ONLY (load-bearing constraint).** An instance's `run_as_uid` column selects WHICH DevPlace user's identity and `api_key` are injected (`PRAVDA_API_KEY`, `PRAVDA_USER_UID`), resolved in `api.pravda_env` ahead of the `created_by`/`owner_uid` fallback chain. It does NOT change the container OS user, which is ALWAYS `pravda` (uid 1000) - that is required for the bind-mounted `/app` (DooD uid maps 1:1 to host). Validate it against an existing user (`api.validate_run_as`). +- **Run-as user = identity + API key ONLY (load-bearing constraint).** An instance's `run_as_uid` column selects WHICH DevPlace user's identity and `api_key` are injected (`DEVPLACE_API_KEY`, `DEVPLACE_USER_UID`), resolved in `api.pravda_env` ahead of the `created_by`/`owner_uid` fallback chain. It does NOT change the container OS user, which is ALWAYS `pravda` (uid 1000) - that is required for the bind-mounted `/app` (DooD uid maps 1:1 to host). Validate it against an existing user (`api.validate_run_as`). - **Boot source precedence (load-bearing).** New columns `boot_language` (`none`|`python`|`bash`) + `boot_script` (multiline source) sit alongside the legacy `boot_command`. Precedence in `api.run_spec_for`: `boot_script` (by language) > `boot_command` > image CMD (`sleep infinity`). When a boot script is set, the reconciler writes it into the workspace (`api.materialize_boot_script`, `.devplace_boot.py`/`.devplace_boot.sh`, excluded from sync) before launch and runs `python|bash /app/.devplace_boot.`. `api.validate_boot` enforces the language set and a 100k char cap. - **start_on_boot is per-container.** The `start_on_boot` integer flag (0/1) forces `desired_state=running` ONLY for flagged instances when `ContainerService` first runs (`_boot_pass`, one-time); all others keep their last desired_state. - **Status-change choke point.** Every reconciler status transition flows through `service._set_status(inst, changes, reason=)`, which diffs old vs new `status`, writes an `instance_events` `status_change` row AND an audit `container.instance.status` `record_system` event on change (formerly-silent `running->stopped`/`->paused`/exit transitions are now logged). The admin detail page renders the events list as a **Status history** card. New code that changes an instance's status in the reconciler must use `_set_status`, never a raw `store.update_instance(... status ...)`. @@ -219,9 +239,9 @@ Devii also drives the **user's own browser** through a `client` tool channel (sa Devii's project filesystem tools include surgical **line-range editing** (`project_read_lines`, `project_replace_lines`, `project_insert_lines`, `project_delete_lines`, `project_append_file`) so it edits large text files without resending them, and `Dispatcher._run_http` enforces an **overwrite-protection guard**: `project_write_file` is rejected only when the target `(slug, path)` **already exists** (the guard probes `GET /projects/{slug}/files/raw`) and the agent has not read it this session (call `project_read_file` first); creating a new file needs no prior read. Reads are recorded in the per-session `Dispatcher._read_files` set. This steers the agent to update files rather than blindly overwrite them and is agent-scoped only (the human UI and public HTTP API are unaffected). The `WRITING AND EDITING FILES` system-prompt rule mirrors it. ### Per-user customization (`devplacepy/customization.py`) -Users and guests inject their own **CSS** (look) and **JS** (behaviour), scoped to a **page type** or **globally**, configured **only through Devii** (no HTTP routes - persistence is owner-scoped from the trusted Devii session, like `LessonStore`). It is **per-owner only**: the code runs solely in that owner's own browser sessions (self-XSS, like a userscript), never in anyone else's view - never key injection off URL-supplied data. **Page type** = the matched route template (`request.scope["route"].path`, e.g. `/posts/{slug}`) via `page_type_for(request)`; one value covers all instances of a type (all posts), never one post; also rendered as `data-page-type` on `
` and surfaced to Devii via `DeviiClient._context().pageType`. **Owner** = `owner_for(request)`: user uid, else the `DEVII_GUEST_COOKIE` (`constants.py`, shared with `routers/devii.py`), else none. **Storage** (`database.py`): table `user_customizations` (per `owner_kind`/`owner_id`/`scope`/`lang`) with helpers `get_custom_overrides` (merges global-then-page-type, cached under the `"customizations"` cache-version name via `sync_local_cache`/`bump_cache_version`), `set_custom_override`, `delete_custom_override`. **Injection**: template globals `custom_css_tag`/`custom_js_tag` (registered in `templating.py`) emit a `