Update

Browse Source
This commit is contained in:
retoor 2026-08-01 01:14:24 +02:00
parent e02919a1db
commit 6c161401de
101 changed files with 20 additions and 6737 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
.claude/settings.local.json
View File

@ -1,20 +0,0 @@
{
"permissions": {
"allow": [
"Bash(python *)",
"Bash(DEVPLACE_DISABLE_SERVICES=1 python -)",
"Bash(command -v hawk)",
"Bash(export DEVPLACE_DISABLE_SERVICES=1)",
"Bash(export DEVPLACE_DATABASE_URL=\"sqlite:///tmp/devplace_verify.db\")",
"Bash(rm -f /tmp/devplace_verify.db)",
"Bash(export DEVPLACE_DATABASE_URL=\"sqlite:////tmp/devplace_verify.db\")",
"Bash",
"Edit(/home/retoor/projects/devplacepy/devplacepy/routers/projects/containers/instances.py)",
"Edit(/home/retoor/projects/devplacepy/devplacepy/static/js/components/ContainerTerminal.js)",
"Edit(/home/retoor/projects/devplacepy/devplacepy/services/containers/store.py)",
"Verify",
"Edit(/home/retoor/projects/devplacepy/devplacepy/static/js/MessagesLayout.js)",
"Write(/home/retoor/projects/devplacepy/devplacepy/static/css/messages.css)"
]
}
}

3
.gitignore vendored
View File

@ -14,7 +14,10 @@ notification-private.pem
notification-private.pkcs8.pem notification-private.pkcs8.pem
notification-public.pem notification-public.pem
.pytest_cache/ .pytest_cache/
.ruff_cache/
.opencode .opencode
.dpc/
.claude/settings.local.json
devii_*.db devii_*.db
devii_*.db-shm devii_*.db-shm
devii_*.db-wal devii_*.db-wal

313
CHANGELOG.md
View File

@ -1,313 +0,0 @@
## 2026-06-19 🟢
- Block and mute user relations with API endpoints, CLI emoji-sync command, and content filtering
## 2026-06-18 🟢
- News service with image dedup, AI grading, featured/landing auto-rotation and admin lock
- Server-rendered content pipeline with Telegram pairing, response timing, and admin user index
- AI Markdown reformatting for news articles with usage metering and sidebar cleanup
## 2026-06-17 🟢
- Backup download restricted to primary admin, admin-hidden projects invisible to other admins
- Access token system with CLI management and wildcard file type support
- Access token issuance with JSON/form login endpoint and token lifecycle management
## 2026-06-16 🔥 Big day!
- Gateway admin UI with provider and model routing for OpenAI gateway
- Backup management CLI commands and service layer with configurable data directories
- Audit logging for admin trash restore/purge and notification clear, plus SEO noindex for private projects and sitemap docs page refactor
- Instance lookup by name in addition to uid and slug, new terminal session service
- Keyboard-aware input visibility with ResizeObserver fallback for mobile message layout
- E2E comment hierarchy seed helpers for gists, news and projects
- Optimistic message insertion disabled to prevent duplicate bubbles
- Router tree documented in AGENTS.md with 14 new route entries
- Add dpc binary to container image and set executable permissions
- Remove .html and .svg from allowed upload types and MIME mappings
## 2026-06-15 🟢
- ASGI lifespan handler with background service orchestration and lock-based worker coordination
- Message chunking with sentence-aware splitting and configurable character limits
- Enforce hard test-coverage standard across DevPlace workflows and agents
- Audio file support with inline player and expanded allowed upload types
- Gist comment form integration with card-scoped comment targeting
- Bot account API key adoption for per-user gateway spend attribution
## 2026-06-14 🔥 Massive day!
- Admin/internal database API with CRUD, natural-language query, and read-only SQL execution
- DeepSearch research job queue with CLI prune/clear, Chroma vector store, and date-aware system message composition
- DeepSearch multi-agent researcher with grounded RAG chat and per-session vector store
- SEO Diagnostics tool with CLI management, live WebSocket progress, and static asset cache-busting
- OpenAI-compatible embeddings endpoint with model mapping and usage tracking
- Stealth HTTP client with curl_cffi transport adapter replacing raw httpx for outbound requests
- Bot monitor with live age badges and zoomable screenshots
- Author-interleaved feed ordering across all feed views and tabs
- Author diversity via interleaving (no per-author cap) for home and feed
- devRant API client library and example scripts in Python and JavaScript
- TTLCache-backed cache version reads with invalidation on bump
- Random client IP spoofing for load-testing traffic
- Deepsearch chat component attribute naming from data-* to direct properties
- Replace `python -m agents.validator` with `hawk` across all agent markdown files
## 2026-06-13 🔥 Massive day!
- Three-tier test suite with unit, API, and E2E directories mirroring source and endpoint paths
- Soft-delete audit for bookmarks, comments, follows, polls, project files, reactions, and bug create request event
- Notification preferences with per-user per-channel toggles and admin defaults
- Author diversity enforcement across home page and feed with personalized landing for authenticated users
- Shared free-text search across feed, gists, and projects listings
- Docs search with agent-powered Docii chat and admin-configurable search mode
- Devii agent audit log query action with filterable paginated endpoint
- Router directory-tree convention with admin audit log, AI quota, and container management endpoints
- Initial maintenance agent fleet with per-dimension code quality enforcers
- Unified blob sharding on uuid7 random tail across attachments, project files, and zip service
- Consolidated runtime data directory layout with migration CLI command
- Context-aware window control button visibility with font size boundary detection and minimize/normalize size presets
- Overflow-managed profile tabs with a "more" dropdown for narrow screens
- Startup jitter, randomized browser fingerprinting, and short comment styles for bot realism
- Sidebar search form with hidden field support and configurable placeholder
- Prevent titlebar double-click maximize when clicking buttons in FloatingWindow and DeviiTerminal
- Pin test server to single worker and use upsert for rate-limit settings to prevent spurious 429s
- DEVPLACE_DISABLE_RATE_LIMIT env var to bypass rate limiter in tests and middleware
- Fallback to location.origin when DEVPLACE_DOCS.base is missing
- Add claude-manual task-oriented guide page with cross-reference from claude.html
- Remove PWA install button and associated installer module
- Removed stale test files and fixed Gitea env teardown and ingress proxy test cleanup
- Locustfile seed data expansion and route exclusion documentation
## 2026-06-12 🔥 Massive day!
- Agent report system with codenames, timestamped output streams, and write-budget enforcement
- Tool-scoped payload filtering for worker agents with orchestration tool isolation
- Agent isolation and result caching in Maestro review sweep
- Concurrent read-only fleet check mode with per-agent cost tracking and contextvar-isolated findings
- Admin analytics and AI usage API response keys renamed, password change toggle added
- Gitea-backed bug tracker with list/detail/comment/status and AI-enhanced filing
- Bug detail page with admin/member role rendering and viewer_is_admin context flag
- Changed-files fast mode for maintenance agents with write-allowlist guard
- Partial config save with error reporting and password manager suppression
- Admin route cache-disabling headers via Cache-Control, Pragma and Expires
- Dirty-field tracking and server-side value sync for service config forms
- Rename `is_admin` to `viewer_is_admin` in bug detail schema, router, and template
- Bug tracker unavailable page with JSON and HTML 503 error responses
- Bot comments avoid repeating sibling opinions via thread-aware distinctness prompt
- Default Gitea repository changed from pydevplace to devplacepy
- Remove pytest-xdist parallel test execution, switch to serial single-process test runner
## 2026-06-11 🔥 Massive day!
- Platform-wide soft delete with deleted_at/deleted_by columns and admin trash management
- Owner-or-admin soft-delete enforcement on all content endpoints
- Unified image lightbox with attribute-wired opening and per-user media tab with soft delete
- Autonomous maintenance agent fleet with CLI entry point, Makefile targets, and dependency-free validator
- Seed-finding guided fix mode for maintenance agents with incomplete report tracking
- Audit log tables with CLI recording hooks
- Resolve merge conflict in pagination template and add admin-audit-log endpoint to docs API
- Bots documentation pages and session stop/reset commands
- Reduced nested comment indentation from 1.5rem to 0.25rem per depth level
- Reduce comment indentation multiplier and padding for nested replies
- Switch to dynamic viewport height and remove autofocus from message input
- Inline message layout with responsive height and auto-scroll
- Optional label attribute with hidden empty state for dp-upload component
- Mandatory retoor header added to all devplacepy source files
## 2026-06-10 🟢
- Port conflict detection and test isolation hardening across admin, avatar, bugs, landing, messages, and customization tests
- Container proxy routing via container IP instead of host port, with fake backend network simulation
- XDG-compliant devii tasks database path with DEVII_HOME override
- Project editing endpoint with 125k char body limit and remote URL attachment guard
## 2026-06-09 🔥 Big day!
- Container manager with Dockerfile CRUD, image builds, instance lifecycle, ingress proxy, and CLI commands
- Async project fork service with job queue, CLI management, and shared container image build
- Parallel test execution with per-worker isolated databases, data dirs, and uvicorn subprocesses via pytest-xdist
- Per-user customization suppression toggles with profile UI and Devii tool
- Customization toggle UI with enable/disable state management
- Unified shared Http and Poller utilities across all frontend modules, replacing inline fetch and setInterval patterns
- Responsive refinements for sub-360px screens, touch targets, safe-area insets, and mobile window controls
- Click-to-open profile dropdown with keyboard and outside-click dismissal
- Migrate hardcoded spacing values to CSS custom properties across multiple stylesheets
- Unicode escape normalization for emoji constants across codebase
- Consolidated upload ignore rules into a single directory-level gitignore entry
- Removed unused imports across routers, database, and services
- Remove project_set_private from confirmation-required actions and fix async test helpers
## 2026-06-08 🟢
- Admin AI quota management with CLI and admin panel reset controls
- API key management CLI with backfill command and auth support across session, API key, and HTTP Basic
- Per-project filesystem with directory and file CRUD, upload, and inline editing
- Async zip job framework with CLI management and zip archive download endpoints
- Add mistune dependency to project
## 2026-06-06 🟢
- Reactions, bookmarks, polls, extended sessions, and operational settings
## 2026-06-05 🔥 Massive day!
- Batch attachment linking, deduplicated mention notifications, and idempotent badge milestone checks
- Cursor-based load-more pagination across feed, gists, news and projects
- Canonical slug redirects, cursor-based next-page links, and OG image extraction across feed, gists, news, posts, projects, and profile
- TTLCache with LRU eviction, CLI role management, content unit helpers, database query functions, follow API with XP rewards, and news service with AI grading
- Unified comment form component with mobile touch optimizations across all CSS
- Inline comment previews on post cards with per-comment reply forms
- Comment template with threaded voting, author display, and attachment support
- Post-login redirect with `next` parameter and unauthenticated comment redirect to login
- Login redirect for unauthenticated admin, next parameter support with external URL rejection, and inline comment reply forms
- Seed comments created for all posts instead of only the first
- Replace uuid4 with uuid7 via uuid_utils for push notification JWT jti claims
- Coverage instrumentation for CI and local test runs with HTML report artifact
- Coverage configuration with subprocess measurement support
- Sitemap TTL configurable via environment variable and news_images schema migration
- Kill stale server process and add startup failure detection for Locust targets
## 2026-06-02 🟢
- Multi-worker service lock with cascading vote/comment cleanup on content deletion
## 2026-05-30 🟢
- Leaderboard route with gamification system (XP, levels, badges, stars) and content creation refactor
## 2026-05-28 🟢
- Cursor-based pagination for feed, notifications, and votes with thumbnail extension fallback
- Push registration returns creation flag and only sends welcome notification on first registration
## 2026-05-27 🟢
- AJAX vote buttons with live count updates across posts, gists, projects, and comments
- CSS-only card-link overlay replacing JS-driven data-href navigation
## 2026-05-25 🟢
- Unified notification click-to-navigate with comment anchor highlighting and dismiss refactor
## 2026-05-23 🔥 Massive day!
- Web push notifications with PWA manifest and service worker registration
- Web push notifications with PWA offline shell and install prompt
- Unified badge, notification, and content enrichment system with star tracking helpers
- Aggregate star counts across posts, projects and gists for profile and top-author ranking
- Content editing and deletion with cascading cleanup, avatar image helper, HTTP form POST, text input cursor management, and toast flash utility
- Share button with clipboard copy across detail pages, structured data schemas for gists and news articles, configurable site URL and rate limit, and production proxy headers support
- Production deployment workflow via git merge master into production
- Automatic production deployment on successful master push
- Removed automatic production deployment from CI pipeline
- Admin settings form with Pydantic validation and model-driven save
- Pydantic form models with validation for signup, login, password reset, comments, bugs, admin actions, and posts
- Type-safe integer settings with empty-value skip on admin save
- Input validation tests for votes, posts, profile, and signup endpoints
- Rate-limit environment variable and expanded Locust seed data for gists, notifications, and uploads
- TTLCache with ETag-based HTTP caching for avatar endpoint
- Dynamic language sidebar filtering based on existing gist language codes
- Vendor static assets for CodeMirror, highlight.js, marked, and emoji picker
- Test server log capture via tempfile with reduced log verbosity
- DOMPurify XSS sanitization for client-side rendered markdown content
- Add mobile-web-app-capable meta tag for PWA support
- Topnav notification bell selector scoped to /notifications href
- Fix notification bell icon locator to use explicit href selector instead of first match
- Remove stale import of get_users_by_uids from project_detail endpoint
## 2026-05-22 🟢
- News article HTML sanitization CLI command and database migration
## 2026-05-19 🟢
- Avatar generation exception logging with full traceback
- Fix multiavatar import path and add required arguments to function call
## 2026-05-16 🟢
- Clickable post titles and content with downvote support on feed and detail pages
- Interactive vote buttons and clickable post titles on profile page
- Handle @-mention with preceding text in content rendering
- Unread notification cache invalidation across comments, follows, messages, votes, and mentions
- Compact send button, attachment upload container, and auto-scroll on message thread load
- GistEditor lazy init with modal observer, CodeMirror Rust mode removed, emoji picker module type, source textarea required removed, projects tab spacing and settings button removed
- Add space between icon and label in feed navigation tabs
## 2026-05-15 🟢
- Python 3.13 base image, default port 10500, and nginx template to conf.d migration
- Responsive mobile navigation and messages layout with hamburger menu and back button
- Responsive breakpoint widened from 768px to 1024px for topnav, breadcrumb and page layouts
## 2026-05-14 🟢
- Migrate from deprecated `datetime.utcnow()` to timezone-aware `datetime.now(timezone.utc)` across the entire codebase
- Wait-for-url stabilization in noindex tests for messages and notifications pages
- Disable parallel test execution in CI pipeline
## 2026-05-13 🟢
- Migrate all TemplateResponse calls to pass request as first positional argument
- Attachment linking and deletion refactored into dedicated module with batch support
- Parallelised integration test suite with xdist worker port isolation
- Remove deprecated imghdr dependency and fix icon spacing in bug report buttons
- Replace hardcoded pytest.BASE_URL with conftest BASE_URL in attachment tests
- CI trigger branch from main to master
## 2026-05-12 🟢
- News service with admin curation, landing page articles, and comment support
- Mention notification system across bugs, comments, messages, posts, and projects with user search API
- Gists page with code snippet sharing, voting, and comment integration
## 2026-05-11 🔥 Big day!
- Unified threaded comment system with polymorphic target support across posts, projects, and bugs
- News management system with admin panel, pagination, and SEO sitemap integration
- News background service framework with CLI management, bug reports router, and admin services monitoring
- Admin panel with user management CLI, SEO metadata, and production deployment config
- Multiavatar local SVG generation with WAL mode SQLite and Locust load testing
- CI branch target renamed from main to master and test fixtures refactored for explicit login and seeded database
- Test fixture improvements with debug logging, stderr capture, and extended startup timeout
- Remove hawk static analysis step from CI test workflow
## 2026-05-10 🚀 First commit!
- Initial project scaffold with FastAPI SSR app, auth, feed, posts, comments, projects, profile, messages, notifications, and voting
- DiceBear avatar proxy with style picker on signup and profile, threaded comments
- Image upload support for posts with daily topic display on landing and feed
────────────────────────────────────────────────────────────
Summary: 194 commits over 29 active days. The project launched on May 10 with the initial FastAPI scaffold, auth, feed, and core content features. The biggest pushes came on June 13 (23 commits) delivering the three-tier test suite, soft-delete audit system, notification preferences, and author diversity enforcement; June 23 (23 commits) adding web push notifications, PWA support, content editing/deletion, and production deployment workflows; and June 14 (14 commits) introducing the admin database API, DeepSearch research system, SEO diagnostics, and the stealth HTTP client.

2
CLAUDE.md
View File

@ -159,7 +159,7 @@ Nested `CLAUDE.md` files (loaded automatically by Claude Code only when a file i
| `devplacepy/templates/CLAUDE.md` | Modal system, CDN libraries, shared template partials | | `devplacepy/templates/CLAUDE.md` | Modal system, CDN libraries, shared template partials |
| `tests/CLAUDE.md` | Detailed testing patterns and pitfalls | | `tests/CLAUDE.md` | Detailed testing patterns and pitfalls |
`isslop/` at the repo root is a separate, standalone sibling project (own `pyproject.toml`, `Makefile`, port 18732) with its own `isslop/CLAUDE.md` - it is not a nested subsystem of the `devplacepy` package. The integrated engine that DevPlace actually runs (`devplace isslop analyze`, the `/tools/isslop` job service) is a distinct implementation documented in `devplacepy/services/jobs/CLAUDE.md`. The AI usage analyzer (`devplace isslop analyze`, the `/tools/isslop` job service) lives entirely inside the package at `devplacepy/services/jobs/isslop/` and is documented in `devplacepy/services/jobs/CLAUDE.md`. There is no repo-root `isslop/` project.
## Architecture ## Architecture

348
devplacepy/database/remote.py
View File

@ -1,348 +0,0 @@
# retoor <retoor@molodetz.nl>
import inspect
import os
import httpx
from devplacepy.cache import TTLCache
from devplacepy_services.base.db_codec import (
decode_value,
encode_args,
is_write,
is_write_sql,
)
_SERVICE_URL = os.environ.get("DEVPLACE_DB_SERVICE_URL", "http://127.0.0.1:10601").rstrip("/")
_INTERNAL_KEY = os.environ.get("DEVPLACE_GATEWAY_INTERNAL_KEY", "").strip()
_CLIENT: httpx.Client | None = None
# Section 7.3: settings reads tolerate up to 5s staleness. patch_module()
# generically RPCs every devplacepy.database call, bypassing the local
# TTL cache get_setting/get_int_setting had in-process - without this,
# every settings read (rate limiting, maintenance mode, admin dashboards)
# pays a full HTTP round trip to the database broker.
_SETTINGS_CACHE_TTL_SECONDS = 5
_SETTINGS_CACHE = TTLCache(ttl=_SETTINGS_CACHE_TTL_SECONDS, max_size=512)
_CACHED_SETTINGS_FNS = frozenset({"get_setting", "get_int_setting"})
def _service_url() -> str:
return os.environ.get("DEVPLACE_DB_SERVICE_URL", _SERVICE_URL).rstrip("/")
def _headers() -> dict[str, str]:
headers: dict[str, str] = {}
key = os.environ.get("DEVPLACE_GATEWAY_INTERNAL_KEY", _INTERNAL_KEY).strip()
if key:
headers["X-Internal-Key"] = key
return headers
def _client() -> httpx.Client:
global _CLIENT
if _CLIENT is None:
_CLIENT = httpx.Client(timeout=30.0)
return _CLIENT
def _post(path: str, body: dict) -> object:
response = _client().post(
f"{_service_url()}/{path.lstrip('/')}",
json=body,
headers=_headers(),
)
if response.status_code >= 400:
payload = response.json() if response.content else {}
message = payload.get("error", "Database service request failed")
raise RuntimeError(message)
if not response.content:
return None
return decode_value(response.json())
def _invoke_cached(fn_name: str, args, kwargs):
cache_key = f"{fn_name}:{args!r}:{sorted(kwargs.items())!r}"
cached = _SETTINGS_CACHE.get(cache_key)
if cached is not None:
return cached
value = _invoke(fn_name, args, kwargs, write=False)
_SETTINGS_CACHE.set(cache_key, value)
return value
def _invoke(fn_name: str, args, kwargs, *, write: bool = False):
encoded_args, encoded_kwargs = encode_args(args, kwargs)
payload = {
"fn": fn_name,
"args": encoded_args,
"kwargs": encoded_kwargs,
"write": write,
}
result = _post("internal/invoke", payload)
if isinstance(result, dict) and "result" in result:
return result["result"]
return result
class RemoteSearchClause:
def __init__(self, term, fields, author_field=None):
self.term = term.strip()
self.fields = tuple(fields)
self.author_field = author_field
class RemoteUidInClause:
def __init__(self, field, uids):
self.field = field
self.uids = frozenset(uids)
class RemoteTable:
def __init__(self, db: "RemoteDb", name: str) -> None:
self._db = db
self._name = name
self._column_cache = None
def __getattr__(self, name: str):
def caller(*args, **kwargs):
return self._db._table_op(self._name, name, args, kwargs)
return caller
def has_column(self, name: str) -> bool:
cache = self._column_cache
if cache is None:
sample = self.find(_limit=1)
row = next(iter(sample), None)
cache = set(row.keys()) if row else set()
self._column_cache = cache
return name in cache
def count(self, **kwargs):
return self._db._table_op(self._name, "count", [], kwargs)
@property
def table(self):
return self
@property
def exists(self) -> bool:
return self._name in self._db.tables
class RemoteDb:
def __init__(self) -> None:
self._tables_cache: list[str] | None = None
@property
def tables(self) -> list[str]:
if self._tables_cache is None:
result = _post("internal/db-op", {"op": "tables"})
self._tables_cache = list(result or [])
return self._tables_cache
def __getitem__(self, name: str) -> RemoteTable:
return RemoteTable(self, name)
def query(self, sql: str, **params):
encoded_args, encoded_kwargs = encode_args((sql,), params)
result = _post(
"internal/db-op",
{
"op": "query",
"args": encoded_args,
"kwargs": encoded_kwargs,
"write": is_write_sql(sql),
},
)
return result or []
def _table_op(self, table: str, method: str, args, kwargs, *, write: bool = False):
encoded_args, encoded_kwargs = encode_args(args, kwargs)
result = _post(
"internal/db-op",
{
"op": "table_op",
"table": table,
"method": method,
"args": encoded_args,
"kwargs": encoded_kwargs,
"write": write,
},
)
if method in {"insert", "update", "delete"}:
self._tables_cache = None
return result
@property
def executable(self):
return self
@property
def in_transaction(self) -> bool:
return False
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
_LOCAL_REMOTE = frozenset(
{
"get_table",
"refresh_snapshot",
"_in_clause",
"_now_iso",
"text_search_clause",
}
)
def _remote_text_search_clause(
table, search, fields=("title", "description"), author_field=None
):
term = (search or "").strip()
if not term:
return None
if type(table).__name__ == "RemoteTable":
return RemoteSearchClause(term, fields, author_field)
from devplacepy.database.content import text_search_clause as local_clause
return local_clause(table, search, fields, author_field=author_field)
def _remote_get_table(name: str):
import devplacepy.database.core as core
return core.db[name]
def _remote_refresh_snapshot() -> None:
return None
def patch_module(module) -> None:
import devplacepy.database as db_module
for name in db_module.__all__:
if name in _LOCAL_REMOTE:
continue
target = getattr(module, name, None)
if target is None or not callable(target):
continue
if inspect.isclass(target):
continue
def make_wrapper(fn_name: str, fn_write: bool):
if fn_name in _CACHED_SETTINGS_FNS:
def wrapper(*args, **kwargs):
return _invoke_cached(fn_name, args, kwargs)
wrapper.__name__ = fn_name
return wrapper
def wrapper(*args, **kwargs):
return _invoke(fn_name, args, kwargs, write=fn_write)
wrapper.__name__ = fn_name
return wrapper
setattr(module, name, make_wrapper(name, is_write(name)))
def activate() -> None:
import devplacepy.database.core as core
core.db = RemoteDb()
import devplacepy.database as db_module
patch_module(db_module)
for submodule_name in (
"settings",
"users",
"relations",
"pagination",
"soft_delete",
"engagement",
"usage",
"awards",
"seo_meta",
"activity",
"customization",
"email",
"notifications",
"forks",
"follows",
"deepsearch",
"ranking",
"comments",
"content",
"attachments_data",
"stats",
"schema",
):
try:
submodule = __import__(
f"devplacepy.database.{submodule_name}",
fromlist=[submodule_name],
)
except ImportError:
continue
patch_module(submodule)
for external_name in (
"devplacepy.services.statistics.tracking",
"devplacepy.services.base",
"devplacepy.attachments",
"devplacepy.project_files",
):
try:
external = __import__(external_name, fromlist=[external_name.split(".")[-1]])
except ImportError:
continue
if hasattr(external, "db"):
external.db = RemoteDb()
db_module.db = core.db
db_module.get_table = _remote_get_table
core.get_table = _remote_get_table
db_module.refresh_snapshot = _remote_refresh_snapshot
core.refresh_snapshot = _remote_refresh_snapshot
db_module.text_search_clause = _remote_text_search_clause
import devplacepy.database.content as content_module
content_module.text_search_clause = _remote_text_search_clause
for submodule_name in (
"settings",
"users",
"relations",
"pagination",
"soft_delete",
"engagement",
"usage",
"awards",
"seo_meta",
"activity",
"customization",
"email",
"notifications",
"forks",
"follows",
"deepsearch",
"ranking",
"comments",
"content",
"attachments_data",
"stats",
"schema",
):
try:
submodule = __import__(
f"devplacepy.database.{submodule_name}",
fromlist=[submodule_name],
)
except ImportError:
continue
if hasattr(submodule, "db"):
submodule.db = core.db

29
devplacepy/db_client.py
View File

@ -1,29 +0,0 @@
# retoor <retoor@molodetz.nl>
import os
def _activate() -> None:
if os.environ.get("DEVPLACE_DB_SERVICE") == "1":
return
if os.environ.get("DEVPLACE_REMOTE_DB") == "1":
from devplacepy.database.remote import activate
activate()
_activate()
import devplacepy.database as _database
def _remote_table(table) -> bool:
return type(table).__name__ == "RemoteTable"
def __getattr__(name: str):
return getattr(_database, name)
def __dir__():
return sorted(name for name in dir(_database) if not name.startswith("_"))

2
devplacepy/services/audit/CLAUDE.md
View File

@ -2,7 +2,7 @@ This file documents the audit log subsystem. Claude Code auto-loads it when a fi
## Audit log (`services/audit/`) ## Audit log (`services/audit/`)
**Admin-only, append-only** record of every state-changing action. The authoritative event catalogue is `events.md` (its storage model, relation vocabulary, and per-event specs are authoritative); the catalogue currently spans 223 keys across 38 domains. **Admin-only, append-only** record of every state-changing action. The authoritative event catalogue is `events.md` (its storage model, relation vocabulary, and per-event specs are authoritative); the catalogue currently spans 288 keys across 42 domains.
### Package layout ### Package layout

1
devplacepy/services/containers/CLAUDE.md
View File

@ -120,6 +120,7 @@ The security hotpatch that used to run per build is now baked into `ppy.Dockerfi
- `COPY`s the **sudo superclone** (`files/sudo`) over `/usr/local/bin/sudo` (+ symlink `/usr/bin/sudo`; the real `sudo` package is not installed). - `COPY`s the **sudo superclone** (`files/sudo`) over `/usr/local/bin/sudo` (+ symlink `/usr/bin/sudo`; the real `sudo` package is not installed).
- `COPY`s the **`aptroot` fakeroot wrapper** (`files/aptroot`, symlinked over `apt`/`apt-get`/`dpkg` in `/usr/local/bin` so pravda installs system packages without root). - `COPY`s the **`aptroot` fakeroot wrapper** (`files/aptroot`, symlinked over `apt`/`apt-get`/`dpkg` in `/usr/local/bin` so pravda installs system packages without root).
- `COPY`s **`pagent`** (`files/pagent`, the stdlib AI agent; reads `DEVPLACE_OPENAI_URL`+`DEVPLACE_API_KEY`, falling back to its public endpoint + `DEEPSEEK_API_KEY`) to `/usr/bin/pagent.py`, plus `files/.vimrc` to `/home/pravda/.vimrc` (whose AI helper - `AiEditSelection` - targets the same gateway as pagent via `DEVPLACE_OPENAI_URL`/`DEVPLACE_API_KEY`, with a public fallback, never `api.openai.com`). - `COPY`s **`pagent`** (`files/pagent`, the stdlib AI agent; reads `DEVPLACE_OPENAI_URL`+`DEVPLACE_API_KEY`, falling back to its public endpoint + `DEEPSEEK_API_KEY`) to `/usr/bin/pagent.py`, plus `files/.vimrc` to `/home/pravda/.vimrc` (whose AI helper - `AiEditSelection` - targets the same gateway as pagent via `DEVPLACE_OPENAI_URL`/`DEVPLACE_API_KEY`, with a public fallback, never `api.openai.com`).
- `COPY`s **`dpc`** (`files/dpc`, DevPlace Code, the Claude-Code-class coding agent) to `/usr/bin/dpc`, and **`bot.py`** to `/usr/bin/botje.py`. **`dpc` is the one prebuilt binary in this repository**: a stripped ELF 64-bit x86-64 executable, 3578664 bytes, sha256 `24f7fbb068e8461e085c5d49ea92556afc10452e4610de7784b87650f96377dd`, linked against GCC (Ubuntu 15.2.0-4ubuntu4) 15.2. Its source is NOT in this repository and there is no build recipe here, so unlike every other file in `files/` it cannot be reviewed before it is installed root-owned onto `PATH` in every user container. Record a new checksum here whenever it is replaced.
- Evicts any pre-existing uid-1000 user, creates user **`pravda` at `1000:1000`**. - Evicts any pre-existing uid-1000 user, creates user **`pravda` at `1000:1000`**.
- Hands pravda ownership of the toolchain AND the OS package trees (`chown -R pravda` over `/usr/local/lib`, `/usr/local/bin`, `/usr/lib/python3`, `/opt`, `/app`, `/home/pravda`, plus `/usr/lib`, `/usr/bin`, `/usr/sbin`, `/usr/share`, `/usr/include`, `/etc`, `/var/lib`, `/var/cache`, `/var/log`, `/srv` so `apt`/`dpkg` can write; `~/.local/bin` on `PATH`). - Hands pravda ownership of the toolchain AND the OS package trees (`chown -R pravda` over `/usr/local/lib`, `/usr/local/bin`, `/usr/lib/python3`, `/opt`, `/app`, `/home/pravda`, plus `/usr/lib`, `/usr/bin`, `/usr/sbin`, `/usr/share`, `/usr/include`, `/etc`, `/var/lib`, `/var/cache`, `/var/log`, `/srv` so `apt`/`dpkg` can write; `~/.local/bin` on `PATH`).
- Ends on `USER pravda`. - Ends on `USER pravda`.

2
devplacepy/services/registry.py
View File

@ -42,7 +42,7 @@ def admin_shell_manager() -> ServiceManager:
Never supervised (no set_lock_owner/supervise call) - used only for Never supervised (no set_lock_owner/supervise call) - used only for
describe()/config metadata/key derivation, all of which read or write describe()/config metadata/key derivation, all of which read or write
through db_client to the shared service_state / site_settings tables. through the database layer to the shared service_state / site_settings tables.
The owning Tier 3 process is the only one that ever ticks these services. The owning Tier 3 process is the only one that ever ticks these services.
""" """
global _SHELL_MANAGER global _SHELL_MANAGER

1
devplacepy_services/__init__.py
View File

@ -1 +0,0 @@
# retoor <retoor@molodetz.nl>

1
devplacepy_services/audit/SERVICE.md
View File

@ -1 +0,0 @@
# Audit Service (stub)

0
devplacepy_services/audit/__init__.py
View File

23
devplacepy_services/audit/main.py
View File

@ -1,23 +0,0 @@
import devplacepy_services.base.bootstrap
from devplacepy.services.audit import AuditService
from devplacepy_services.base.service import BaseMicroservice, build_standard_app
class AuditMicroservice(BaseMicroservice):
name = "audit"
title = "Audit"
default_port = 10639
workers = 1
stateful = True
depends_on = ["database"]
managed_services = [AuditService()]
use_background = True
run_supervisor = True
def build_app(self):
return build_standard_app(self)
_service = AuditMicroservice()
app = _service.build_app()

1
devplacepy_services/backup/SERVICE.md
View File

@ -1 +0,0 @@
# Backup Service (stub)

0
devplacepy_services/backup/__init__.py
View File

23
devplacepy_services/backup/main.py
View File

@ -1,23 +0,0 @@
import devplacepy_services.base.bootstrap
from devplacepy.services.backup.service import BackupService
from devplacepy_services.base.service import BaseMicroservice, build_standard_app
class BackupMicroservice(BaseMicroservice):
name = "backup"
title = "Backup"
default_port = 10633
workers = 1
stateful = True
depends_on = ["database"]
managed_services = [BackupService()]
use_background = True
run_supervisor = True
def build_app(self):
return build_standard_app(self)
_service = BackupMicroservice()
app = _service.build_app()

5
devplacepy_services/base/__init__.py
View File

@ -1,5 +0,0 @@
# retoor <retoor@molodetz.nl>
from devplacepy_services.base.service import BaseMicroservice
__all__ = ["BaseMicroservice"]

61
devplacepy_services/base/auth.py
View File

@ -1,61 +0,0 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
import os
from typing import Annotated
from fastapi import Header, Request
from devplacepy_services.base.errors import http_error
from devplacepy_services.base.schemas import InternalUser
_KEY_CACHE = ""
def internal_gateway_key() -> str:
global _KEY_CACHE
if _KEY_CACHE:
return _KEY_CACHE
env_key = os.environ.get("DEVPLACE_GATEWAY_INTERNAL_KEY", "").strip()
if env_key:
_KEY_CACHE = env_key
return env_key
try:
from devplacepy.db_client import internal_gateway_key as load_key
loaded = load_key().strip()
if loaded:
_KEY_CACHE = loaded
return loaded
except ImportError:
return ""
def set_internal_gateway_key(value: str) -> None:
global _KEY_CACHE
_KEY_CACHE = value.strip()
def validate_internal_key(presented: str | None) -> bool:
expected = internal_gateway_key()
if not expected:
return True
if not presented:
return False
return presented.strip() == expected
def require_internal_key(request: Request) -> None:
if validate_internal_key(request.headers.get("X-Internal-Key")):
return
raise http_error(401, "Unauthorized", "unauthorized")
async def internal_user(
request: Request,
x_authenticated_user: Annotated[str | None, Header()] = None,
) -> InternalUser:
require_internal_key(request)
uid = x_authenticated_user.strip() if x_authenticated_user else None
return InternalUser(uid=uid or None)

4
devplacepy_services/base/bootstrap.py
View File

@ -1,4 +0,0 @@
import os
os.environ["DEVPLACE_REMOTE_DB"] = "1"
import devplacepy.db_client

67
devplacepy_services/base/cache.py
View File

@ -1,67 +0,0 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
import time
from collections import OrderedDict
class TTLCache:
def __init__(self, ttl: int, max_size: int = 0):
self.ttl = ttl
self.max_size = max_size
self._store: OrderedDict[str, tuple[object, float]] = OrderedDict()
def get(self, key: str):
entry = self._store.get(key)
if entry is None:
return None
value, expiry = entry
if time.time() >= expiry:
self._store.pop(key, None)
return None
self._store.move_to_end(key)
return value
def set(self, key: str, value) -> None:
self._store[key] = (value, time.time() + self.ttl)
self._store.move_to_end(key)
if self.max_size and len(self._store) > self.max_size:
self._store.popitem(last=False)
def pop(self, key: str) -> None:
self._store.pop(key, None)
def clear(self) -> None:
self._store.clear()
class CacheStateGate:
def __init__(self, *, ttl: int = 60, max_staleness: float = 5.0):
self.ttl = ttl
self.max_staleness = max_staleness
self._cache = TTLCache(ttl=ttl)
self._versions: dict[str, int] = {}
self._bumped_at: dict[str, float] = {}
def get_version(self, name: str) -> int:
return int(self._versions.get(name, 0))
def bump(self, name: str) -> int:
next_version = self.get_version(name) + 1
self._versions[name] = next_version
self._bumped_at[name] = time.time()
self._cache.clear()
return next_version
def get(self, key: str, version_name: str, loader):
bumped_at = self._bumped_at.get(version_name, 0.0)
if bumped_at and (time.time() - bumped_at) < self.max_staleness:
self._cache.pop(key)
version_key = f"{version_name}:{self.get_version(version_name)}"
cached = self._cache.get(version_key)
if cached is not None:
return cached
value = loader()
self._cache.set(version_key, value)
return value

153
devplacepy_services/base/compound.py
View File

@ -1,153 +0,0 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
from typing import Any
from pydantic import BaseModel, Field
class InvokeIn(BaseModel):
fn: str
args: list[Any] = Field(default_factory=list)
kwargs: dict[str, Any] = Field(default_factory=dict)
write: bool = False
class CacheBumpIn(BaseModel):
name: str
class UsersByUidsIn(BaseModel):
uids: list[str]
class CommentCountsIn(BaseModel):
post_uids: list[str]
class VoteCountsIn(BaseModel):
target_uids: list[str]
class ReactionsIn(BaseModel):
target_type: str
target_uids: list[str]
user: dict[str, Any] | None = None
class BookmarksIn(BaseModel):
user_uid: str
target_type: str
target_uids: list[str]
class PollsIn(BaseModel):
post_uids: list[str]
user: dict[str, Any] | None = None
class RecentCommentsIn(BaseModel):
post_uids: list[str]
limit: int = 3
user: dict[str, Any] | None = None
class CommentsIn(BaseModel):
target_type: str
target_uid: str
user: dict[str, Any] | None = None
class FollowBundleIn(BaseModel):
user_uid: str
target_uids: list[str] | None = None
class RelationsIn(BaseModel):
viewer_uid: str | None = None
class OnlineUsersIn(BaseModel):
cutoff_iso: str
limit: int = 30
class NotificationPrefsIn(BaseModel):
user_uid: str
class LeaderboardBundleIn(BaseModel):
limit: int = 50
offset: int = 0
viewer_uid: str | None = None
class SiteSidebarIn(BaseModel):
authors_limit: int = 5
class AwardsBundleIn(BaseModel):
receiver_uid: str
page: int = 1
per_page: int = 12
profile_user: dict[str, Any] | None = None
class AttachmentsIn(BaseModel):
resource_type: str
resource_uids: list[str]
class UserMediaIn(BaseModel):
user_uid: str
page: int = 1
per_page: int = 24
class SeoMetaIn(BaseModel):
target_type: str
uids: list[str]
class FeedPageIn(BaseModel):
user: dict[str, Any] | None = None
tab: str = "all"
topic: str | None = None
search: str = ""
before: str | None = None
class PostDetailIn(BaseModel):
post_uid: str
user: dict[str, Any] | None = None
class ProfileBundleIn(BaseModel):
profile_uid: str
viewer: dict[str, Any] | None = None
class MessagesPageIn(BaseModel):
user_uid: str
class NotificationsPageIn(BaseModel):
user_uid: str
class LeaderboardPageIn(BaseModel):
viewer_uid: str | None = None
class ProjectDetailIn(BaseModel):
project_uid: str
class GistDetailIn(BaseModel):
gist_uid: str
user: dict[str, Any] | None = None
class GameStateIn(BaseModel):
user_uid: str | None = None

70
devplacepy_services/base/config.py
View File

@ -1,70 +0,0 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
import os
from pathlib import Path
from devplacepy_services.base.manifest import (
FORBIDDEN_GAPS,
PORT_HINTS,
SERVICE_URL_ENV,
SERVICES,
build_profiles,
)
PROFILES = build_profiles()
_BROKER_URL_ENV = {
"main": "DEVPLACE_DB_SERVICE_URL",
"devii_tasks": "DEVPLACE_DEVII_TASKS_STORE_URL",
"devii_lessons": "DEVPLACE_DEVII_LESSONS_STORE_URL",
}
def port_profile() -> str:
return os.environ.get("DEVPLACE_PORT_PROFILE", "micro-dev")
def data_dir() -> Path:
root = Path(__file__).resolve().parents[2]
return Path(os.environ.get("DEVPLACE_DATA_DIR", str(root / "data")))
def service_url(name: str) -> str:
env_key = SERVICE_URL_ENV.get(name)
if env_key:
override = os.environ.get(env_key, "").strip().rstrip("/")
if override:
return override
profile = port_profile()
host = os.environ.get("DEVPLACE_SERVICE_HOST", "127.0.0.1")
port = PROFILES[profile][name]
return f"http://{host}:{port}"
def broker_url(name: str) -> str:
env_key = _BROKER_URL_ENV.get(name)
if env_key:
override = os.environ.get(env_key, "").strip().rstrip("/")
if override:
return override
if name == "main":
return service_url("database")
if name == "devii_tasks":
return f"{service_url('devii')}/internal/store/devii_tasks"
if name == "devii_lessons":
return f"{service_url('devii')}/internal/store/devii_lessons"
raise KeyError(name)
def sqlite_read_pool_size() -> int:
raw = os.environ.get("DEVPLACE_SQLITE_READ_POOL", "8").strip()
try:
return max(1, int(raw))
except ValueError:
return 8
def service_spec(name: str):
return SERVICES[name]

236
devplacepy_services/base/db_client.py
View File

@ -1,236 +0,0 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
from typing import Any
from devplacepy_services.base.compound import (
AttachmentsIn,
AwardsBundleIn,
BookmarksIn,
CacheBumpIn,
CommentCountsIn,
CommentsIn,
FeedPageIn,
FollowBundleIn,
LeaderboardBundleIn,
NotificationPrefsIn,
OnlineUsersIn,
PollsIn,
ReactionsIn,
RecentCommentsIn,
RelationsIn,
SeoMetaIn,
SiteSidebarIn,
UserMediaIn,
UsersByUidsIn,
VoteCountsIn,
)
from devplacepy_services.base.errors import http_error
from devplacepy_services.base.http import internal_request
class BrokerClient:
def __init__(self, name: str, url: str | None = None) -> None:
self.name = name
if url is not None:
self.url = url.rstrip("/")
else:
from devplacepy_services.base.config import broker_url
self.url = broker_url(name).rstrip("/")
async def post(self, path: str, body: dict[str, Any] | None = None) -> Any:
target = f"{self.url}/{path.lstrip('/')}"
response = await internal_request("POST", target, json=body or {})
if response.status_code >= 400:
payload = response.json() if response.content else {}
message = payload.get("error", "Broker request failed")
code = payload.get("code", "broker_error")
raise http_error(response.status_code, message, code)
if not response.content:
return None
return response.json()
async def get(self, path: str, *, params: dict[str, Any] | None = None) -> Any:
target = f"{self.url}/{path.lstrip('/')}"
response = await internal_request("GET", target, params=params)
if response.status_code >= 400:
payload = response.json() if response.content else {}
message = payload.get("error", "Broker request failed")
code = payload.get("code", "broker_error")
raise http_error(response.status_code, message, code)
if not response.content:
return None
return response.json()
main = BrokerClient("main")
devii_tasks = BrokerClient("devii_tasks")
devii_lessons = BrokerClient("devii_lessons")
async def get_setting(key: str, default: str = "") -> str:
payload = await main.get(f"settings/{key}", params={"default": default})
return payload.get("value", default) if isinstance(payload, dict) else default
async def get_int_setting(key: str, default: int) -> int:
raw = await get_setting(key, str(default))
try:
return int(raw)
except (TypeError, ValueError):
return default
async def bump_cache_version(name: str) -> None:
await main.post("/cache/bump", CacheBumpIn(name=name).model_dump())
async def get_users_by_uids(uids: list[str]) -> dict:
return await main.post("/compound/users-by-uids", UsersByUidsIn(uids=uids).model_dump())
async def get_comment_counts_by_post_uids(post_uids: list[str]) -> dict:
return await main.post(
"/compound/comment-counts", CommentCountsIn(post_uids=post_uids).model_dump()
)
async def get_vote_counts(target_uids: list[str]) -> tuple[dict, dict]:
payload = await main.post(
"/compound/vote-counts", VoteCountsIn(target_uids=target_uids).model_dump()
)
return payload.get("ups", {}), payload.get("downs", {})
async def get_reactions_by_targets(
target_type: str, target_uids: list[str], user: dict | None = None
) -> dict:
return await main.post(
"/compound/reactions",
ReactionsIn(target_type=target_type, target_uids=target_uids, user=user).model_dump(),
)
async def get_user_bookmarks(
user_uid: str, target_type: str, target_uids: list[str]
) -> set[str]:
payload = await main.post(
"/compound/bookmarks",
BookmarksIn(
user_uid=user_uid, target_type=target_type, target_uids=target_uids
).model_dump(),
)
return set(payload or [])
async def get_polls_by_post_uids(post_uids: list[str], user: dict | None = None) -> dict:
return await main.post(
"/compound/polls", PollsIn(post_uids=post_uids, user=user).model_dump()
)
async def get_recent_comments_by_post_uids(
post_uids: list[str], limit: int = 3, user: dict | None = None
) -> dict:
return await main.post(
"/compound/recent-comments",
RecentCommentsIn(post_uids=post_uids, limit=limit, user=user).model_dump(),
)
async def load_comments(
target_type: str, target_uid: str, user: dict | None = None
) -> list:
return await main.post(
"/compound/comments",
CommentsIn(target_type=target_type, target_uid=target_uid, user=user).model_dump(),
)
async def get_follow_bundle(
user_uid: str, target_uids: list[str] | None = None
) -> dict:
return await main.post(
"/compound/follow-bundle",
FollowBundleIn(user_uid=user_uid, target_uids=target_uids).model_dump(),
)
async def get_user_relations_bundle(viewer_uid: str | None = None) -> dict:
return await main.post(
"/compound/relations", RelationsIn(viewer_uid=viewer_uid).model_dump()
)
async def get_online_users(cutoff_iso: str, limit: int = 30) -> list:
return await main.post(
"/compound/online-users",
OnlineUsersIn(cutoff_iso=cutoff_iso, limit=limit).model_dump(),
)
async def get_notification_prefs(user_uid: str) -> list:
return await main.post(
"/compound/notification-prefs",
NotificationPrefsIn(user_uid=user_uid).model_dump(),
)
async def get_leaderboard_bundle(
limit: int = 50, offset: int = 0, viewer_uid: str | None = None
) -> dict:
return await main.post(
"/compound/leaderboard",
LeaderboardBundleIn(limit=limit, offset=offset, viewer_uid=viewer_uid).model_dump(),
)
async def get_site_sidebar(authors_limit: int = 5) -> dict:
return await main.post(
"/compound/site-sidebar", SiteSidebarIn(authors_limit=authors_limit).model_dump()
)
async def get_awards_bundle(
receiver_uid: str,
page: int = 1,
per_page: int = 12,
profile_user: dict | None = None,
) -> dict:
return await main.post(
"/compound/awards",
AwardsBundleIn(
receiver_uid=receiver_uid,
page=page,
per_page=per_page,
profile_user=profile_user,
).model_dump(),
)
async def get_attachments_batch(resource_type: str, resource_uids: list[str]) -> dict:
return await main.post(
"/compound/attachments",
AttachmentsIn(resource_type=resource_type, resource_uids=resource_uids).model_dump(),
)
async def get_user_media(
user_uid: str, page: int = 1, per_page: int = 24
) -> dict:
return await main.post(
"/compound/user-media",
UserMediaIn(user_uid=user_uid, page=page, per_page=per_page).model_dump(),
)
async def get_seo_metadata_batch(target_type: str, uids: list[str]) -> dict:
return await main.post(
"/compound/seo-meta", SeoMetaIn(target_type=target_type, uids=uids).model_dump()
)
async def build_feed_page(**kwargs) -> dict:
return await main.post("/compound/feed-page", FeedPageIn(**kwargs).model_dump())

126
devplacepy_services/base/db_codec.py
View File

@ -1,126 +0,0 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
from datetime import date, datetime
from pathlib import Path
from typing import Any
WRITE_PREFIXES = (
"set_",
"add_",
"delete_",
"create_",
"update_",
"upsert_",
"record_",
"migrate_",
"backfill_",
"mark_",
"invalidate_",
"ensure_",
"restore",
"purge",
"revoke_",
"recompute_",
"soft_delete",
"bump_",
"init_",
"clear_settings",
)
WRITE_EXACT = frozenset(
{
"soft_delete_in",
"restore_event",
"purge_event",
"delete_engagement",
"soft_delete_engagement",
"delete_fork_relations",
"soft_delete_fork_relations",
"delete_attachments",
"delete_attachment_record",
"_delete_attachment_file",
"_index",
"_drop_index",
"_uid_index",
"_ensure_cache_state",
"_refresh_query_planner_stats",
"_backfill_gamification",
}
)
REMOTE_TABLE_MARKER = "__remote_table__"
def is_write(name: str) -> bool:
if name in WRITE_EXACT:
return True
return any(name.startswith(prefix) for prefix in WRITE_PREFIXES)
SQL_WRITE_KEYWORDS = frozenset(
{"INSERT", "UPDATE", "DELETE", "REPLACE", "CREATE", "ALTER", "DROP"}
)
def is_write_sql(sql: str) -> bool:
first_word = sql.strip().split(None, 1)[0].upper() if sql.strip() else ""
return first_word in SQL_WRITE_KEYWORDS
def encode_value(value: Any) -> Any:
if value is None or isinstance(value, (bool, int, float, str)):
return value
if type(value).__name__ == "RemoteTable":
return {REMOTE_TABLE_MARKER: value._name}
if isinstance(value, (datetime, date)):
return value.isoformat()
if isinstance(value, Path):
return str(value)
if isinstance(value, frozenset):
return [encode_value(item) for item in value]
if isinstance(value, set):
return [encode_value(item) for item in value]
if isinstance(value, tuple):
return [encode_value(item) for item in value]
if isinstance(value, list):
return [encode_value(item) for item in value]
if isinstance(value, dict):
return {str(key): encode_value(item) for key, item in value.items()}
if hasattr(value, "items") and callable(value.items):
try:
return {str(key): encode_value(item) for key, item in value.items()}
except TypeError:
pass
return str(value)
def encode_args(args: tuple | list, kwargs: dict) -> tuple[list, dict]:
return [encode_value(item) for item in args], {
str(key): encode_value(value) for key, value in kwargs.items()
}
def decode_value(value: Any) -> Any:
if isinstance(value, list):
return [decode_value(item) for item in value]
if isinstance(value, dict):
return {key: decode_value(item) for key, item in value.items()}
return value
def decode_arg(value: Any) -> Any:
if isinstance(value, list):
return [decode_arg(item) for item in value]
if isinstance(value, dict):
if set(value) == {REMOTE_TABLE_MARKER}:
from devplacepy.db_client import get_table
return get_table(value[REMOTE_TABLE_MARKER])
return {key: decode_arg(item) for key, item in value.items()}
return value
def encode_result(value: Any) -> Any:
return encode_value(value)

63
devplacepy_services/base/errors.py
View File

@ -1,63 +0,0 @@
# retoor <retoor@molodetz.nl>
from fastapi import HTTPException
from fastapi.responses import JSONResponse
from devplacepy_services.base.schemas import ErrorOut
def error_response(status_code: int, message: str, code: str) -> JSONResponse:
return JSONResponse(
ErrorOut(error=message, code=code).model_dump(),
status_code=status_code,
)
def http_error(status_code: int, message: str, code: str) -> HTTPException:
return HTTPException(
status_code=status_code,
detail=ErrorOut(error=message, code=code).model_dump(),
)
def error_code_for_status(status_code: int) -> str:
if status_code == 404:
return "not_found"
if status_code == 401:
return "unauthorized"
if status_code == 403:
return "forbidden"
if status_code == 422:
return "validation_error"
if status_code >= 500:
return "internal_error"
return "error"
def error_message_for_status(status_code: int, detail: str | None = None) -> str:
if status_code == 404:
return "Not found"
if status_code == 401:
return "Unauthorized"
if status_code == 403:
return "Forbidden"
if status_code >= 500:
return "Internal server error"
if detail:
return detail
return "Request failed"
def sanitize_detail(detail: object, status_code: int = 400) -> tuple[str, str]:
if isinstance(detail, dict):
if "error" in detail and "code" in detail:
return str(detail["error"]), str(detail["code"])
if "message" in detail:
return str(detail["message"]), error_code_for_status(status_code)
if isinstance(detail, list):
return "Validation failed", "validation_error"
if isinstance(detail, str):
return error_message_for_status(status_code, detail), error_code_for_status(status_code)
if detail is None:
return error_message_for_status(status_code), error_code_for_status(status_code)
return error_message_for_status(status_code, str(detail)), error_code_for_status(status_code)

57
devplacepy_services/base/health.py
View File

@ -1,57 +0,0 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
import asyncio
import time
from fastapi import APIRouter
from devplacepy_services.base.config import service_url
from devplacepy_services.base.http import internal_request
from devplacepy_services.base.schemas import HealthOut, HealthStatsOut
from devplacepy_services.base.service import BaseMicroservice
from devplacepy_services.base.stats import COUNTERS
async def _dep_status(name: str) -> str:
url = f"{service_url(name)}/health"
try:
response = await internal_request("GET", url, timeout=2.0)
if response.status_code == 200:
payload = response.json()
if payload.get("status") == "ok":
return "ok"
return "degraded"
return "down"
except Exception:
return "down"
def health_router(service: BaseMicroservice) -> APIRouter:
router = APIRouter()
@router.get("/health", response_model=HealthOut)
async def health() -> HealthOut:
deps: dict[str, str] = {}
if service.depends_on:
results = await asyncio.gather(
*[_dep_status(dep) for dep in service.depends_on]
)
for dep_name, status in zip(service.depends_on, results, strict=True):
deps[dep_name] = status
overall = "ok"
if deps and any(status != "ok" for status in deps.values()):
overall = "degraded"
uptime_s = int(time.monotonic() - service.started_at)
stats = COUNTERS.snapshot()
return HealthOut(
service=service.name,
status=overall,
uptime_s=uptime_s,
version=service.version,
deps=deps,
stats=HealthStatsOut(requests=stats["requests"], errors=stats["errors"]),
)
return router

77
devplacepy_services/base/http.py
View File

@ -1,77 +0,0 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
import contextvars
import os
from typing import Any
import httpx
from devplacepy import stealth
INTERNAL_CALLS: contextvars.ContextVar[int] = contextvars.ContextVar(
"internal_calls", default=0
)
REQUEST_ID: contextvars.ContextVar[str] = contextvars.ContextVar("request_id", default="")
_POOL: httpx.AsyncClient | None = None
def bump_internal_calls() -> None:
INTERNAL_CALLS.set(INTERNAL_CALLS.get() + 1)
def current_internal_calls() -> int:
return INTERNAL_CALLS.get()
def current_request_id() -> str:
return REQUEST_ID.get()
async def startup_pool() -> None:
global _POOL
if _POOL is None:
_POOL = httpx.AsyncClient(
limits=httpx.Limits(max_connections=32, max_keepalive_connections=32),
timeout=httpx.Timeout(30.0),
http2=False,
)
async def shutdown_pool() -> None:
global _POOL
if _POOL is not None:
await _POOL.aclose()
_POOL = None
def stealth_async_client(**kwargs: Any) -> httpx.AsyncClient:
return stealth.stealth_async_client(**kwargs)
async def internal_request(
method: str,
url: str,
*,
json: dict | None = None,
params: dict | None = None,
headers: dict[str, str] | None = None,
timeout: float = 30.0,
) -> httpx.Response:
await startup_pool()
assert _POOL is not None
bump_internal_calls()
merged: dict[str, str] = {}
request_id = current_request_id()
if request_id:
merged["X-Request-Id"] = request_id
internal_key = os.environ.get("DEVPLACE_GATEWAY_INTERNAL_KEY", "").strip()
if internal_key:
merged["X-Internal-Key"] = internal_key
if headers:
merged.update(headers)
return await _POOL.request(
method, url, json=json, params=params, headers=merged, timeout=timeout
)

215
devplacepy_services/base/manifest.py
View File

@ -1,215 +0,0 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
from dataclasses import dataclass
from typing import Literal
WorkersSpec = int | Literal["auto"]
@dataclass(frozen=True)
class ServiceSpec:
name: str
module: str
port: int
tier: int
workers: WorkersSpec
stateful: bool
depends_on: tuple[str, ...]
health_client_timeout: float = 2.0
@dataclass(frozen=True)
class IngressRoute:
prefix: str
service: str
websocket: bool = False
SERVICE_SPECS: tuple[ServiceSpec, ...] = (
ServiceSpec("database", "devplacepy_services.database.main:app", 10601, 1, 1, True, ()),
ServiceSpec("pubsub", "devplacepy_services.pubsub.main:app", 10602, 1, 1, True, ("database",)),
ServiceSpec(
"web",
"devplacepy_services.web.main:app",
10500,
2,
"auto",
False,
("database", "pubsub"),
health_client_timeout=4.0,
),
ServiceSpec(
"gateway",
"devplacepy_services.gateway.main:app",
10620,
2,
"auto",
False,
("database",),
health_client_timeout=4.0,
),
ServiceSpec(
"jobs",
"devplacepy_services.jobs.main:app",
10630,
3,
1,
True,
("database", "pubsub", "gateway"),
health_client_timeout=8.0,
),
ServiceSpec(
"devii",
"devplacepy_services.devii.main:app",
10631,
3,
2,
True,
("database", "pubsub", "gateway"),
health_client_timeout=4.0,
),
ServiceSpec("bot", "devplacepy_services.bot.main:app", 10632, 3, 1, True, ("database", "gateway")),
ServiceSpec("backup", "devplacepy_services.backup.main:app", 10633, 3, 1, True, ("database",)),
ServiceSpec(
"containers",
"devplacepy_services.containers.main:app",
10634,
3,
1,
True,
("database",),
health_client_timeout=4.0,
),
ServiceSpec("telegram", "devplacepy_services.telegram.main:app", 10635, 3, 1, True, ("database", "pubsub")),
ServiceSpec("email", "devplacepy_services.email.main:app", 10636, 3, 1, True, ("database",)),
ServiceSpec(
"news",
"devplacepy_services.news.main:app",
10637,
3,
1,
True,
("database", "gateway"),
),
ServiceSpec("gitea", "devplacepy_services.gitea.main:app", 10638, 3, 1, True, ("database", "gateway")),
ServiceSpec("audit", "devplacepy_services.audit.main:app", 10639, 3, 1, True, ("database",)),
ServiceSpec("xmlrpc", "devplacepy_services.xmlrpc.main:app", 10649, 3, 1, True, ("database",)),
)
SERVICES: dict[str, ServiceSpec] = {spec.name: spec for spec in SERVICE_SPECS}
TIER_ORDER: dict[int, tuple[str, ...]] = {
1: ("database", "pubsub"),
2: ("web", "gateway"),
3: (
"jobs",
"devii",
"bot",
"backup",
"containers",
"telegram",
"email",
"news",
"gitea",
"audit",
"xmlrpc",
),
}
SERVICE_URL_ENV: dict[str, str] = {
"web": "DEVPLACE_WEB_URL",
"database": "DEVPLACE_DB_SERVICE_URL",
"pubsub": "DEVPLACE_PUBSUB_URL",
"gateway": "DEVPLACE_GATEWAY_URL",
"jobs": "DEVPLACE_JOBS_URL",
"devii": "DEVPLACE_DEVII_URL",
"bot": "DEVPLACE_BOT_URL",
"backup": "DEVPLACE_BACKUP_URL",
"containers": "DEVPLACE_CONTAINERS_URL",
"telegram": "DEVPLACE_TELEGRAM_URL",
"email": "DEVPLACE_EMAIL_URL",
"news": "DEVPLACE_NEWS_URL",
"gitea": "DEVPLACE_GITEA_URL",
"audit": "DEVPLACE_AUDIT_URL",
"xmlrpc": "DEVPLACE_XMLRPC_URL",
}
INGRESS_ROUTES: tuple[IngressRoute, ...] = (
IngressRoute("/openai", "gateway"),
IngressRoute("/devii", "devii", websocket=True),
IngressRoute("/pubsub", "pubsub", websocket=True),
IngressRoute("/zips", "jobs"),
IngressRoute("/forks", "jobs"),
IngressRoute("/tools", "jobs", websocket=True),
IngressRoute("/xmlrpc", "xmlrpc"),
)
INGRESS_WS_PREFIXES: tuple[tuple[str, str], ...] = (
(
"/projects/",
"containers",
),
)
FORBIDDEN_GAPS: tuple[tuple[int, int], ...] = (
(10520, 10599),
(10610, 10619),
(10650, 10699),
(10750, 10799),
(20550, 20599),
(20650, 20699),
)
PORT_HINTS: dict[int, str] = {
10500: "web service",
20500: "test orchestrator",
10502: "locust",
}
_MICRO_DEV_PORTS: dict[str, int] = {spec.name: spec.port for spec in SERVICE_SPECS}
XMLRPC_RAW_SOCKET_PORT_DEV = 10648
def xmlrpc_raw_socket_port(profile: str) -> int:
if profile == "micro-test":
return XMLRPC_RAW_SOCKET_PORT_DEV + 10000
if profile == "micro-prod":
return XMLRPC_RAW_SOCKET_PORT_DEV + 100
return XMLRPC_RAW_SOCKET_PORT_DEV
def build_profiles() -> dict[str, dict[str, int]]:
return {
"micro-dev": dict(_MICRO_DEV_PORTS),
"micro-test": {
k: (20500 if k == "web" else v + 10000) for k, v in _MICRO_DEV_PORTS.items()
},
"micro-prod": {
k: (10500 if k == "web" else v + 100) for k, v in _MICRO_DEV_PORTS.items()
},
"locust": {"app": 10502, "ui": 10503},
}
def orchestrator_services_dict() -> dict[str, dict]:
return {
spec.name: {
"module": spec.module,
"port": spec.port,
"tier": spec.tier,
"workers": spec.workers,
"stateful": spec.stateful,
"depends_on": list(spec.depends_on),
}
for spec in SERVICE_SPECS
}
def health_client_timeout(name: str) -> float:
spec = SERVICES.get(name)
if spec is None:
return 2.0
return spec.health_client_timeout

121
devplacepy_services/base/middleware.py
View File

@ -1,121 +0,0 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
import json
import logging
import time
import uuid_utils
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.requests import Request
from starlette.responses import Response
from devplacepy_services.base.auth import validate_internal_key
from devplacepy_services.base.errors import error_response, sanitize_detail
from devplacepy_services.base.http import INTERNAL_CALLS, REQUEST_ID, current_internal_calls
from devplacepy_services.base.stats import COUNTERS
logger = logging.getLogger(__name__)
PUBLIC_PATHS = {"/health"}
class RequestStatsMiddleware(BaseHTTPMiddleware):
async def dispatch(self, request: Request, call_next):
COUNTERS.record_request()
response = await call_next(request)
if response.status_code >= 500:
COUNTERS.record_error()
return response
class InternalCallCounterMiddleware(BaseHTTPMiddleware):
async def dispatch(self, request: Request, call_next):
token_calls = INTERNAL_CALLS.set(0)
response = await call_next(request)
service_name = getattr(request.app.state, "service_name", "")
if service_name in {"web", "gateway"}:
response.headers["X-Internal-Calls"] = str(current_internal_calls())
INTERNAL_CALLS.reset(token_calls)
return response
class InternalAuthMiddleware(BaseHTTPMiddleware):
async def dispatch(self, request: Request, call_next):
path = request.url.path
if path in PUBLIC_PATHS:
return await call_next(request)
if not validate_internal_key(request.headers.get("X-Internal-Key")):
return error_response(401, "Unauthorized", "unauthorized")
return await call_next(request)
class SanitizeErrorsMiddleware(BaseHTTPMiddleware):
async def dispatch(self, request: Request, call_next):
request_id = request.headers.get("X-Request-Id") or uuid_utils.uuid7().hex
token_id = REQUEST_ID.set(request_id)
started = time.perf_counter()
try:
response = await call_next(request)
except Exception:
COUNTERS.record_error()
logger.exception(
"unhandled service error",
extra={
"request_id": request_id,
"service": getattr(request.app.state, "service_name", ""),
"path": request.url.path,
},
)
response = error_response(500, "Internal server error", "internal_error")
finally:
REQUEST_ID.reset(token_id)
duration_ms = int((time.perf_counter() - started) * 1000)
service_name = getattr(request.app.state, "service_name", "")
logger.info(
"request",
extra={
"request_id": request_id,
"service": service_name,
"path": request.url.path,
"status": response.status_code,
"duration_ms": duration_ms,
"internal_calls": current_internal_calls(),
},
)
if response.status_code < 400:
return response
if not hasattr(response, "body_iterator"):
return response
content_type = response.headers.get("content-type", "")
if "application/json" not in content_type:
if response.status_code == 404:
return error_response(404, "Not found", "not_found")
if response.status_code == 401:
return error_response(401, "Unauthorized", "unauthorized")
if response.status_code >= 500:
return error_response(500, "Internal server error", "internal_error")
return error_response(response.status_code, "Request failed", "error")
body = b""
async for chunk in response.body_iterator:
body += chunk
try:
parsed = json.loads(body)
except json.JSONDecodeError:
return error_response(response.status_code, "Request failed", "error")
if isinstance(parsed, dict) and "error" in parsed and "code" in parsed:
return Response(
content=body,
status_code=response.status_code,
media_type="application/json",
)
if isinstance(parsed, dict) and isinstance(parsed.get("error"), dict) and "message" in parsed["error"]:
return Response(
content=body,
status_code=response.status_code,
media_type="application/json",
)
if isinstance(parsed, dict) and "detail" in parsed:
message, code = sanitize_detail(parsed["detail"], response.status_code)
return error_response(response.status_code, message, code)
return error_response(response.status_code, "Request failed", "error")

37
devplacepy_services/base/proxy.py
View File

@ -1,37 +0,0 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
from starlette.requests import Request
HOP_HEADERS = frozenset(
{
"connection",
"keep-alive",
"proxy-authenticate",
"proxy-authorization",
"te",
"trailers",
"transfer-encoding",
"upgrade",
"host",
"content-length",
"content-encoding",
}
)
def forward_headers(request: Request, prefix: str) -> dict[str, str]:
headers = {
k: v for k, v in request.headers.items() if k.lower() not in HOP_HEADERS
}
headers["X-Forwarded-Prefix"] = prefix
headers["X-Script-Name"] = prefix
headers["X-Forwarded-Host"] = request.headers.get(
"host", request.url.hostname or ""
)
headers["X-Forwarded-Proto"] = request.headers.get(
"x-forwarded-proto", request.url.scheme
)
headers["Accept-Encoding"] = "identity"
return headers

13
devplacepy_services/base/pubsub_client.py
View File

@ -1,13 +0,0 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
from typing import Any
from devplacepy_services.base.config import service_url
from devplacepy_services.base.http import internal_request
async def publish(topic: str, data: dict[str, Any]) -> None:
url = f"{service_url('pubsub')}/internal/publish"
await internal_request("POST", url, json={"topic": topic, "data": data})

29
devplacepy_services/base/schemas.py
View File

@ -1,29 +0,0 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
from pydantic import BaseModel, ConfigDict
class ErrorOut(BaseModel):
error: str
code: str
class HealthStatsOut(BaseModel):
requests: int
errors: int
class HealthOut(BaseModel):
service: str
status: str
uptime_s: int
version: str
deps: dict[str, str]
stats: HealthStatsOut
class InternalUser(BaseModel):
model_config = ConfigDict(frozen=True)
uid: str | None = None

134
devplacepy_services/base/service.py
View File

@ -1,134 +0,0 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
import os
import subprocess
import time
from contextlib import asynccontextmanager
from typing import Literal
from fastapi import APIRouter, FastAPI
from fastapi.exceptions import HTTPException
from devplacepy_services.base.config import PROFILES, port_profile
from devplacepy_services.base.errors import error_response, sanitize_detail
from devplacepy_services.base.http import shutdown_pool, startup_pool
from devplacepy_services.base.middleware import (
InternalAuthMiddleware,
InternalCallCounterMiddleware,
RequestStatsMiddleware,
SanitizeErrorsMiddleware,
)
class BaseMicroservice:
name: str = ""
title: str = ""
default_port: int = 0
workers: int | Literal["auto"] = 1
stateful: bool = True
depends_on: list[str] = []
managed_services: list = []
use_background: bool = False
run_supervisor: bool = False
def __init__(self) -> None:
self.started_at = time.monotonic()
self.version = self._resolve_version()
def _resolve_version(self) -> str:
env_version = os.environ.get("DEVPLACE_VERSION", "").strip()
if env_version:
return env_version
try:
result = subprocess.run(
["git", "rev-parse", "--short", "HEAD"],
capture_output=True,
text=True,
timeout=2,
check=False,
)
if result.returncode == 0:
return result.stdout.strip() or "unknown"
except (OSError, subprocess.TimeoutExpired):
pass
return "unknown"
def resolved_port(self) -> int:
if self.default_port:
return self.default_port
profile = port_profile()
return PROFILES[profile][self.name]
def apply_base_middleware(self, app: FastAPI, *, internal_auth: bool = False) -> None:
app.add_middleware(SanitizeErrorsMiddleware)
if internal_auth:
app.add_middleware(InternalAuthMiddleware)
app.add_middleware(InternalCallCounterMiddleware)
app.add_middleware(RequestStatsMiddleware)
@asynccontextmanager
async def lifespan(self, app: FastAPI):
app.state.service_name = self.name
self.started_at = time.monotonic()
await startup_pool()
background = None
if self.use_background:
from devplacepy.services.background import background
await background.start()
background = background
if self.managed_services:
from devplacepy.services.manager import service_manager
for svc in self.managed_services:
service_manager.register(svc)
service_manager.set_lock_owner(True)
if self.run_supervisor:
import asyncio
asyncio.get_running_loop().call_soon(service_manager.supervise)
yield
if self.managed_services:
from devplacepy.services.manager import service_manager
await service_manager.shutdown_all()
if background is not None:
await background.stop()
await shutdown_pool()
def create_app(self) -> FastAPI:
app = FastAPI(title=self.title, lifespan=self.lifespan)
app.state.service_name = self.name
@app.exception_handler(HTTPException)
async def http_exception_handler(_request, exc: HTTPException):
if isinstance(exc.detail, dict) and "error" in exc.detail and "code" in exc.detail:
return error_response(exc.status_code, exc.detail["error"], exc.detail["code"])
message, code = sanitize_detail(exc.detail, exc.status_code)
return error_response(exc.status_code, message, code)
return app
def build_app(self) -> FastAPI:
raise NotImplementedError
def build_standard_app(
service: BaseMicroservice,
*,
routers: list[tuple[APIRouter, str] | tuple[APIRouter]] | None = None,
internal_auth: bool = False,
) -> FastAPI:
from devplacepy_services.base.health import health_router
app = service.create_app()
service.apply_base_middleware(app, internal_auth=internal_auth)
app.include_router(health_router(service))
for entry in routers or []:
if len(entry) == 2:
app.include_router(entry[0], prefix=entry[1])
else:
app.include_router(entry[0])
return app

110
devplacepy_services/base/sqlite_broker.py
View File

@ -1,110 +0,0 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
import asyncio
from collections.abc import Callable
from pathlib import Path
from typing import Any, TypeVar
import dataset
from devplacepy_services.base.config import sqlite_read_pool_size
T = TypeVar("T")
PRAGMAS = [
"PRAGMA journal_mode=WAL",
"PRAGMA synchronous=NORMAL",
"PRAGMA busy_timeout=30000",
"PRAGMA cache_size=-8000",
"PRAGMA temp_store=MEMORY",
"PRAGMA mmap_size=268435456",
]
class SQLiteFileBroker:
name: str
path: Path
read_pool_size: int
def __init__(self, name: str, path: Path, read_pool_size: int | None = None) -> None:
self.name = name
self.path = Path(path)
self.read_pool_size = read_pool_size or sqlite_read_pool_size()
self._write_db: Any = None
self._read_pool: asyncio.Queue[Any] | None = None
self._write_queue: asyncio.Queue[Any] | None = None
self._write_worker_task: asyncio.Task | None = None
def _connect_rw(self):
return dataset.connect(
f"sqlite:///{self.path}",
engine_kwargs={
"connect_args": {
"timeout": 30,
"check_same_thread": False,
},
},
on_connect_statements=PRAGMAS,
)
def _connect_ro(self):
abs_path = self.path.resolve().as_posix()
return dataset.connect(
f"sqlite:///file:{abs_path}?uri=true&mode=ro",
engine_kwargs={
"connect_args": {
"timeout": 30,
"check_same_thread": False,
"uri": True,
},
},
on_connect_statements=PRAGMAS,
)
async def startup(self) -> None:
self.path.parent.mkdir(parents=True, exist_ok=True)
self._write_db = self._connect_rw()
self._read_pool = asyncio.Queue()
for _ in range(self.read_pool_size):
await self._read_pool.put(self._connect_ro())
self._write_queue = asyncio.Queue()
self._write_worker_task = asyncio.create_task(self._write_worker())
async def shutdown(self) -> None:
if self._write_worker_task is not None:
self._write_worker_task.cancel()
try:
await self._write_worker_task
except asyncio.CancelledError:
pass
self._write_worker_task = None
async def read(self, fn: Callable[..., T], *args, **kwargs) -> T:
assert self._read_pool is not None
db = await self._read_pool.get()
try:
return await asyncio.to_thread(fn, db, *args, **kwargs)
finally:
await self._read_pool.put(db)
async def write(self, fn: Callable[..., T], *args, **kwargs) -> T:
assert self._write_queue is not None
loop = asyncio.get_running_loop()
future = loop.create_future()
await self._write_queue.put((fn, args, kwargs, future))
return await future
async def _write_worker(self) -> None:
assert self._write_queue is not None
while True:
fn, args, kwargs, future = await self._write_queue.get()
try:
result = fn(self._write_db, *args, **kwargs)
future.set_result(result)
except Exception as exc:
future.set_exception(exc)
async def read_after_write(self, fn: Callable[..., T], *args, **kwargs) -> T:
return await asyncio.to_thread(fn, self._write_db, *args, **kwargs)

26
devplacepy_services/base/stats.py
View File

@ -1,26 +0,0 @@
# retoor <retoor@molodetz.nl>
from dataclasses import dataclass, field
import threading
@dataclass
class RequestCounters:
requests: int = 0
errors: int = 0
_lock: threading.Lock = field(default_factory=threading.Lock, repr=False)
def record_request(self) -> None:
with self._lock:
self.requests += 1
def record_error(self) -> None:
with self._lock:
self.errors += 1
def snapshot(self) -> dict[str, int]:
with self._lock:
return {"requests": self.requests, "errors": self.errors}
COUNTERS = RequestCounters()

40
devplacepy_services/base/testing.py
View File

@ -1,40 +0,0 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
from typing import Any
from fastapi import FastAPI
from starlette.testclient import TestClient
from devplacepy_services.base.auth import set_internal_gateway_key
class ServiceTestClient:
def __init__(
self,
app: FastAPI,
*,
internal_key: str = "test-internal-key",
authenticated_user: str | None = None,
) -> None:
set_internal_gateway_key(internal_key)
headers: dict[str, str] = {"X-Internal-Key": internal_key}
if authenticated_user:
headers["X-Authenticated-User"] = authenticated_user
self._client = TestClient(app, headers=headers)
def get(self, path: str, **kwargs: Any):
return self._client.get(path, **kwargs)
def post(self, path: str, **kwargs: Any):
return self._client.post(path, **kwargs)
def put(self, path: str, **kwargs: Any):
return self._client.put(path, **kwargs)
def delete(self, path: str, **kwargs: Any):
return self._client.delete(path, **kwargs)
def close(self) -> None:
self._client.close()

1
devplacepy_services/bot/SERVICE.md
View File

@ -1 +0,0 @@
# Bot Service (stub)

0
devplacepy_services/bot/__init__.py
View File

23
devplacepy_services/bot/main.py
View File

@ -1,23 +0,0 @@
import devplacepy_services.base.bootstrap
from devplacepy.services.bot.service import BotsService
from devplacepy_services.base.service import BaseMicroservice, build_standard_app
class BotService(BaseMicroservice):
name = "bot"
title = "Bot"
default_port = 10632
workers = 1
stateful = True
depends_on = ["database", "gateway"]
managed_services = [BotsService()]
use_background = True
run_supervisor = True
def build_app(self):
return build_standard_app(self)
_service = BotService()
app = _service.build_app()

1
devplacepy_services/containers/SERVICE.md
View File

@ -1 +0,0 @@
# Containers Service (stub)

0
devplacepy_services/containers/__init__.py
View File

24
devplacepy_services/containers/main.py
View File

@ -1,24 +0,0 @@
import devplacepy_services.base.bootstrap
from devplacepy.routers.projects import containers
from devplacepy.services.containers.service import ContainerService
from devplacepy_services.base.service import BaseMicroservice, build_standard_app
class ContainersService(BaseMicroservice):
name = "containers"
title = "Containers"
default_port = 10634
workers = 1
stateful = True
depends_on = ["database"]
managed_services = [ContainerService()]
use_background = True
run_supervisor = True
def build_app(self):
return build_standard_app(self, routers=[(containers.router, "/projects")])
_service = ContainersService()
app = _service.build_app()

1
devplacepy_services/database/SERVICE.md
View File

@ -1 +0,0 @@
# Database Service (stub)

0
devplacepy_services/database/__init__.py
View File

52
devplacepy_services/database/broker_setup.py
View File

@ -1,52 +0,0 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
import os
from pathlib import Path
from devplacepy.config import DATA_DIR, DATABASE_URL
from devplacepy_services.base.sqlite_broker import SQLiteFileBroker
def _db_path() -> Path:
# DEVPLACE_DATABASE_URL is the documented override (tests use a dedicated
# tempfile for isolation - see tests/conftest.py). DATABASE_URL already
# falls back to DATA_DIR / "devplace.db" when the env var is unset, so
# deriving the broker's path from it (instead of hardcoding DATA_DIR
# directly) keeps the broker and every other DEVPLACE_DATABASE_URL
# consumer (devplacepy/database/core.py) pointed at the same file.
if DATABASE_URL.startswith("sqlite:///"):
raw = DATABASE_URL[len("sqlite:///") :]
if raw and raw != ":memory:":
return Path(raw)
return DATA_DIR / "devplace.db"
_broker = SQLiteFileBroker("main", _db_path())
def get_broker() -> SQLiteFileBroker:
return _broker
async def startup() -> None:
await _broker.startup()
from devplacepy_services.database.db_patch import patch_all_db, set_fallback_db
patch_all_db()
set_fallback_db(_broker._write_db)
os.environ["DEVPLACE_DB_SERVICE"] = "1"
from devplacepy.database import init_db
init_db()
from devplacepy.database.settings import internal_gateway_key
key = (internal_gateway_key() or "").strip()
if key:
os.environ["DEVPLACE_GATEWAY_INTERNAL_KEY"] = key
(DATA_DIR / ".internal_key").write_text(key + "\n")
async def shutdown() -> None:
await _broker.shutdown()

135
devplacepy_services/database/compounds_page.py
View File

@ -1,135 +0,0 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
from devplacepy.database import (
get_daily_topic,
get_polls_by_post_uids,
get_reactions_by_targets,
get_recent_comments_by_post_uids,
get_site_stats,
get_top_authors,
get_user_bookmarks,
get_users_by_uids,
)
from devplacepy.database.attachments_data import get_attachments_by_type
from devplacepy.routers.feed import get_feed_posts
def build_feed_page(
user: dict | None = None,
tab: str = "all",
topic: str | None = None,
search: str = "",
before: str | None = None,
) -> dict:
posts, next_cursor = get_feed_posts(user, tab, topic, search, before)
stats = get_site_stats()
top_authors = get_top_authors(5)
daily_topic = get_daily_topic()
post_uids = [item["post"]["uid"] for item in posts]
attachments_map = get_attachments_by_type("post", post_uids)
recent_comments = get_recent_comments_by_post_uids(post_uids, 3, user)
reactions_map = get_reactions_by_targets("post", post_uids, user)
bookmark_set = (
get_user_bookmarks(user["uid"], "post", post_uids) if user else set()
)
polls_map = get_polls_by_post_uids(post_uids, user)
for item in posts:
uid = item["post"]["uid"]
item["attachments"] = attachments_map.get(uid, [])
item["recent_comments"] = recent_comments.get(uid, [])
item["reactions"] = reactions_map.get(uid, {"counts": {}, "mine": []})
item["bookmarked"] = uid in bookmark_set
item["poll"] = polls_map.get(uid)
return {
"posts": posts,
"next_cursor": next_cursor,
"stats": stats,
"top_authors": top_authors,
"daily_topic": daily_topic,
"online_users": [],
"current_tab": tab,
"current_topic": topic,
"search": search,
}
def build_post_detail(post_uid: str, user: dict | None = None) -> dict:
from devplacepy.database import get_table, load_comments
post = get_table("posts").find_one(uid=post_uid, deleted_at=None)
if not post:
return {"post": None, "author": None, "comments": [], "attachments": []}
author = get_users_by_uids([post["user_uid"]]).get(post["user_uid"])
comments = load_comments("post", post_uid, user)
attachments = get_attachments_by_type("post", [post_uid]).get(post_uid, [])
return {
"post": post,
"author": author,
"comments": comments,
"attachments": attachments,
}
def build_profile_bundle(profile_uid: str, viewer: dict | None = None) -> dict:
from devplacepy.database import get_table
from devplacepy.database.follows import get_follow_counts
user = get_table("users").find_one(uid=profile_uid)
if not user:
return {"user": None, "follow_counts": {}, "awards": [], "posts_count": 0}
follow_counts = get_follow_counts(profile_uid)
return {
"user": user,
"follow_counts": follow_counts,
"awards": [],
"posts_count": 0,
"viewer_relation": {},
}
def build_messages_page(user_uid: str) -> dict:
return {"threads": [], "unread": 0, "partners": {}}
def build_notifications_page(user_uid: str) -> dict:
return {"notifications": [], "actors": {}, "unread_count": 0}
def build_leaderboard_page(viewer_uid: str | None = None) -> dict:
from devplacepy.database import get_leaderboard, get_user_rank
return {
"leaderboard": get_leaderboard(),
"viewer_rank": get_user_rank(viewer_uid) if viewer_uid else None,
}
def build_project_detail(project_uid: str) -> dict:
from devplacepy.database import get_table
project = get_table("projects").find_one(uid=project_uid, deleted_at=None)
owner = None
if project:
owner = get_users_by_uids([project["user_uid"]]).get(project["user_uid"])
return {"project": project, "owner": owner, "files": [], "containers": []}
def build_gist_detail(gist_uid: str, user: dict | None = None) -> dict:
from devplacepy.database import get_table, load_comments
gist = get_table("gists").find_one(uid=gist_uid, deleted_at=None)
author = None
if gist:
author = get_users_by_uids([gist["user_uid"]]).get(gist["user_uid"])
comments = load_comments("gist", gist_uid, user) if gist else []
return {"gist": gist, "author": author, "comments": comments}
def build_game_state(user_uid: str | None = None) -> dict:
return {"store": {}, "leaderboard": []}
def build_admin_dashboard() -> dict:
return {"stats": get_site_stats(), "service_states": []}

138
devplacepy_services/database/compounds_primitive.py
View File

@ -1,138 +0,0 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
from devplacepy.database import (
get_attachments_by_type,
get_comment_counts_by_post_uids,
get_follow_counts,
get_following_among,
get_leaderboard,
get_muted_uids,
get_blocked_uids,
get_user_relations,
get_notification_prefs,
get_online_users,
get_polls_by_post_uids,
get_prominent_award,
get_reactions_by_targets,
get_recent_comments_by_post_uids,
get_seo_metadata_batch,
get_site_stats,
get_top_authors,
get_user_awards,
get_user_bookmarks,
get_user_media,
get_user_rank,
get_users_by_uids,
get_vote_counts,
load_comments,
)
def users_by_uids(uids: list[str]) -> dict:
return get_users_by_uids(uids)
def comment_counts(post_uids: list[str]) -> dict:
return get_comment_counts_by_post_uids(post_uids)
def vote_counts(target_uids: list[str]) -> dict:
ups, downs = get_vote_counts(target_uids)
return {"ups": ups, "downs": downs}
def reactions(target_type: str, target_uids: list[str], user: dict | None = None) -> dict:
return get_reactions_by_targets(target_type, target_uids, user)
def bookmarks(user_uid: str, target_type: str, target_uids: list[str]) -> list[str]:
result = get_user_bookmarks(user_uid, target_type, target_uids)
return sorted(result)
def polls(post_uids: list[str], user: dict | None = None) -> dict:
return get_polls_by_post_uids(post_uids, user)
def recent_comments(
post_uids: list[str], limit: int = 3, user: dict | None = None
) -> dict:
return get_recent_comments_by_post_uids(post_uids, limit, user)
def comments(target_type: str, target_uid: str, user: dict | None = None) -> list:
return load_comments(target_type, target_uid, user)
def follow_bundle(user_uid: str, target_uids: list[str] | None = None) -> dict:
payload = {
"counts": get_follow_counts(user_uid),
"following_among": sorted(get_following_among(user_uid, target_uids or [])),
}
return payload
def relations_bundle(viewer_uid: str | None) -> dict:
relations = get_user_relations(viewer_uid)
return {
"relations": {
"block": sorted(relations["block"]),
"mute": sorted(relations["mute"]),
},
"blocked_uids": sorted(get_blocked_uids(viewer_uid)),
"muted_uids": sorted(get_muted_uids(viewer_uid)),
}
def online_users_bundle(cutoff_iso: str, limit: int = 30) -> list:
return get_online_users(cutoff_iso, limit)
def notification_prefs_bundle(user_uid: str) -> list:
return get_notification_prefs(user_uid)
def leaderboard_bundle(
limit: int = 50, offset: int = 0, viewer_uid: str | None = None
) -> dict:
return {
"leaderboard": get_leaderboard(limit, offset),
"viewer_rank": get_user_rank(viewer_uid) if viewer_uid else None,
}
def site_sidebar(authors_limit: int = 5) -> dict:
return {
"stats": get_site_stats(),
"top_authors": get_top_authors(authors_limit),
}
def awards_bundle(
receiver_uid: str,
page: int = 1,
per_page: int = 12,
profile_user: dict | None = None,
) -> dict:
items, pagination = get_user_awards(receiver_uid, page, per_page)
prominent = get_prominent_award(profile_user) if profile_user else None
return {
"items": items,
"pagination": pagination,
"prominent": prominent,
}
def attachments_batch(resource_type: str, resource_uids: list[str]) -> dict:
return get_attachments_by_type(resource_type, resource_uids)
def user_media_bundle(user_uid: str, page: int = 1, per_page: int = 24) -> dict:
items, pagination = get_user_media(user_uid, page, per_page)
return {"items": items, "pagination": pagination}
def seo_meta_batch(target_type: str, uids: list[str]) -> dict:
return get_seo_metadata_batch(target_type, uids)

119
devplacepy_services/database/db_patch.py
View File

@ -1,119 +0,0 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
import contextvars
import importlib
from typing import Any
_DB_MODULES = (
"schema",
"content",
"pagination",
"stats",
"forks",
"awards",
"customization",
"soft_delete",
"usage",
"email",
"follows",
"settings",
"seo_meta",
"deepsearch",
"activity",
"attachments_data",
"users",
"notifications",
"comments",
"ranking",
"engagement",
"relations",
)
_EXTERNAL_DB_MODULES = (
"devplacepy.services.audit.store",
"devplacepy.services.backup.store",
"devplacepy.services.openai_gateway.routing",
"devplacepy.services.openai_gateway.usage",
"devplacepy.services.openai_gateway.analytics",
"devplacepy.services.base",
"devplacepy.services.jobs.queue",
"devplacepy.services.containers.store",
"devplacepy.services.devii.store",
"devplacepy.services.devii.hub",
"devplacepy.attachments",
"devplacepy.project_files",
)
# The read pool runs on asyncio.to_thread worker threads while the write
# lane runs on the main event loop task - a plain module-global "current
# connection" swapped in/out per call is a data race between them (a
# concurrent read could reassign the global to a read-only connection
# while a write is still mid-flight, raising "attempt to write a readonly
# database"). asyncio.to_thread propagates a COPY of the calling task's
# contextvars.Context into the new thread, so storing the active
# connection in a ContextVar gives each call its own isolated view with
# no cross-talk, while every db-layer module keeps a single shared proxy
# object as its permanent `db` binding.
_CURRENT_DB: contextvars.ContextVar[Any] = contextvars.ContextVar("current_db", default=None)
# Fallback for any code path that touches the db outside an explicit
# use_db()-scoped call (e.g. startup's own init_db()) - write-once at
# broker startup, read-only afterward, so it carries no race of its own.
_fallback_db: Any = None
def _resolve() -> Any:
db = _CURRENT_DB.get()
if db is not None:
return db
if _fallback_db is not None:
return _fallback_db
raise RuntimeError("no database connection configured for this context")
class _ContextDb:
def __getattr__(self, name: str) -> Any:
return getattr(_resolve(), name)
def __getitem__(self, name: str) -> Any:
return _resolve()[name]
def __enter__(self):
return _resolve().__enter__()
def __exit__(self, exc_type, exc, tb):
return _resolve().__exit__(exc_type, exc, tb)
_CONTEXT_DB = _ContextDb()
def patch_all_db() -> None:
import devplacepy.database as database_pkg
import devplacepy.database.core as core
core.db = _CONTEXT_DB
database_pkg.db = _CONTEXT_DB
for name in _DB_MODULES:
mod = importlib.import_module(f"devplacepy.database.{name}")
if hasattr(mod, "db"):
mod.db = _CONTEXT_DB
for name in _EXTERNAL_DB_MODULES:
mod = importlib.import_module(name)
if hasattr(mod, "db"):
mod.db = _CONTEXT_DB
def set_fallback_db(db_conn) -> None:
global _fallback_db
_fallback_db = db_conn
def use_db(db_conn) -> contextvars.Token:
return _CURRENT_DB.set(db_conn)
def reset_db(token: contextvars.Token) -> None:
_CURRENT_DB.reset(token)

40
devplacepy_services/database/invoke_registry.py
View File

@ -1,40 +0,0 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
import inspect
from typing import Any
import devplacepy.database as db_module
from devplacepy_services.base.db_codec import decode_arg, encode_result, is_write
def build_registry() -> dict[str, Any]:
registry: dict[str, Any] = {}
for name in db_module.__all__:
target = getattr(db_module, name, None)
if target is None or not callable(target):
continue
if inspect.isclass(target):
continue
registry[name] = target
return registry
REGISTRY = build_registry()
_encode_result = encode_result
def run_with_db(db_conn, fn, args, kwargs):
from devplacepy_services.database.db_patch import use_db, reset_db
token = use_db(db_conn)
try:
decoded_args = decode_arg(args)
decoded_kwargs = decode_arg(kwargs)
return fn(*decoded_args, **decoded_kwargs)
finally:
reset_db(token)

52
devplacepy_services/database/main.py
View File

@ -1,52 +0,0 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
from contextlib import asynccontextmanager
from fastapi import FastAPI
from devplacepy_services.base.health import health_router
from devplacepy_services.base.http import shutdown_pool, startup_pool
from devplacepy_services.base.middleware import (
InternalAuthMiddleware,
InternalCallCounterMiddleware,
RequestStatsMiddleware,
SanitizeErrorsMiddleware,
)
from devplacepy_services.base.service import BaseMicroservice
from devplacepy_services.database import broker_setup
from devplacepy_services.database.routes import router as database_router
class DatabaseService(BaseMicroservice):
name = "database"
title = "Database"
default_port = 10601
workers = 1
stateful = True
depends_on = []
@asynccontextmanager
async def lifespan(self, app: FastAPI):
app.state.service_name = self.name
self.started_at = __import__("time").monotonic()
await startup_pool()
await broker_setup.startup()
yield
await broker_setup.shutdown()
await shutdown_pool()
def build_app(self):
app = self.create_app()
app.add_middleware(SanitizeErrorsMiddleware)
app.add_middleware(InternalAuthMiddleware)
app.add_middleware(InternalCallCounterMiddleware)
app.add_middleware(RequestStatsMiddleware)
app.include_router(health_router(self))
app.include_router(database_router)
return app
_service = DatabaseService()
app = _service.build_app()

421
devplacepy_services/database/routes.py
View File

@ -1,421 +0,0 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
from typing import Any
from fastapi import APIRouter, Depends
from pydantic import BaseModel, Field
from devplacepy.database import bump_cache_version, get_int_setting, get_setting
from devplacepy_services.base.auth import internal_user
from devplacepy_services.base.compound import (
AttachmentsIn,
AwardsBundleIn,
BookmarksIn,
CacheBumpIn,
CommentCountsIn,
CommentsIn,
FeedPageIn,
FollowBundleIn,
GameStateIn,
GistDetailIn,
InvokeIn,
LeaderboardBundleIn,
LeaderboardPageIn,
MessagesPageIn,
NotificationPrefsIn,
NotificationsPageIn,
OnlineUsersIn,
PollsIn,
PostDetailIn,
ProfileBundleIn,
ProjectDetailIn,
ReactionsIn,
RecentCommentsIn,
RelationsIn,
SeoMetaIn,
SiteSidebarIn,
UserMediaIn,
UsersByUidsIn,
VoteCountsIn,
)
from devplacepy_services.base.db_codec import is_write_sql
from devplacepy_services.base.schemas import InternalUser
from devplacepy_services.database import compounds_page, compounds_primitive
from devplacepy_services.database.broker_setup import get_broker
from devplacepy_services.database.invoke_registry import (
REGISTRY,
_encode_result,
is_write,
run_with_db,
)
router = APIRouter()
class DbOpIn(BaseModel):
op: str
table: str | None = None
method: str | None = None
args: list[Any] = Field(default_factory=list)
kwargs: dict[str, Any] = Field(default_factory=dict)
write: bool = False
def _db_tables(db_conn):
return list(db_conn.tables)
def _db_query(db_conn, sql: str, **params):
return list(db_conn.query(sql, **params))
def _db_table_op(db_conn, table: str, method: str, args: list, kwargs: dict):
target = db_conn[table]
fn = getattr(target, method)
result = fn(*args, **kwargs)
if hasattr(result, "__iter__") and not isinstance(result, (str, bytes, dict)):
try:
return [_encode_result(row) for row in result]
except TypeError:
pass
return _encode_result(result)
@router.post("/internal/invoke")
async def internal_invoke(body: InvokeIn, _: InternalUser = Depends(internal_user)):
fn = REGISTRY.get(body.fn)
if fn is None:
return {"error": f"Unknown function: {body.fn}", "code": "unknown_function"}
write = body.write or is_write(body.fn)
broker = get_broker()
if write:
result = await broker.write(run_with_db, fn, body.args, body.kwargs)
else:
result = await broker.read(run_with_db, fn, body.args, body.kwargs)
return {"result": _encode_result(result)}
@router.post("/internal/db-op")
async def internal_db_op(body: DbOpIn, _: InternalUser = Depends(internal_user)):
broker = get_broker()
if body.op == "tables":
tables = await broker.read(_db_tables)
return tables
if body.op == "query":
sql = body.args[0] if body.args else ""
write = body.write or is_write_sql(sql)
if write:
rows = await broker.write(_db_query, sql, **body.kwargs)
else:
rows = await broker.read(_db_query, sql, **body.kwargs)
return [_encode_result(row) for row in rows]
if body.op == "table_op":
if not body.table or not body.method:
return {"error": "table and method required", "code": "invalid_request"}
write_methods = {"insert", "update", "delete", "create_column_by_example"}
write = body.write or body.method in write_methods
if write:
result = await broker.write(
_db_table_op, body.table, body.method, body.args, body.kwargs
)
else:
result = await broker.read(
_db_table_op, body.table, body.method, body.args, body.kwargs
)
return result
return {"error": f"Unknown op: {body.op}", "code": "unknown_op"}
@router.get("/settings/{key}")
async def settings_get(key: str, default: str = "", _: InternalUser = Depends(internal_user)):
broker = get_broker()
value = await broker.read(run_with_db, get_setting, [key, default], {})
return {"key": key, "value": value}
@router.post("/cache/bump")
async def cache_bump(body: CacheBumpIn, _: InternalUser = Depends(internal_user)):
broker = get_broker()
await broker.write(run_with_db, bump_cache_version, [body.name], {})
return {"ok": True, "name": body.name}
@router.post("/compound/users-by-uids")
async def compound_users_by_uids(body: UsersByUidsIn, _: InternalUser = Depends(internal_user)):
broker = get_broker()
return await broker.read(run_with_db, compounds_primitive.users_by_uids, [body.uids], {})
@router.post("/compound/comment-counts")
async def compound_comment_counts(body: CommentCountsIn, _: InternalUser = Depends(internal_user)):
broker = get_broker()
return await broker.read(
run_with_db, compounds_primitive.comment_counts, [body.post_uids], {}
)
@router.post("/compound/vote-counts")
async def compound_vote_counts(body: VoteCountsIn, _: InternalUser = Depends(internal_user)):
broker = get_broker()
return await broker.read(
run_with_db, compounds_primitive.vote_counts, [body.target_uids], {}
)
@router.post("/compound/reactions")
async def compound_reactions(body: ReactionsIn, _: InternalUser = Depends(internal_user)):
broker = get_broker()
return await broker.read(
run_with_db,
compounds_primitive.reactions,
[body.target_type, body.target_uids, body.user],
{},
)
@router.post("/compound/bookmarks")
async def compound_bookmarks(body: BookmarksIn, _: InternalUser = Depends(internal_user)):
broker = get_broker()
return await broker.read(
run_with_db,
compounds_primitive.bookmarks,
[body.user_uid, body.target_type, body.target_uids],
{},
)
@router.post("/compound/polls")
async def compound_polls(body: PollsIn, _: InternalUser = Depends(internal_user)):
broker = get_broker()
return await broker.read(
run_with_db, compounds_primitive.polls, [body.post_uids, body.user], {}
)
@router.post("/compound/recent-comments")
async def compound_recent_comments(
body: RecentCommentsIn, _: InternalUser = Depends(internal_user)
):
broker = get_broker()
return await broker.read(
run_with_db,
compounds_primitive.recent_comments,
[body.post_uids, body.limit, body.user],
{},
)
@router.post("/compound/comments")
async def compound_comments(body: CommentsIn, _: InternalUser = Depends(internal_user)):
broker = get_broker()
return await broker.read(
run_with_db,
compounds_primitive.comments,
[body.target_type, body.target_uid, body.user],
{},
)
@router.post("/compound/follow-bundle")
async def compound_follow_bundle(body: FollowBundleIn, _: InternalUser = Depends(internal_user)):
broker = get_broker()
return await broker.read(
run_with_db,
compounds_primitive.follow_bundle,
[body.user_uid, body.target_uids],
{},
)
@router.post("/compound/relations")
async def compound_relations(body: RelationsIn, _: InternalUser = Depends(internal_user)):
broker = get_broker()
return await broker.read(
run_with_db, compounds_primitive.relations_bundle, [body.viewer_uid], {}
)
@router.post("/compound/online-users")
async def compound_online_users(body: OnlineUsersIn, _: InternalUser = Depends(internal_user)):
broker = get_broker()
return await broker.read(
run_with_db,
compounds_primitive.online_users_bundle,
[body.cutoff_iso, body.limit],
{},
)
@router.post("/compound/notification-prefs")
async def compound_notification_prefs(
body: NotificationPrefsIn, _: InternalUser = Depends(internal_user)
):
broker = get_broker()
return await broker.read(
run_with_db,
compounds_primitive.notification_prefs_bundle,
[body.user_uid],
{},
)
@router.post("/compound/leaderboard")
async def compound_leaderboard(
body: LeaderboardBundleIn, _: InternalUser = Depends(internal_user)
):
broker = get_broker()
return await broker.read(
run_with_db,
compounds_primitive.leaderboard_bundle,
[body.limit, body.offset, body.viewer_uid],
{},
)
@router.post("/compound/site-sidebar")
async def compound_site_sidebar(body: SiteSidebarIn, _: InternalUser = Depends(internal_user)):
broker = get_broker()
return await broker.read(
run_with_db, compounds_primitive.site_sidebar, [body.authors_limit], {}
)
@router.post("/compound/awards")
async def compound_awards(body: AwardsBundleIn, _: InternalUser = Depends(internal_user)):
broker = get_broker()
return await broker.read(
run_with_db,
compounds_primitive.awards_bundle,
[body.receiver_uid, body.page, body.per_page, body.profile_user],
{},
)
@router.post("/compound/attachments")
async def compound_attachments(body: AttachmentsIn, _: InternalUser = Depends(internal_user)):
broker = get_broker()
return await broker.read(
run_with_db,
compounds_primitive.attachments_batch,
[body.resource_type, body.resource_uids],
{},
)
@router.post("/compound/user-media")
async def compound_user_media(body: UserMediaIn, _: InternalUser = Depends(internal_user)):
broker = get_broker()
return await broker.read(
run_with_db,
compounds_primitive.user_media_bundle,
[body.user_uid, body.page, body.per_page],
{},
)
@router.post("/compound/seo-meta")
async def compound_seo_meta(body: SeoMetaIn, _: InternalUser = Depends(internal_user)):
broker = get_broker()
return await broker.read(
run_with_db,
compounds_primitive.seo_meta_batch,
[body.target_type, body.uids],
{},
)
@router.post("/compound/feed-page")
async def compound_feed_page(body: FeedPageIn, _: InternalUser = Depends(internal_user)):
broker = get_broker()
return await broker.read(
run_with_db, compounds_page.build_feed_page, [], body.model_dump()
)
@router.post("/compound/post-detail")
async def compound_post_detail(body: PostDetailIn, _: InternalUser = Depends(internal_user)):
broker = get_broker()
return await broker.read(
run_with_db,
compounds_page.build_post_detail,
[body.post_uid, body.user],
{},
)
@router.post("/compound/profile-bundle")
async def compound_profile_bundle(
body: ProfileBundleIn, _: InternalUser = Depends(internal_user)
):
broker = get_broker()
return await broker.read(
run_with_db,
compounds_page.build_profile_bundle,
[body.profile_uid, body.viewer],
{},
)
@router.post("/compound/messages-page")
async def compound_messages_page(body: MessagesPageIn, _: InternalUser = Depends(internal_user)):
broker = get_broker()
return await broker.read(
run_with_db, compounds_page.build_messages_page, [body.user_uid], {}
)
@router.post("/compound/notifications-page")
async def compound_notifications_page(
body: NotificationsPageIn, _: InternalUser = Depends(internal_user)
):
broker = get_broker()
return await broker.read(
run_with_db, compounds_page.build_notifications_page, [body.user_uid], {}
)
@router.post("/compound/leaderboard-page")
async def compound_leaderboard_page(
body: LeaderboardPageIn, _: InternalUser = Depends(internal_user)
):
broker = get_broker()
return await broker.read(
run_with_db, compounds_page.build_leaderboard_page, [body.viewer_uid], {}
)
@router.post("/compound/project-detail")
async def compound_project_detail(
body: ProjectDetailIn, _: InternalUser = Depends(internal_user)
):
broker = get_broker()
return await broker.read(
run_with_db, compounds_page.build_project_detail, [body.project_uid], {}
)
@router.post("/compound/gist-detail")
async def compound_gist_detail(body: GistDetailIn, _: InternalUser = Depends(internal_user)):
broker = get_broker()
return await broker.read(
run_with_db,
compounds_page.build_gist_detail,
[body.gist_uid, body.user],
{},
)
@router.post("/compound/game-state")
async def compound_game_state(body: GameStateIn, _: InternalUser = Depends(internal_user)):
broker = get_broker()
return await broker.read(
run_with_db, compounds_page.build_game_state, [body.user_uid], {}
)
@router.post("/compound/admin-dashboard")
async def compound_admin_dashboard(_: InternalUser = Depends(internal_user)):
broker = get_broker()
return await broker.read(run_with_db, compounds_page.build_admin_dashboard, [], {})

1
devplacepy_services/devii/SERVICE.md
View File

@ -1 +0,0 @@
# Devii Service (stub)

0
devplacepy_services/devii/__init__.py
View File

40
devplacepy_services/devii/main.py
View File

@ -1,40 +0,0 @@
import devplacepy_services.base.bootstrap
from contextlib import asynccontextmanager
from fastapi import FastAPI
from devplacepy.routers import devii
from devplacepy.services.devii import DeviiService
from devplacepy_services.base.service import BaseMicroservice, build_standard_app
from devplacepy_services.devii import store_brokers
from devplacepy_services.devii.store_routes import router as store_router
class DeviiMicroservice(BaseMicroservice):
name = "devii"
title = "Devii"
default_port = 10631
workers = 2
stateful = True
depends_on = ["database", "pubsub", "gateway"]
managed_services = [DeviiService()]
use_background = True
run_supervisor = True
@asynccontextmanager
async def lifespan(self, app: FastAPI):
await store_brokers.startup()
async with super(DeviiMicroservice, self).lifespan(app):
yield
await store_brokers.shutdown()
def build_app(self):
return build_standard_app(
self,
routers=[(store_router,), (devii.router, "/devii")],
)
_service = DeviiMicroservice()
app = _service.build_app()

27
devplacepy_services/devii/store_brokers.py
View File

@ -1,27 +0,0 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
from devplacepy.config import DEVII_LESSONS_DB, DEVII_TASKS_DB
from devplacepy_services.base.sqlite_broker import SQLiteFileBroker
_tasks = SQLiteFileBroker("devii_tasks", DEVII_TASKS_DB)
_lessons = SQLiteFileBroker("devii_lessons", DEVII_LESSONS_DB)
def broker_for(name: str) -> SQLiteFileBroker:
if name == "devii_tasks":
return _tasks
if name == "devii_lessons":
return _lessons
raise KeyError(name)
async def startup() -> None:
await _tasks.startup()
await _lessons.startup()
async def shutdown() -> None:
await _tasks.shutdown()
await _lessons.shutdown()

63
devplacepy_services/devii/store_routes.py
View File

@ -1,63 +0,0 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
from typing import Any
from fastapi import APIRouter, Depends
from pydantic import BaseModel, Field
from devplacepy_services.base.auth import internal_user
from devplacepy_services.base.schemas import InternalUser
from devplacepy_services.database.invoke_registry import _encode_result
from devplacepy_services.database.routes import DbOpIn, _db_query, _db_table_op, _db_tables
from devplacepy_services.devii.store_brokers import broker_for
router = APIRouter()
class StoreDbOpIn(BaseModel):
op: str
table: str | None = None
method: str | None = None
args: list[Any] = Field(default_factory=list)
kwargs: dict[str, Any] = Field(default_factory=dict)
write: bool = False
def _make_handler(store_name: str):
async def handler(body: StoreDbOpIn, _: InternalUser = Depends(internal_user)):
broker = broker_for(store_name)
if body.op == "tables":
return await broker.read(_db_tables)
if body.op == "query":
sql = body.args[0] if body.args else ""
rows = await broker.read(_db_query, sql, **body.kwargs)
return [_encode_result(row) for row in rows]
if body.op == "table_op":
if not body.table or not body.method:
return {"error": "table and method required", "code": "invalid_request"}
write_methods = {"insert", "update", "delete", "create_column_by_example"}
write = body.write or body.method in write_methods
if write:
return await broker.write(
_db_table_op, body.table, body.method, body.args, body.kwargs
)
return await broker.read(
_db_table_op, body.table, body.method, body.args, body.kwargs
)
return {"error": f"Unknown op: {body.op}", "code": "unknown_op"}
return handler
router.add_api_route(
"/internal/store/devii_tasks/db-op",
_make_handler("devii_tasks"),
methods=["POST"],
)
router.add_api_route(
"/internal/store/devii_lessons/db-op",
_make_handler("devii_lessons"),
methods=["POST"],
)

1
devplacepy_services/email/SERVICE.md
View File

@ -1 +0,0 @@
# Email Service (stub)

0
devplacepy_services/email/__init__.py
View File

20
devplacepy_services/email/main.py
View File

@ -1,20 +0,0 @@
import devplacepy_services.base.bootstrap
from devplacepy_services.base.service import BaseMicroservice, build_standard_app
from devplacepy_services.email.routes import router
class EmailService(BaseMicroservice):
name = "email"
title = "Email"
default_port = 10636
workers = 1
stateful = True
depends_on = ["database"]
def build_app(self):
return build_standard_app(self, routers=[(router,)])
_service = EmailService()
app = _service.build_app()

9
devplacepy_services/email/routes.py
View File

@ -1,9 +0,0 @@
from fastapi import APIRouter
from fastapi.responses import JSONResponse
router = APIRouter()
@router.get("/status")
async def email_status():
return JSONResponse({"service": "email", "status": "stub"})

1
devplacepy_services/gateway/SERVICE.md
View File

@ -1 +0,0 @@
# Gateway Service (stub)

0
devplacepy_services/gateway/__init__.py
View File

24
devplacepy_services/gateway/main.py
View File

@ -1,24 +0,0 @@
import devplacepy_services.base.bootstrap
from devplacepy.routers import openai_gateway
from devplacepy.services.openai_gateway import GatewayService as OpenAIGatewayService
from devplacepy_services.base.service import BaseMicroservice, build_standard_app
class GatewayService(BaseMicroservice):
name = "gateway"
title = "Gateway"
default_port = 10620
workers = "auto"
stateful = False
depends_on = ["database"]
managed_services = [OpenAIGatewayService()]
use_background = True
run_supervisor = True
def build_app(self):
return build_standard_app(self, routers=[(openai_gateway.router, "/openai")])
_service = GatewayService()
app = _service.build_app()

1
devplacepy_services/gitea/SERVICE.md
View File

@ -1 +0,0 @@
# Gitea Service (stub)

0
devplacepy_services/gitea/__init__.py
View File

23
devplacepy_services/gitea/main.py
View File

@ -1,23 +0,0 @@
import devplacepy_services.base.bootstrap
from devplacepy.services.gitea.service import IssueTrackerService
from devplacepy_services.base.service import BaseMicroservice, build_standard_app
class GiteaService(BaseMicroservice):
name = "gitea"
title = "Gitea"
default_port = 10638
workers = 1
stateful = True
depends_on = ["database", "gateway"]
managed_services = [IssueTrackerService()]
use_background = True
run_supervisor = True
def build_app(self):
return build_standard_app(self)
_service = GiteaService()
app = _service.build_app()

1
devplacepy_services/jobs/SERVICE.md
View File

@ -1 +0,0 @@
# Jobs Service (stub)

0
devplacepy_services/jobs/__init__.py
View File

49
devplacepy_services/jobs/main.py
View File

@ -1,49 +0,0 @@
import devplacepy_services.base.bootstrap
from devplacepy.routers import forks, tools, zips
from devplacepy.services.jobs.award_service import AwardService
from devplacepy.services.jobs.deepsearch.service import DeepsearchService
from devplacepy.services.jobs.fork_service import ForkService
from devplacepy.services.jobs.issue_create_service import IssueCreateService
from devplacepy.services.jobs.isslop.service import IsslopService
from devplacepy.services.jobs.planning_service import PlanningReportService
from devplacepy.services.jobs.seo.service import SeoService
from devplacepy.services.jobs.seo_meta_service import SeoMetaService
from devplacepy.services.jobs.zip_service import ZipService
from devplacepy_services.base.service import BaseMicroservice, build_standard_app
class JobsService(BaseMicroservice):
name = "jobs"
title = "Jobs"
default_port = 10630
workers = 1
stateful = True
depends_on = ["database", "pubsub", "gateway"]
managed_services = [
ZipService(),
ForkService(),
SeoService(),
SeoMetaService(),
AwardService(),
DeepsearchService(),
IsslopService(),
IssueCreateService(),
PlanningReportService(),
]
use_background = True
run_supervisor = True
def build_app(self):
return build_standard_app(
self,
routers=[
(zips.router, "/zips"),
(forks.router, "/forks"),
(tools.router, "/tools"),
],
)
_service = JobsService()
app = _service.build_app()

1
devplacepy_services/news/SERVICE.md
View File

@ -1 +0,0 @@
# News Service (stub)

0
devplacepy_services/news/__init__.py
View File

23
devplacepy_services/news/main.py
View File

@ -1,23 +0,0 @@
import devplacepy_services.base.bootstrap
from devplacepy.services.news import NewsService
from devplacepy_services.base.service import BaseMicroservice, build_standard_app
class NewsMicroservice(BaseMicroservice):
name = "news"
title = "News"
default_port = 10637
workers = 1
stateful = True
depends_on = ["database", "gateway"]
managed_services = [NewsService()]
use_background = True
run_supervisor = True
def build_app(self):
return build_standard_app(self)
_service = NewsMicroservice()
app = _service.build_app()

1
devplacepy_services/pubsub/SERVICE.md
View File

@ -1 +0,0 @@
# PubSub Service (stub)

0
devplacepy_services/pubsub/__init__.py
View File

31
devplacepy_services/pubsub/main.py
View File

@ -1,31 +0,0 @@
import devplacepy_services.base.bootstrap
from devplacepy.routers import pubsub
from devplacepy.services.live_view_relay import LiveViewRelayService
from devplacepy.services.notification_relay import NotificationRelayService
from devplacepy.services.presence_relay import PresenceRelayService
from devplacepy.services.pubsub import PubSubService
from devplacepy_services.base.service import BaseMicroservice, build_standard_app
class PubsubService(BaseMicroservice):
name = "pubsub"
title = "PubSub"
default_port = 10602
workers = 1
stateful = True
depends_on = ["database"]
managed_services = [
PubSubService(),
PresenceRelayService(),
NotificationRelayService(),
LiveViewRelayService(),
]
run_supervisor = True
def build_app(self):
return build_standard_app(self, routers=[(pubsub.router, "/pubsub")])
_service = PubsubService()
app = _service.build_app()

1
devplacepy_services/telegram/SERVICE.md
View File

@ -1 +0,0 @@
# Telegram Service (stub)

0
devplacepy_services/telegram/__init__.py
View File

24
devplacepy_services/telegram/main.py
View File

@ -1,24 +0,0 @@
import devplacepy_services.base.bootstrap
from devplacepy.services.telegram.outbox_service import TelegramOutboxService
from devplacepy.services.telegram.service import TelegramService
from devplacepy_services.base.service import BaseMicroservice, build_standard_app
class TelegramMicroservice(BaseMicroservice):
name = "telegram"
title = "Telegram"
default_port = 10635
workers = 1
stateful = True
depends_on = ["database", "pubsub"]
managed_services = [TelegramService(), TelegramOutboxService()]
use_background = True
run_supervisor = True
def build_app(self):
return build_standard_app(self)
_service = TelegramMicroservice()
app = _service.build_app()

1
devplacepy_services/web/SERVICE.md
View File

@ -1 +0,0 @@
# Web Service (stub)

0
devplacepy_services/web/__init__.py
View File

616
devplacepy_services/web/factory.py
View File

@ -1,616 +0,0 @@
import os
import devplacepy.db_client
import asyncio
import logging
import time
from collections import defaultdict
from contextlib import asynccontextmanager
from pathlib import Path
from fastapi import FastAPI, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
from fastapi.staticfiles import StaticFiles
from starlette.middleware.gzip import GZipMiddleware
from devplacepy.cache import TTLCache
from devplacepy.config import (
PORT,
STATIC_DIR,
STATIC_VERSION,
UPLOADS_DIR,
ensure_data_dirs,
)
from devplacepy.db_client import (
db,
get_blocked_uids,
get_comment_counts_by_post_uids,
get_int_setting,
get_news_images_by_uids,
get_setting,
get_table,
get_user_post_count,
get_user_stars,
get_users_by_uids,
get_vote_counts,
interleave_by_author,
)
from devplacepy.responses import json_error, respond, wants_json
from devplacepy.routers import (
admin,
auth,
avatar,
awards,
bookmarks,
comments,
dbapi,
devrant,
docs,
feed,
follow,
game,
gists,
issues,
leaderboard,
media,
messages,
news,
notifications,
polls,
posts,
profile,
projects,
proxy,
push,
reactions,
relations,
seo,
uploads,
votes,
)
from devplacepy.schemas import LandingOut, ValidationErrorOut
from devplacepy.seo import base_seo_context, site_url, website_schema
from devplacepy.services import presence
from devplacepy.services.audit import record as audit
from devplacepy.services.correction import PENDING_SCOPE_KEY
from devplacepy.templating import templates
from devplacepy.utils import client_ip, get_current_user, safe_next, time_ago
from devplacepy_services.web.ingress import mount_ingress
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
)
logger = logging.getLogger(__name__)
_rate_limit_store = defaultdict(list)
RATE_LIMIT = int(os.environ.get("DEVPLACE_RATE_LIMIT", "60"))
RATE_WINDOW = 60
WEB_WORKERS = max(1, int(os.environ.get("DEVPLACE_WEB_WORKERS", "1")))
RATE_LIMIT_DISABLED = os.environ.get("DEVPLACE_DISABLE_RATE_LIMIT") == "1"
HOT_SETTINGS_TTL = 2.0
_hot_settings_value: dict = {}
_hot_settings_at = 0.0
_last_rate_sweep = 0.0
RATE_SWEEP_INTERVAL = 60.0
def _hot_settings() -> dict:
global _hot_settings_value, _hot_settings_at
now = time.monotonic()
if not _hot_settings_value or now - _hot_settings_at >= HOT_SETTINGS_TTL:
_hot_settings_value = {
"maintenance_mode": get_setting("maintenance_mode", "0"),
"rate_limit_per_minute": max(
1, get_int_setting("rate_limit_per_minute", RATE_LIMIT)
),
"rate_limit_window_seconds": max(
1, get_int_setting("rate_limit_window_seconds", RATE_WINDOW)
),
}
_hot_settings_at = now
return _hot_settings_value
def _sweep_rate_limit_store(window_start: float) -> None:
stale = [
ip
for ip, timestamps in _rate_limit_store.items()
if not timestamps or timestamps[-1] <= window_start
]
for ip in stale:
del _rate_limit_store[ip]
def _worker_rate_limit(limit: int) -> int:
return max(1, -(-limit // WEB_WORKERS))
INLINE_MEDIA_EXTENSIONS = {
".jpg",
".jpeg",
".png",
".gif",
".webp",
".bmp",
".tiff",
".mp4",
".webm",
".ogv",
".mov",
".m4v",
".mp3",
}
class UploadStaticFiles(StaticFiles):
async def get_response(self, path, scope):
response = await super().get_response(path, scope)
disposition = (
"inline"
if Path(path).suffix.lower() in INLINE_MEDIA_EXTENSIONS
else "attachment"
)
response.headers["Content-Disposition"] = disposition
response.headers["Cache-Control"] = "public, max-age=604800"
return response
class CachedStaticFiles(StaticFiles):
async def get_response(self, path, scope):
response = await super().get_response(path, scope)
if Path(path).name == "service-worker.js":
response.headers["Cache-Control"] = "no-cache"
else:
response.headers["Cache-Control"] = "public, max-age=31536000, immutable"
return response
class FallbackStaticFiles(StaticFiles):
async def get_response(self, path, scope):
response = await super().get_response(path, scope)
if Path(path).name == "service-worker.js":
response.headers["Cache-Control"] = "no-cache"
else:
response.headers["Cache-Control"] = "public, max-age=3600"
return response
@asynccontextmanager
async def lifespan(app: FastAPI):
ensure_data_dirs()
from devplacepy.services.statistics.tracking import start_visit_flusher
start_visit_flusher()
from devplacepy.config import SERVICE_LOCK_FILE
from devplacepy.services.dbapi.service import DbApiJobService
from devplacepy.services.manager import service_manager
from devplacepy.services.weblock import acquire_web_lock
web_lock_owner = acquire_web_lock(SERVICE_LOCK_FILE)
if web_lock_owner:
service_manager.register(DbApiJobService())
service_manager.set_lock_owner(True)
service_manager.supervise()
logger.info(f"DevPlace web started on port {PORT}")
yield
logger.info("Shutting down web...")
if web_lock_owner:
await service_manager.shutdown_all()
from devplacepy.services.statistics.tracking import flush_visits
flush_visits()
_AUTH_FORM_PAGES = {
"/auth/signup": ("signup.html", "Join DevPlace"),
"/auth/login": ("login.html", "Sign In"),
"/auth/forgot-password": ("forgot_password.html", "Reset Password"),
}
_FRIENDLY_ERRORS = {
("username", "too_short"): "Username must be between 3 and 32 characters",
("username", "too_long"): "Username must be between 3 and 32 characters",
("password", "too_short"): "Password must be at least 6 characters",
}
def _friendly_error(err):
field = err["loc"][-1] if err.get("loc") else ""
key = (field, err.get("type", "").replace("string_", ""))
if key in _FRIENDLY_ERRORS:
return _FRIENDLY_ERRORS[key]
msg = err.get("msg", "Invalid input")
prefix = "Value error, "
return msg[len(prefix) :] if msg.startswith(prefix) else msg
_home_cache = TTLCache(ttl=int(os.environ.get("DEVPLACE_HOME_CACHE_TTL", "60")), max_size=4)
def _landing_news():
cached = _home_cache.get("news")
if cached is not None:
return cached
articles = []
if "news" in db.tables:
news_table = get_table("news")
raw = list(
news_table.find(show_on_landing=1, order_by=["-synced_at"], _limit=6)
)
images_by_news = get_news_images_by_uids([a["uid"] for a in raw])
for a in raw:
articles.append(
{
"uid": a["uid"],
"slug": a.get("slug", ""),
"title": a.get("title", ""),
"description": (a.get("description", "") or "")[:250],
"url": a.get("url", ""),
"source_name": a.get("source_name", ""),
"grade": a.get("grade", 0),
"featured": a.get("featured", 0),
"synced_at": a.get("synced_at", "") or "",
"time_ago": time_ago(a["synced_at"]) if a.get("synced_at") else "",
"image_url": a.get("image_url", "") or images_by_news.get(a["uid"], ""),
}
)
_home_cache.set("news", articles)
return articles
def _landing_recent_posts(blocked):
if not blocked:
cached = _home_cache.get("posts")
if cached is not None:
return cached
posts = []
if "posts" in db.tables:
posts_table = get_table("posts")
fetch_limit = 24 if blocked else 6
raw_posts = list(
posts_table.find(deleted_at=None, order_by=["-created_at"], _limit=fetch_limit)
)
if blocked:
raw_posts = [p for p in raw_posts if p["user_uid"] not in blocked]
raw_posts = raw_posts[:6]
raw_posts = interleave_by_author(raw_posts)
if raw_posts:
post_uids = [p["uid"] for p in raw_posts]
author_uids = [p["user_uid"] for p in raw_posts]
authors = get_users_by_uids(author_uids)
comment_counts = get_comment_counts_by_post_uids(post_uids)
upvotes, downvotes = get_vote_counts(post_uids)
for p in raw_posts:
posts.append(
{
"post": p,
"author": authors.get(p["user_uid"]),
"time_ago": time_ago(p["created_at"]),
"comment_count": comment_counts.get(p["uid"], 0),
"stars": upvotes.get(p["uid"], 0) - downvotes.get(p["uid"], 0),
"slug": p.get("slug", "") or p["uid"],
}
)
if not blocked:
_home_cache.set("posts", posts)
return posts
def create_web_app() -> FastAPI:
from devplacepy_services.base.middleware import (
InternalCallCounterMiddleware,
RequestStatsMiddleware,
SanitizeErrorsMiddleware,
)
app = FastAPI(
title="DevPlace",
docs_url="/swagger",
redoc_url=None,
openapi_url="/openapi.json",
lifespan=lifespan,
)
app.add_middleware(SanitizeErrorsMiddleware)
app.add_middleware(InternalCallCounterMiddleware)
app.add_middleware(RequestStatsMiddleware)
app.mount(
"/static/uploads",
UploadStaticFiles(directory=str(UPLOADS_DIR), check_dir=False),
name="uploads",
)
app.mount(
f"/static/v{STATIC_VERSION}",
CachedStaticFiles(directory=str(STATIC_DIR)),
name="static_versioned",
)
app.mount("/static", FallbackStaticFiles(directory=str(STATIC_DIR)), name="static")
@app.exception_handler(404)
async def not_found(request: Request, exc):
if wants_json(request):
return json_error(404, "Not found")
seo_ctx = base_seo_context(
request,
title="Not Found - DevPlace",
description="The page you requested does not exist.",
robots="noindex",
)
return templates.TemplateResponse(
request,
"error.html",
{
**seo_ctx,
"request": request,
"error_code": 404,
"error_message": "Page not found",
},
status_code=404,
)
@app.exception_handler(500)
async def server_error(request: Request, exc):
logger.exception("500 error on %s %s", request.method, request.url.path)
if wants_json(request):
return json_error(500, "Internal server error")
seo_ctx = base_seo_context(
request,
title="Server Error - DevPlace",
description="Something went wrong.",
robots="noindex",
)
return templates.TemplateResponse(
request,
"error.html",
{
**seo_ctx,
"request": request,
"error_code": 500,
"error_message": "Internal server error",
},
status_code=500,
)
@app.exception_handler(RequestValidationError)
async def on_validation_error(request: Request, exc: RequestValidationError):
errors = [_friendly_error(e) for e in exc.errors()]
if wants_json(request):
fields: dict = {}
for raw, friendly in zip(exc.errors(), errors):
name = raw["loc"][-1] if raw.get("loc") else "_"
fields.setdefault(str(name), []).append(friendly)
return JSONResponse(
ValidationErrorOut(fields=fields, messages=errors).model_dump(mode="json"),
status_code=422,
)
path = request.url.path
page = _AUTH_FORM_PAGES.get(path)
if page is None and path.startswith("/auth/reset-password/"):
page = ("reset_password.html", "Set New Password")
if page:
template_name, title = page
context = {
**base_seo_context(request, title=title, robots="noindex,nofollow"),
"request": request,
"errors": errors,
}
try:
form = await request.form()
context.update({k: v for k, v in form.items() if isinstance(v, str)})
except Exception:
pass
if "token" in request.path_params:
context["token"] = request.path_params["token"]
return templates.TemplateResponse(
request, template_name, context, status_code=400
)
referer = safe_next(request.headers.get("referer"), "/feed")
return RedirectResponse(url=referer, status_code=303)
mount_ingress(app)
app.include_router(auth.router, prefix="/auth")
app.include_router(feed.router, prefix="/feed")
app.include_router(posts.router, prefix="/posts")
app.include_router(comments.router, prefix="/comments")
app.include_router(projects.router, prefix="/projects")
app.include_router(profile.router, prefix="/profile")
app.include_router(messages.router, prefix="/messages")
app.include_router(notifications.router, prefix="/notifications")
app.include_router(votes.router, prefix="/votes")
app.include_router(reactions.router, prefix="/reactions")
app.include_router(bookmarks.router, prefix="/bookmarks")
app.include_router(polls.router, prefix="/polls")
app.include_router(avatar.router, prefix="/avatar")
app.include_router(awards.router, prefix="/awards")
app.include_router(follow.router, prefix="/follow")
app.include_router(relations.router)
app.include_router(leaderboard.router, prefix="/leaderboard")
app.include_router(admin.router, prefix="/admin")
app.include_router(seo.router)
app.include_router(push.router)
app.include_router(docs.router)
app.include_router(issues.router, prefix="/issues")
app.include_router(gists.router, prefix="/gists")
app.include_router(news.router, prefix="/news")
app.include_router(uploads.router, prefix="/uploads")
app.include_router(media.router, prefix="/media")
app.include_router(proxy.router, prefix="/p")
app.include_router(devrant.router, prefix="/api")
app.include_router(dbapi.router, prefix="/dbapi")
app.include_router(game.router, prefix="/game")
@app.middleware("http")
async def await_pending_corrections(request: Request, call_next):
response = await call_next(request)
pending = request.scope.get(PENDING_SCOPE_KEY)
if pending:
await asyncio.gather(*pending, return_exceptions=True)
return response
@app.middleware("http")
async def add_security_headers(request: Request, call_next):
response = await call_next(request)
if not response.headers.get("X-Robots-Tag"):
response.headers["X-Robots-Tag"] = "index, follow"
response.headers["X-Content-Type-Options"] = "nosniff"
response.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains"
response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin"
if not request.url.path.startswith("/p/"):
response.headers["X-Frame-Options"] = "DENY"
response.headers["Content-Security-Policy"] = (
"object-src 'none'; base-uri 'self'; "
"frame-ancestors 'none'; form-action 'self'"
)
if request.url.path.startswith("/admin"):
response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate, max-age=0"
response.headers["Pragma"] = "no-cache"
response.headers["Expires"] = "0"
return response
@app.middleware("http")
async def rate_limit_middleware(request: Request, call_next):
if RATE_LIMIT_DISABLED:
return await call_next(request)
if request.method in (
"POST",
"PUT",
"DELETE",
"PATCH",
) and not request.url.path.startswith(("/openai", "/xmlrpc")):
settings = _hot_settings()
limit = _worker_rate_limit(settings["rate_limit_per_minute"])
window = settings["rate_limit_window_seconds"]
ip = client_ip(request, default="unknown")
now = time.time()
window_start = now - window
global _last_rate_sweep
if now - _last_rate_sweep >= RATE_SWEEP_INTERVAL:
_sweep_rate_limit_store(window_start)
_last_rate_sweep = now
timestamps = [t for t in _rate_limit_store.get(ip, ()) if t > window_start]
if len(timestamps) >= limit:
_rate_limit_store[ip] = timestamps
audit.record(
request,
"security.rate_limit.block",
result="denied",
summary=f"request from {ip} blocked by rate limit",
metadata={"ip": ip, "limit": limit, "window_seconds": window},
)
retry_after = {"Retry-After": str(window)}
if wants_json(request):
response = json_error(429, "Rate limit exceeded. Try again later.")
response.headers["Retry-After"] = str(window)
return response
return HTMLResponse(
"Rate limit exceeded. Try again later.",
status_code=429,
headers=retry_after,
)
timestamps.append(now)
_rate_limit_store[ip] = timestamps
return await call_next(request)
_MAINTENANCE_ALLOWED_PREFIXES = ("/static", "/avatar", "/auth", "/admin", "/openai")
@app.middleware("http")
async def maintenance_middleware(request: Request, call_next):
if _hot_settings()["maintenance_mode"] != "1":
return await call_next(request)
if request.url.path.startswith(_MAINTENANCE_ALLOWED_PREFIXES):
return await call_next(request)
user = get_current_user(request)
if user and user.get("role") == "Admin":
return await call_next(request)
message = get_setting(
"maintenance_message",
"DevPlace is undergoing scheduled maintenance. Please check back shortly.",
)
audit.record(
request,
"security.maintenance.block",
user=user,
result="denied",
summary="non-admin request blocked by maintenance mode",
)
if wants_json(request):
return json_error(503, message)
seo_ctx = base_seo_context(
request, title="Maintenance - DevPlace", description=message, robots="noindex"
)
return templates.TemplateResponse(
request,
"error.html",
{**seo_ctx, "request": request, "error_code": 503, "error_message": message},
status_code=503,
)
@app.middleware("http")
async def track_presence(request: Request, call_next):
path = request.url.path
if not path.startswith(("/static", "/avatar")):
user = get_current_user(request)
if user:
presence.touch(user["uid"])
return await call_next(request)
@app.middleware("http")
async def visit_statistics(request: Request, call_next):
from devplacepy.services.statistics.tracking import track_visit
response = await call_next(request)
track_visit(request, response.status_code)
return response
@app.middleware("http")
async def response_timing(request: Request, call_next):
start = time.perf_counter()
request.state.request_start = start
response = await call_next(request)
response.headers["X-Response-Time"] = f"{(time.perf_counter() - start) * 1000:.1f}ms"
return response
app.add_middleware(GZipMiddleware, minimum_size=512, compresslevel=6)
@app.get("/")
async def landing(request: Request):
user = get_current_user(request)
landing_articles = _landing_news()
blocked = get_blocked_uids(user["uid"]) if user else frozenset()
landing_posts = _landing_recent_posts(blocked)
base = site_url(request)
seo_ctx = base_seo_context(
request,
title="DevPlace - The Developer Social Network",
description="Track industry shifts. Discover bold releases. Share what you're building in an open, uncensored environment.",
breadcrumbs=[],
schemas=[website_schema(base)],
)
return respond(
request,
"landing.html",
{
**seo_ctx,
"request": request,
"user": user,
"is_authenticated": bool(user),
"user_post_count": get_user_post_count(user["uid"]) if user else 0,
"user_stars": get_user_stars(user["uid"]) if user else 0,
"landing_articles": landing_articles,
"landing_posts": landing_posts,
},
model=LandingOut,
)
return app
app = create_web_app()

216
devplacepy_services/web/ingress.py
View File

@ -1,216 +0,0 @@
import asyncio
import logging
from urllib.parse import urlparse
import httpx
import uuid_utils
import websockets
from starlette.datastructures import Headers
from starlette.requests import Request
from starlette.responses import Response
from starlette.routing import Route
from starlette.websockets import WebSocket
from devplacepy_services.base.config import service_url
from devplacepy_services.base.errors import error_response
from devplacepy_services.base.manifest import INGRESS_ROUTES as _INGRESS_SPECS
from devplacepy_services.base.proxy import HOP_HEADERS
logger = logging.getLogger(__name__)
INGRESS_ROUTES = [(route.prefix, route.service) for route in _INGRESS_SPECS]
TIMEOUTS = {
"xmlrpc": 120.0,
}
def _forward_headers_from_scope(scope) -> dict[str, str]:
original = Headers(scope=scope)
headers = {
k: v for k, v in original.items() if k.lower() not in HOP_HEADERS
}
headers["X-Request-Id"] = headers.get("X-Request-Id") or uuid_utils.uuid7().hex
headers["Accept-Encoding"] = "identity"
# Mirror nginx's `proxy_set_header Host $host` (Appendix F) so an
# upstream-generated absolute URL (redirect, url_for) reflects the
# public :10500 endpoint the browser is actually talking to, not the
# upstream service's own internal bind address/port.
original_host = original.get("host")
if original_host:
headers["Host"] = original_host
return headers
def _forward_headers(request: Request) -> dict[str, str]:
return _forward_headers_from_scope(request.scope)
_WS_HANDSHAKE_HEADERS = frozenset(
{
"sec-websocket-key",
"sec-websocket-version",
"sec-websocket-extensions",
"sec-websocket-protocol",
"sec-websocket-accept",
}
)
def _forward_ws_headers(scope) -> dict[str, str]:
headers = _forward_headers_from_scope(scope)
for key in list(headers):
if key.lower() in _WS_HANDSHAKE_HEADERS:
del headers[key]
return headers
def _target_path(scope, prefix: str) -> str:
# Starlette's Mount rewrites scope["root_path"] to the cumulative mount
# prefix but leaves scope["path"] as the FULL original request path (it
# does not strip the prefix) - so the full path alone is already the
# correct upstream path; concatenating root_path in front double-prefixes it.
return scope.get("path", "") or prefix
def _upstream_http_url(service: str, path: str, query: str) -> str:
base = service_url(service).rstrip("/")
url = f"{base}{path}"
if query:
url = f"{url}?{query}"
return url
def _upstream_ws_url(service: str, path: str, query: str) -> str:
parsed = urlparse(service_url(service))
scheme = "wss" if parsed.scheme == "https" else "ws"
netloc = parsed.netloc
url = f"{scheme}://{netloc}{path}"
if query:
url = f"{url}?{query}"
return url
class IngressProxy:
def __init__(self, prefix: str, service: str) -> None:
self.prefix = prefix
self.service = service
self.timeout = TIMEOUTS.get(service, 30.0)
async def __call__(self, scope, receive, send) -> None:
if scope["type"] == "http":
await self._proxy_http(scope, receive, send)
elif scope["type"] == "websocket":
await self._proxy_ws(scope, receive, send)
async def _proxy_http(self, scope, receive, send) -> None:
request = Request(scope, receive)
path = _target_path(scope, self.prefix)
query = scope.get("query_string", b"").decode()
url = _upstream_http_url(self.service, path, query)
body = await request.body()
headers = _forward_headers(request)
try:
async with httpx.AsyncClient(
timeout=self.timeout, follow_redirects=False
) as client:
upstream = await client.request(
request.method,
url,
headers=headers,
content=body,
)
except httpx.HTTPError as exc:
logger.warning("ingress %s upstream error: %s", self.prefix, exc)
response = error_response(
502, "Upstream service unavailable", "upstream_error"
)
await response(scope, receive, send)
return
out_headers = {
k: v
for k, v in upstream.headers.items()
if k.lower() not in HOP_HEADERS and k.lower() != "set-cookie"
}
response = Response(
content=upstream.content,
status_code=upstream.status_code,
headers=out_headers,
media_type=upstream.headers.get("content-type"),
)
for cookie in upstream.headers.get_list("set-cookie"):
response.headers.append("set-cookie", cookie)
await response(scope, receive, send)
async def _proxy_ws(self, scope, receive, send) -> None:
client_ws = WebSocket(scope, receive, send)
path = _target_path(scope, self.prefix)
query = scope.get("query_string", b"").decode()
upstream_url = _upstream_ws_url(self.service, path, query)
headers = _forward_ws_headers(scope)
await client_ws.accept()
try:
async with websockets.connect(
upstream_url,
open_timeout=10,
max_size=None,
additional_headers=headers,
) as upstream:
await _pump(client_ws, upstream)
except Exception as exc:
logger.debug("ingress ws %s failed: %s", self.prefix, exc)
try:
await client_ws.close(code=1011)
except Exception:
pass
async def _pump(client_ws: WebSocket, upstream) -> None:
async def client_to_upstream():
try:
while True:
message = await client_ws.receive()
if message["type"] == "websocket.disconnect":
break
if message.get("text") is not None:
await upstream.send(message["text"])
elif message.get("bytes") is not None:
await upstream.send(message["bytes"])
except Exception:
pass
finally:
await upstream.close()
async def upstream_to_client():
try:
async for message in upstream:
if isinstance(message, (bytes, bytearray)):
await client_ws.send_bytes(bytes(message))
else:
await client_ws.send_text(message)
except Exception:
pass
finally:
try:
await client_ws.close()
except Exception:
pass
await asyncio.gather(client_to_upstream(), upstream_to_client())
def mount_ingress(app) -> None:
for prefix, service in INGRESS_ROUTES:
proxy = IngressProxy(prefix, service)
# A bare hit on the prefix itself (no trailing slash, nothing after -
# e.g. a POST to /xmlrpc or GET /tools) never matches Mount's own
# path regex (it requires a "/" plus content after the prefix), so
# Starlette's router-level redirect_slashes fallback 307s it to
# "<prefix>/" before the Mount ever sees it. If the upstream service's
# own router registers an exact route at its mount root (as /tools
# and /xmlrpc both do), THAT redirects back to the bare prefix -
# an infinite loop between the two opposite trailing-slash
# conventions. Registering an explicit Route at the exact prefix
# bypasses Mount's regex/redirect fallback entirely for that one path.
app.router.routes.append(Route(prefix, endpoint=proxy, methods=None))
app.mount(prefix, proxy)

21
devplacepy_services/web/main.py
View File

@ -1,21 +0,0 @@
from devplacepy_services.base.health import health_router
from devplacepy_services.base.service import BaseMicroservice
from devplacepy_services.web.factory import create_web_app
class WebService(BaseMicroservice):
name = "web"
title = "Web"
default_port = 10500
workers = "auto"
stateful = False
depends_on = ["database", "pubsub"]
def build_app(self):
app = create_web_app()
app.include_router(health_router(self))
return app
_service = WebService()
app = _service.build_app()

1
devplacepy_services/xmlrpc/SERVICE.md
View File

@ -1 +0,0 @@
# XML-RPC Service (stub)

0
devplacepy_services/xmlrpc/__init__.py
View File

24
devplacepy_services/xmlrpc/main.py
View File

@ -1,24 +0,0 @@
import devplacepy_services.base.bootstrap
from devplacepy.routers import xmlrpc
from devplacepy.services.xmlrpc import XmlrpcService
from devplacepy_services.base.service import BaseMicroservice, build_standard_app
class XmlrpcMicroservice(BaseMicroservice):
name = "xmlrpc"
title = "XML-RPC"
default_port = 10649
workers = 1
stateful = True
depends_on = ["database"]
managed_services = [XmlrpcService()]
use_background = True
run_supervisor = True
def build_app(self):
return build_standard_app(self, routers=[(xmlrpc.router, "/xmlrpc")])
_service = XmlrpcMicroservice()
app = _service.build_app()

589
fplan.md
View File

@ -1,589 +0,0 @@
# Code Farm Economy Rebalance — Implementation Plan
Status: planning document only, no code changed yet. Target: execute phase by phase against the live `devplacepy` repository. The game is in production; every phase below is additive-only (new columns/tables/functions with safe defaults, no renames, no destructive migrations) so a partially-completed rollout never breaks an existing farm, and `init_db` stays idempotent as today.
## Ground rules for whoever executes this plan
1. **Never rename or remove an existing column, table, function signature, route, or schema field.** Extend with new optional fields/params carrying safe defaults.
2. **Every new `game_farms`/`game_plots` column must default to a value that reproduces today's behavior for a legacy row that has never touched the new feature** — `0` for counters/booleans, `""` for timestamps, matching the existing `store/common.py::_lvl()` convention (`int(farm.get(key) or 0)`).
3. **Reuse the established patterns before inventing new ones.** This codebase already has four proven shapes for "a shop of upgrades": `CI_TIERS` (tiered, coin-priced, single column), `PERKS` (leveled, coin-priced, resets on prestige), `LEGACY_UPGRADES` (leveled, star-priced, survives prestige), and the atomic-upsert-counter pattern in `database._add_usage`. Every new system below is explicitly modeled on one of these four — do not invent a fifth shape without a documented reason.
4. **No new background service, no new tick.** The game's hard architectural invariant is "pure + timestamp-driven, no background tick" (`services/game/CLAUDE.md`). Every new mechanic below is either (a) computed lazily inside `serialize_farm`/`store` calls the same way `_auto_harvest` already is, or (b) an explicit admin/user action. None of them run on a scheduler.
5. **Follow the "four faces of one route" workflow from the root `CLAUDE.md`** for every new route: HTML + JSON via `respond(..., model=XOut)`, a Devii `Action` in `services/devii/actions/catalog/game.py`, a `docs_api` entry in the Code Farm group, and a badge/achievement hook where it fits the existing pattern.
6. **Validate, never test-run automatically.** After each phase: `python -c "from devplacepy.main import app"`, grep the touched files for em-dash characters/entities, and manually check JS/CSS/HTML balance. Do **not** run `make test` unless the user explicitly asks — write new tests in the matching tier (`tests/unit/services/game/`, `tests/api/game/`, `tests/e2e/game/`) so the user can run them before deploying.
7. **Update `devplacepy/services/game/CLAUDE.md`, `README.md`, and this repo's docs (`docs_api`, `/docs` prose if relevant) at the end of every phase**, per the root `CLAUDE.md` feature workflow — do not defer documentation to the end of the whole plan.
---
## Why a plan this size, and what is deliberately scoped down
The critique is being taken in full. Six places in it ask for something the current codebase has no state for at all (raid "attempts," real-time raid scouting, weekly long-term goals, cross-user alliances, automatic timed resets). Building those literally, as new bespoke systems, would multiply the surface area and the risk. Instead each of those is folded into or approximated with a mechanism the codebase already has proven safe, and every such decision is called out explicitly below in **"Scope decisions and deviations"** so it can be reviewed before Phase 5+6 ship. Nothing is dropped — everything is mapped to a concrete, buildable primitive.
---
## Phase order
| Phase | Content | Depends on | Risk |
|---|---|---|---|
| 0 | Foundations: split `economy.py` into a package, introduce a single coin/xp credit choke point | none | low (mechanical, behavior-preserving) |
| 1 | Market Saturation | 0 | low |
| 2 | Coin sinks: Infrastructure tiers, upkeep Defense building, Cosmetics/titles | 0 | medium |
| 3 | Mastery track + new crop families | 0, 2 (cosmetics reused for era rewards later) | medium |
| 4 | Secondary leaderboards | 0, 3 (harvests_week, mastery) | low |
| 5 | Era/season system | 0, 1, 2, 3, 4 | high — recommend a second pass |
| 6 | Engagement loops: underdog bonus, weekly contracts, notification polish, (Alliance/Coop sketched, deferred) | 0, 3 | medium |
This mirrors the critique's own "Implementation order" section, with Phase 0 inserted first because Phases 1, 3, 4, and 5 all add a shadow counter next to every coin credit — doing that refactor once, up front, removes the single biggest source of "forgot one call site" bugs in the whole plan.
---
## Phase 0 — Foundations (no gameplay change)
### 0.1 Split `services/game/economy.py` into a package
The codebase already has the exact precedent for this: `store.py` was previously one file and is now the `store/` package (`actions.py`, `common.py`, `farm.py`, `quests.py`, `serialize.py`) re-exported from `store/__init__.py`. `economy.py` is about to roughly triple in size (crops, CI, leveling, perks, legacy, market, infrastructure, defense, mastery, cosmetics, leaderboard scoring, era — 12 concern areas), so apply the same split mechanically, before any new content is added:
- `devplacepy/services/game/economy/__init__.py` — re-exports every name from the submodules below, so `from devplacepy.services.game import economy; economy.CROPS` keeps working unchanged everywhere (`store/*.py`, `routers/game/*.py`, tests).
- `economy/crops.py` — `Crop`, `CROPS`, `CROP_BY_KEY`, `crop_for`, `unlocked_crops`, `crop_payload`.
- `economy/ci.py` — `CiTier`, `CI_TIERS`, `CI_BY_TIER`, `MAX_CI_TIER`, `ci_speed`, `next_ci_tier`, `farm_speed`, `grow_seconds_for`, `water_bonus_seconds`.
- `economy/leveling.py` — `MAX_LEVEL`, `xp_threshold`, `level_for_xp`, `level_progress`, `STARTING_COINS`, `STARTING_PLOTS`, `MAX_PLOTS`, `plot_cost`.
- `economy/perks.py` — `Perk`, `PERKS`, `PERK_BY_KEY`, `perk_for`, `perk_cost`, `perk_value_text`.
- `economy/legacy.py` — `LegacyUpgrade`, `LEGACY_UPGRADES`, `LEGACY_BY_KEY`, `legacy_for`, `legacy_cost`, `legacy_multiplier`, `legacy_value_text`, `prestige_base_plots`, `effective_steal_grace`, `effective_steal_fraction`, `LEGACY_SPEED_STEP`, `LEGACY_MULT_STEP`, `LEGACY_DEFENSE_GRACE`, `LEGACY_DEFENSE_FRACTION`.
- `economy/prestige.py` — `PRESTIGE_MIN_LEVEL`, `PRESTIGE_BONUS`, `prestige_multiplier`, `STAR_BASE`, `stars_for_refactor`.
- `economy/rewards.py` — `effective_plant_cost`, `effective_reward_coins`, `effective_reward_xp`, `steal_reward_coins`, `is_golden`, `GOLDEN_CHANCE`, `GOLDEN_MULTIPLIER`, `WATER_BONUS_PCT`, `MAX_WATERS_PER_PLOT`, `WATER_REWARD_COINS`, `WATER_REWARD_XP`, `STEAL_GRACE_SECONDS`, `STEAL_FRACTION`, `STEAL_COOLDOWN_SECONDS`.
- `economy/daily.py` — `DAILY_BASE`, `DAILY_STREAK_STEP`, `DAILY_STREAK_CAP`, `daily_reward`, `FERTILIZE_FRACTION`, `FERTILIZE_TAX`, `fertilize_click_cost`.
- `economy/quests.py` — `QuestDef`, `QUEST_DEFS`, `QUEST_KINDS`, `DAILY_QUEST_COUNT`, `daily_quests`.
- `economy/scoring.py` — `SCORE_*` constants, `farm_score`.
- `economy/market.py`, `economy/infrastructure.py`, `economy/defense.py`, `economy/mastery.py`, `economy/cosmetics.py`, `economy/era.py` — empty stub modules created now, populated in Phases 1-6.
This is a pure mechanical move: cut/paste function bodies verbatim, no formula changes. Verify with `python -c "from devplacepy.main import app"` and a full-text `grep -rn "from devplacepy.services.game import economy" devplacepy/ tests/` to confirm every caller still resolves every name it uses (the `__init__.py` re-export must be exhaustive — build it by `grep -oP '^(def|class) \w+' economy/*.py` and export every one).
### 0.2 Single coin/xp credit choke point
Today, `store/actions.py` updates `game_farms.coins`/`xp`/`total_harvests` inline, separately, at five call sites: `harvest`, `_auto_harvest`, `steal` (thief side), `claim_daily`, `claim_quest`, and `water` (visitor reward). Phases 1, 3, 4, and 5 each need to add one more shadow counter next to every coin credit (market-tick recording, `lifetime_coins_earned`, `harvests_week`, `era_coins`). Doing that at five scattered sites five times is exactly the kind of duplication the project's DRY convention forbids. Introduce one function now, before any of those counters exist, so every later phase touches one place:
In `store/common.py`, add:
```
def credit_farm(farm: dict, *, coins: int = 0, xp: int = 0, harvests: int = 0, is_kernel_harvest: bool = False, extra: dict | None = None) -> dict:
```
- Computes the new `coins`, `xp` (re-deriving `level` via `economy.level_for_xp`), `total_harvests` exactly as the current inline code does at each site today.
- Merges any `extra` fields (a plain dict of additional column updates) into the same single `_update_farm` call — this is the extension point every later phase hooks into instead of adding a sixth ad-hoc update site.
- Returns the updated farm dict (mirrors what each call site already does by re-reading/merging locally).
- `is_kernel_harvest` is accepted now (unused) so Phase 4's time-to-Kernel tracking has a stable signature to fill in later without touching every call site again.
Refactor `harvest`, `_auto_harvest`, `steal`, `claim_daily`, `claim_quest`, `water` in `store/actions.py` to call `credit_farm(...)` instead of their current inline `_update_farm` math. This step must be **behaviorally invisible** — same coins, same xp, same level, same total_harvests as before, verified by re-reading the diff against the current formulas rather than by running the suite (per the "never run tests unless asked" rule), and by asking the user to run `tests/unit/services/game/store.py` + `tests/api/game/mutations.py` before this phase is considered done, since they already assert exact reward amounts.
---
## Phase 1 — Market Saturation
### Design
A global per-crop production tracker computed from real harvest events (not a fabricated log — a genuinely new table, since today's harvest path never persists an event, only the aggregate `total_harvests` counter). Recent-harvest volume per crop reduces that crop's payout; low-tier crops get a small relief buff while the market is saturated. No existing column or table is touched.
**Deliberate scope decision:** only owner-initiated harvests and auto-harvests count toward saturation, not steals. A steal moves an already-grown build's value from owner to thief; it does not add new supply to the economy. Counting it would double-count the same unit of production. This is a real simplification of "calculated from existing harvest logs" (there is no existing log; a new lightweight one is added) but keeps the causal story of the feature honest: it throttles printing, not raiding.
### New table
`game_market_ticks` (added to the ensure-block in `devplacepy/database/schema.py` right after the existing `game_quests` block, same non-soft-deletable treatment as the other `game_*` tables):
| column | type/default | notes |
|---|---|---|
| `uid` | str | generate_uid(), gets the standard `idx_game_market_ticks_uid` via the `_uid_index` loop |
| `crop_key` | str, `""` | |
| `hour_bucket` | str, `""` | UTC `"YYYY-MM-DDTHH"` |
| `harvests` | int, `0` | |
| `updated_at` | str, `""` | |
Unique index `idx_game_market_ticks_bucket` on `(crop_key, hour_bucket)` — this is both the query index and the `ON CONFLICT` target for the upsert below.
### New module `services/game/store/market.py` (added to the `store/` package, exported from `store/__init__.py`)
- `_ticks()` → `get_table("game_market_ticks")`.
- `_hour_bucket(now: datetime) -> str`.
- `record_harvest_tick(crop_key: str, now: datetime) -> None` — atomic upsert, modeled exactly on `database._add_usage`'s pattern:
```
with db:
db.query(
"INSERT INTO game_market_ticks (uid, crop_key, hour_bucket, harvests, updated_at) "
"VALUES (:uid, :crop_key, :hour_bucket, 1, :now) "
"ON CONFLICT(crop_key, hour_bucket) DO UPDATE SET harvests = harvests + 1, updated_at = :now",
uid=generate_uid(), crop_key=crop_key, hour_bucket=_hour_bucket(now), now=_iso(now),
)
```
The `with db:` wrapper is load-bearing (memory: `dataset-dbquery-no-autocommit` — a raw `db.query` write that does not commit holds the SQLite write lock).
- `_saturation_cache = TTLCache(ttl=30, max_size=32)` keyed by `crop_key` — a display/economic cache, not a correctness cache, same class as the existing `_leaderboard_cache`; no `cache_state` version wiring needed (matches the documented rule: cross-worker version-sync is reserved for correctness-critical caches, this one only affects a rolling 48h window's precision by up to 30s).
- `recent_harvests(crop_key: str, window_hours: int) -> int` — `SELECT COALESCE(SUM(harvests), 0) FROM game_market_ticks WHERE crop_key = :crop_key AND hour_bucket >= :cutoff`, cutoff = `_hour_bucket(now - window_hours)`.
- `prune_ticks(older_than_hours: int = 96) -> int` — deletes buckets older than the cutoff, returns rows removed. Wired into a new CLI command (see below) so the table never grows unbounded, matching the existing prune convention (`zips prune`, `forks prune`, `seo prune`, etc.).
### `economy/market.py`
```
MARKET_WINDOW_HOURS = 48
MARKET_SATURATION_TIERS: tuple[tuple[int, float], ...] = (
(0, 1.00), (40, 0.85), (120, 0.70), (300, 0.55), (600, 0.40),
)
MARKET_BUFFED_CROPS = ("shell", "python", "webapp", "api")
MARKET_BUFF_CAP = 1.15
def market_saturation_factor(recent_harvests: int) -> float: ...
def market_buff_factor(crop_key: str, saturation_factor: float) -> float: ...
```
### Wiring into `economy/rewards.py`
Add an optional trailing parameter, default preserves old behavior for any caller (tests, future code) that omits it:
```
def effective_reward_coins(crop, yield_level=0, prestige=0, legacy_mult_level=0, market_factor=1.0) -> int:
...factor *= market_factor...
def steal_reward_coins(crop, yield_level=0, prestige=0, legacy_mult_level=0, defense_level=0, market_factor=1.0) -> int:
...passes market_factor through to effective_reward_coins...
```
`economy/crops.py::crop_payload(...)` also gains `market_factor: float = 1.0` so the shop preview shows the live, saturation-adjusted number before the player plants (avoids "why did I get less than shown").
### Wiring into `store/actions.py`
At `harvest`, `_auto_harvest`, and `steal`, before computing the coin reward: `recent = market.recent_harvests(crop.key, economy.MARKET_WINDOW_HOURS)`, `sat = economy.market_saturation_factor(recent)`, `factor = sat * economy.market_buff_factor(crop.key, sat)`, pass `market_factor=factor` into the reward call. After a successful `harvest`/`_auto_harvest` (not `steal`, per the scope decision above), call `market.record_harvest_tick(crop.key, now)`.
`store/serialize.py::serialize_farm` computes the same `factor` per crop when building `economy.crop_payload(...)` for the shop list.
### Schema
`GameCropOut` gains `market_state: str = "normal"` (`"normal"`/`"saturated"`/`"boosted"`, derived from whether `market_factor` is `<1`, `>1`, or `==1`). This is the only new field; `reward_coins` already reflects the real payout since it now includes the factor.
### Frontend
`_game_grid.html` / `_game_shop.html` and `GameFarm.js::_shopHtml`/`_gridHtml` render a small inline label next to a crop's reward when `market_state != "normal"` (`"Saturated -30%"` / `"Boosted +15%"`), no new CSS classes beyond one small modifier reusing existing badge styling patterns.
### CLI
`devplace game market prune` → calls `store.market.prune_ticks()`, added to `devplacepy/cli/` next to the other prune subcommands, documented in the root `CLAUDE.md` Commands table.
### Devii / docs
No new route, so no new Devii action — `game_state`/`game_view_farm`/`game_plant` responses automatically carry the new `market_state` field once it exists on `GameCropOut`. Add one sentence to the Code Farm `docs_api` group intro describing the field.
### Tests
`tests/unit/services/game/economy.py` — `market_saturation_factor`/`market_buff_factor` pure-function cases. `tests/unit/services/game/store.py` — `record_harvest_tick`/`recent_harvests` roundtrip. `tests/api/game/mutations.py` — harvesting the same crop repeatedly in a short window measurably reduces payout past the first saturation threshold.
---
## Phase 2 — Coin sinks
Three sub-systems, each modeled on an existing shape.
### 2a. Infrastructure tiers (one-time purchases, modeled on `LEGACY_UPGRADES`'s "owned" framing but boolean, per the critique's literal "new upgrade table, default owned = false")
`economy/infrastructure.py`:
```
@dataclass(frozen=True)
class Infrastructure:
key: str
name: str
icon: str
description: str
cost: int
min_prestige: int
INFRASTRUCTURE: tuple[Infrastructure, ...] = (
Infrastructure("registry", "Private Registry", "📦",
"Rust, Compiler, and Kernel crops grow 15% faster", 25_000_000, 3),
Infrastructure("canary", "Canary Deployments", "🐤",
"Every harvest has a 12% chance to double and a 6% chance to only refund its planting cost", 75_000_000, 8),
Infrastructure("observability", "Observability Suite", "🔭",
"Raises the minimum coins you keep when raided from 10% to 30%", 150_000_000, 15),
)
INFRA_BY_KEY = {i.key: i for i in INFRASTRUCTURE}
```
**Scope decision:** the critique's "see raid attempts in real time" for Observability is dropped — there is no scouting/detection mechanic anywhere in the game, and inventing one (who counts as "attempting," how it's surfaced, whether it needs a new pub/sub topic) is a distinct feature, not a coin sink. Observability instead gets a second, coin-preserving effect (raising the steal-fraction floor) that serves the same "stop feeling helpless against whales' raids" goal without new state.
New `game_farms` columns: `infra_registry=0`, `infra_canary=0`, `infra_observability=0` (int 0/1).
New `store/infrastructure.py`:
- `buy_infrastructure(user, key) -> dict` — `GameError` if unknown key, already owned, `prestige < INFRA_BY_KEY[key].min_prestige`, or insufficient coins; deducts cost (`credit_farm(farm, coins=-cost, extra={f"infra_{key}": 1})`); returns `{"key", "spent"}`.
Wiring:
- `registry` → `economy/ci.py::farm_speed`/`grow_seconds_for` gain a `registry_boost: bool = False` param, applied only for crop keys `{"rust", "haskell", "kernel"}`, threaded from `store` the same way `legacy_speed_level` already is.
- `canary` → in `harvest()`/`_auto_harvest()`, if `farm["infra_canary"]`, roll `random.random()` once per harvest (plain `random`, not the deterministic `is_golden` hash — canary is a fresh roll every time, not a per-planting property, so determinism has no purpose here) against `CANARY_DOUBLE_CHANCE=0.12` then `CANARY_FAIL_CHANCE=0.06`, adjusting the coin gain accordingly (double, or floor at the crop's `effective_plant_cost` refund, or normal).
- `observability` → `economy/legacy.py::effective_steal_fraction`'s floor (`max(0.1, ...)`) becomes `max(0.3 if owner_has_observability else 0.1, ...)`, threaded as a new `observability: bool = False` parameter alongside `defense_level`.
### 2b. Upkeep Defense building (the wealth-proportional sink — the critique's most important ask)
`economy/defense.py`:
```
@dataclass(frozen=True)
class DefenseTier:
level: int
name: str
upgrade_cost: int
upkeep_daily: int
steal_fraction_floor: float
grace_bonus: int
DEFENSE_TIERS: tuple[DefenseTier, ...] = (
DefenseTier(0, "Undefended", 0, 0, 0.10, 0),
DefenseTier(1, "Firewall", 5_000, 500, 0.10, 15),
DefenseTier(2, "WAF", 40_000, 2_500, 0.08, 30),
DefenseTier(3, "SOC Monitoring", 300_000, 15_000, 0.06, 60),
DefenseTier(4, "Zero Trust Mesh", 2_000_000, 100_000, 0.04, 120),
)
UPKEEP_WEALTH_PCT = 0.002 # 0.2% of current coin balance per day, whichever is larger than the flat fee
UPKEEP_GRACE_DAYS = 2 # unpaid days tolerated before the tier decays by one level
def daily_upkeep(tier: DefenseTier, coins: int) -> int:
return max(tier.upkeep_daily, round(coins * UPKEEP_WEALTH_PCT))
```
The `max(flat, wealth_pct)` formula is what makes this a genuine whale sink: a 282M-coin balance owes `max(100_000, 564_000) = 564_000`/day at tier 4, not a trivial flat fee, while a small farm pays the cheap flat floor.
New `game_farms` columns: `defense_level=0` (int), `defense_last_upkeep_at=""` (str timestamp).
New `store/defense.py`:
- `upgrade_defense(user) -> dict` — one-time purchase of the next tier, mirrors `upgrade_perk`'s shape exactly (raises on max tier / insufficient coins).
- `charge_upkeep(farm: dict, now: datetime) -> dict` — called lazily at the very top of `serialize_farm`, in the same place and spirit as `_auto_harvest` (lazy, timestamp-driven, no tick): if `defense_level > 0` and at least one full day elapsed since `defense_last_upkeep_at`, charge `daily_upkeep(tier, coins)` per elapsed day (capped at `UPKEEP_GRACE_DAYS` days of arrears); if coins are insufficient to cover even one day, decrement `defense_level` by one instead of going negative, and reset the timer — this implements "if you don't pay, protection weakens" literally. Returns the updated farm (or the original if nothing was due), same idiom as `_auto_harvest`'s return contract.
Wiring: `defense_level`'s `steal_fraction_floor`/`grace_bonus` combine additively with the existing `legacy_defense` level inside `effective_steal_fraction`/`effective_steal_grace` (both already take a `defense_level` int; extend the call site in `store` to pass `legacy_defense_level + defense_level`-equivalent inputs, or add a second explicit parameter — pick whichever keeps the function signatures readable; document the final choice in the code, not just here).
Schema: `GameFarmOut` gains `defense_level: int = 0`, `defense_tier_name: str = ""`, `defense_upkeep_daily: int = 0`, `defense_next_cost: int = 0`.
Route: `POST /game/defense/upgrade` (no body — mirrors `POST /game/upgrade` for CI), added to `routers/game/index.py` through the same `_respond_action` choke. Devii action `game_upgrade_defense`. `docs_api` entry. Template: new `_game_defense.html` partial + `data-defense-host` in `game.html`, `GameFarm.js::_defenseHtml`.
### 2c. Cosmetics and titles (pure status, coins or coins+stars)
New table `game_cosmetics` (own table, not flat columns, because this catalog is meant to grow over time and later feed Era rewards in Phase 5 — unlike the fixed five-slot Infrastructure/Defense tiers, this is an open-ended, appendable list):
| column | type/default |
|---|---|
| `uid` | str |
| `user_uid` | str |
| `cosmetic_key` | str |
| `purchased_at` | str |
| `created_at` | str |
Unique index `idx_game_cosmetics_owner` on `(user_uid, cosmetic_key)`.
`economy/cosmetics.py`:
```
@dataclass(frozen=True)
class Cosmetic:
key: str
name: str
icon: str
description: str
cost_coins: int
era_key: str | None = None # None = always purchasable; set in Phase 5 for era-exclusive items
COSMETICS: tuple[Cosmetic, ...] = (
Cosmetic("title_architect", "The Architect", "🏛️", "A permanent title shown on the leaderboard", 500_000),
Cosmetic("title_refactorer", "Serial Refactorer", "♻️", "Requires prestige >= 3 to purchase", 250_000),
Cosmetic("title_kernel_hacker", "Kernel Hacker", "⚙️", "Requires having harvested a Kernel", 1_000_000),
Cosmetic("skin_neon", "Neon Terminal", "🌈", "A cosmetic plot skin, no gameplay effect", 750_000),
)
COSMETIC_BY_KEY = {c.key: c for c in COSMETICS}
```
`store/cosmetics.py`: `buy_cosmetic(user, key)`, `owned_cosmetic_keys(user_uid) -> set[str]`, `equip_title(user, key)` (raises if not owned).
New `game_farms` column `active_title=""` (str, one of `COSMETIC_BY_KEY` keys of kind title, or empty).
Routes: `POST /game/cosmetics/buy` (`GameCosmeticForm{key}`), `POST /game/cosmetics/equip` (`GameCosmeticForm{key}`). Schema: `GameFarmOut` gains `owned_cosmetics: list[str] = []`, `active_title: str = ""`; `GameLeaderboardEntryOut` gains `title: str = ""`.
**Scope boundary, stated explicitly:** the active title renders on the Code Farm leaderboard and farm-view page only for v1. It does not touch `_avatar_link.html`, the profile page, or any other sitewide identity surface — that would be a much larger, riskier change (global avatar/identity partial touched by every render site in the app) for a feature that is currently scoped to one subsystem.
### Phase 2 badges
New `ACHIEVEMENTS` entries (group `"Code Farm"`): `infra_bought` -> `[(1, "Enterprise Ready")]`, `defense_upgraded` -> `[(1, "Fort Knox")]`, `cosmetic_bought` -> `[(1, "Style Points")]`. `track_action` calls at the three new success points.
### Phase 2 tests
Unit tests for `daily_upkeep`, `infra` cost/gating, `cosmetic` gating. API tests for each new route's success/failure paths (insufficient coins, already owned, prestige gate). E2E test for the new shop panel appearing and a purchase flowing through.
---
## Phase 3 — Mastery track and new crop families
### Design
An orthogonal permanent track, unlocked once a farm's prestige has crossed a threshold, spent on a small tree of upgrades that open new gameplay rather than bigger numbers, per the critique's stated design philosophy. Modeled directly on `LEGACY_UPGRADES` (leveled, survives prestige, its own currency).
**Two-counter design, load-bearing:** Mastery needs both a *spendable balance* (`mastery_points`, decreases when spent) and a *lifetime-earned total* (`mastery_points_earned_total`, never decreases) because the new crop families must stay unlocked even after a player spends their mastery points down to zero. Gating unlocks on the spendable balance would re-lock content the moment it's spent — a real bug if not caught here.
### `economy/prestige.py` additions
```
MASTERY_UNLOCK_PRESTIGE = 50
MASTERY_PRESTIGE_STEP = 10
def mastery_points_awarded(old_prestige: int, new_prestige: int) -> int:
if new_prestige < MASTERY_UNLOCK_PRESTIGE:
return 0
baseline = max(old_prestige, MASTERY_UNLOCK_PRESTIGE - 1)
return (new_prestige - MASTERY_UNLOCK_PRESTIGE) // MASTERY_PRESTIGE_STEP - max(0, baseline - MASTERY_UNLOCK_PRESTIGE) // MASTERY_PRESTIGE_STEP
```
(Prestige increments by exactly 1 per refactor, so in practice this awards 1 point whenever the new prestige is a multiple of 10 at or above 50 — written as a range difference so it is also correct if a future change ever lets prestige jump by more than 1.)
### `economy/mastery.py`
```
@dataclass(frozen=True)
class MasteryUpgrade:
key: str
name: str
icon: str
description: str
max_level: int
base_cost: int
cost_growth: float
MASTERY_UPGRADES: tuple[MasteryUpgrade, ...] = (
MasteryUpgrade("autoreplant", "Continuous Delivery", "🔁",
"Automatically replant the same crop right after harvest, if affordable", 1, 3, 1.0),
MasteryUpgrade("analytics", "Farm Analytics", "📊",
"Unlocks lifetime stats on your farm HUD", 1, 2, 1.0),
MasteryUpgrade("contracts", "Legacy Contracts", "📜",
"Unlocks a weekly long-term contract slot for Stars and a temporary boost", 1, 4, 1.0),
)
MASTERY_BY_KEY = {m.key: m for m in MASTERY_UPGRADES}
def mastery_cost(m: MasteryUpgrade, level: int) -> int:
return round(m.base_cost * (m.cost_growth ** level))
```
**DRY consolidation, called out explicitly:** the critique lists "Legacy Contracts" under Mastery (Section 2) and "Daily/Weekly contracts" under engagement loops (Section 6) as if they were two systems. They are implemented as **one** mechanism here: the `mastery_contracts` upgrade unlocks a fourth, weekly-cadence slot in the existing `game_quests` engine (see Phase 6). Building two parallel long-goal systems would violate the project's DRY convention for no gameplay benefit.
### `game_farms` new columns
`mastery_points=0`, `mastery_points_earned_total=0`, `mastery_autoreplant=0`, `mastery_analytics=0`, `mastery_contracts=0`, `lifetime_coins_earned=0`, `lifetime_harvests=0`.
`lifetime_coins_earned`/`lifetime_harvests` are accumulate-only counters incremented via the Phase 0 `credit_farm` choke point (one line added there, not five). They start at 0 for every existing farm on the day this ships — that under-counts a veteran's true lifetime total, which is an accepted, explicitly-noted trade-off (the alternative, backfilling from `total_harvests`/no historical coin ledger, is not possible since no such ledger exists; `total_harvests` already exists and is NOT reset, so the analytics panel should show that pre-existing counter alongside the new since-Mastery ones, clearly labeled).
### `store/actions.py::prestige()` change
After computing `new_prestige`, add `mastery_points_awarded(old_prestige, new_prestige)` to both `mastery_points` and `mastery_points_earned_total` in the same reset/update dict — both columns are, like `stars` and `legacy_*`, deliberately **excluded** from the fields that get zeroed on refactor.
### New `store/mastery.py`
- `upgrade_mastery(user, key) -> dict` — spends `mastery_points` (not coins), mirrors `upgrade_legacy`'s exact shape.
Wiring:
- `autoreplant` — extract a small internal helper `_plant_plot(farm, plot, crop, now) -> dict` out of the existing `plant()` body in `store/actions.py` (now genuinely has two callers: `plant()` itself and the auto-replant hook, so this is justified DRY, not premature abstraction). In `harvest()`/`_auto_harvest()`, after crediting and clearing, if `farm["mastery_autoreplant"]` and the same crop is still affordable and unlocked, call `_plant_plot` immediately.
- `analytics` — `GameFarmOut` gains `mastery_analytics_unlocked: bool = False`, `lifetime_coins_earned: int = 0`, `lifetime_harvests: int = 0` (always present on the schema per the existing "empty/zero unless unlocked" convention already used for `perks`/`quests`/`legacy`; template/JS render the panel only when `mastery_analytics_unlocked` is true).
- `contracts` — see Phase 6.
### New crop families
Extend the `Crop` dataclass (in `economy/crops.py`) with two new **trailing, defaulted** fields — safe because every existing `Crop(...)` construction in the `CROPS` tuple is positional with exactly 8 args and none of the 8 existing fields has a default, so appending defaulted fields after them is valid dataclass semantics and changes nothing for the 7 existing crops:
```
@dataclass(frozen=True)
class Crop:
key: str
name: str
icon: str
cost: int
grow_seconds: int
reward_coins: int
reward_xp: int
min_level: int
min_mastery: int = 0
steal_immune: bool = False
era_key: str | None = None # populated in Phase 5
```
New entries appended to `CROPS`:
```
Crop("distsys", "Distributed System", "🕸️", 5_000, 14_400, 9_500, 900, MAX_LEVEL, min_mastery=1),
Crop("mlpipe", "ML Pipeline", "🧠", 12_000, 21_600, 21_000, 1_800, MAX_LEVEL, min_mastery=1),
Crop("secfort", "Security Fortress", "🔐", 30_000, 28_800, 48_000, 3_200, MAX_LEVEL, min_mastery=1, steal_immune=True),
```
`min_level=MAX_LEVEL` means the level gate is always satisfied once a player is capped (a prerequisite for prestige anyway), so the real gate is `min_mastery`, satisfied by `mastery_points_earned_total >= 1` (i.e. having reached prestige 50 at least once and earned a Mastery point — the tree does not need to be spent, only unlocked, to grow these crops; this matches the critique's framing that Mastery "opens new gameplay" independent of what's purchased).
`unlocked_crops(level, mastery_earned=0)` signature gains the new parameter with a default of 0 (backward compatible for any other caller), filters `crop.min_level <= level and crop.min_mastery <= mastery_earned`.
`steal_immune` wiring: in `store/actions.py::steal()`, if `crop.steal_immune`, raise `GameError("This build cannot be raided.")` before any grace/cooldown check; `serialize_plot` sets `can_steal=False`, `steal_reason="immune"` unconditionally for such a plot.
### Phase 3 tests
Unit: `mastery_points_awarded` edge cases (crossing 50, 60, skipping — though prestige only moves by 1, still test the range-difference formula), `unlocked_crops` gating by mastery, `steal_immune` short-circuit. API: `POST /game/legacy`-equivalent `POST /game/mastery` route full cycle. E2E: new crops appear in the shop only after reaching prestige 50 in a seeded test farm (or a lower test-only threshold override, see the existing pattern for other prestige-gated e2e tests).
---
## Phase 4 — Secondary leaderboards
### Design
`store/farm.py::leaderboard()` today does one thing: rank by `economy.farm_score` over the full in-memory `_farms().find()` set. Add sibling ranking functions over the **same already-loaded set** (compute all boards from one query when a caller needs more than one, to avoid N separate full-table scans) rather than N new queries.
### New `game_farms` columns
- `harvests_week=0`, `harvests_week_start=""` (ISO date of the current tracking week's Monday) — incremented via `credit_farm`'s `harvests` param already threaded in Phase 0; lazily reset to 0 when `now`'s ISO week differs from `harvests_week_start`, checked at the same lazy point as `_auto_harvest`/`charge_upkeep` inside `serialize_farm`.
- `prestiged_at=""` (str timestamp) — set every time `prestige()` runs.
- `last_kernel_harvest_prestige=-1` (int, `-1` sentinel meaning "never"), `time_to_kernel_seconds=0` — updated in `harvest()`/`_auto_harvest()` when the harvested `crop.key == "kernel"` and `farm["last_kernel_harvest_prestige"] != farm["prestige"]`: `seconds = (now - parse(prestiged_at)).total_seconds()`, store both fields.
### `economy/scoring.py` additions
```
FAIR_PLAY_ACTIVITY_WEIGHT = 50
FAIR_PLAY_HOARD_DIVISOR = 200_000
MIN_RAIDS_FOR_EFFICIENCY_BOARD = 3
def fair_play_score(harvests_week: int, coins: int) -> int:
return harvests_week * FAIR_PLAY_ACTIVITY_WEIGHT - min(coins, 10**9) // FAIR_PLAY_HOARD_DIVISOR
```
### `store/farm.py` additions
- `leaderboard_prestige(limit=25)` — sort by `(prestige, stars)` desc.
- `leaderboard_harvests_week(limit=25)` — sort by `harvests_week` desc.
- `leaderboard_raid_efficiency(limit=25)` — `SELECT thief_uid, COUNT(*) as raids, SUM(coins) as total FROM game_steals GROUP BY thief_uid HAVING raids >= MIN_RAIDS_FOR_EFFICIENCY_BOARD ORDER BY (total * 1.0 / raids) DESC LIMIT :limit`. **Scope decision, stated explicitly:** this ranks average coins per *successful* raid, not "coins stolen / raids attempted" as literally written in the critique — failed attempts (blocked by cooldown or protection) are never persisted today, and adding a write on every failed attempt would be an easy target for script-spam with no real gameplay value. The metric is renamed "Raid Efficiency" and documented with this caveat in `docs_api` and the game's own leaderboard UI tooltip, so it is never silently misleading.
- `leaderboard_time_to_kernel(limit=25)` — filters `last_kernel_harvest_prestige == prestige and prestige > 0`, sorts `time_to_kernel_seconds` ascending.
- `leaderboard_fair_play(limit=25)` — sorts by `economy.fair_play_score(harvests_week, coins)` descending.
### Route
Extend the existing `GET /game/leaderboard` (do not add a new path — the critique's "parallel boards updated live" is one endpoint with a selector, matching how `admin.ai-usage.{hours}` already parameterizes a topic by value rather than by new routes): add `board: str = Query("score")` to `game_leaderboard` in `routers/game/index.py`, dispatch through a `{name: function}` map, default `"score"` reproduces exactly today's response for any caller that doesn't pass the parameter — **zero behavior change for existing Devii/docs/frontend callers that omit it.**
### Schema
`GameLeaderboardEntryOut` gains, all defaulted to a neutral value so the existing `score` board's JSON is unaffected: `raid_avg: float = 0.0`, `time_to_kernel_seconds: int = 0`, `fair_score: int = 0`, `title: str = ""` (from Phase 2c).
### Frontend
`game.html`'s `data-game-leaderboard` host gets a `<select data-lb-board>` control; `GameFarm.js::_loadLeaderboard(board = "score")` passes `?board=` through, re-renders the same row template with board-specific columns shown/hidden via a small per-board column map.
### Devii / docs
`game_leaderboard` Devii action gains an optional `board` parameter (backward compatible). `docs_api` entry documents the `board` enum and the Raid Efficiency caveat above.
### Phase 4 tests
Unit: each new scoring/sort function against a small in-memory fixture set. API: `GET /game/leaderboard?board=X` for every `X`, plus the no-param case still matches the pre-Phase-4 response shape exactly.
---
## Phase 5 — Era / Season system
This is, by a wide margin, the largest and riskiest phase — it is the one place in this plan recommended for a **second execution pass with a fresh review**, after Phases 0-4 have shipped and produced real signal about whether Market Saturation and the coin sinks already relieved enough of the inequality. It is fully specified here so it can be executed in one pass if that is still the decision, but the recommendation is explicit: pause here and confirm before flipping the first Era on in production.
### Design invariant, stated up front
**Real balances never move.** `coins`, `prestige`, `stars`, `legacy_*`, `mastery_*` are the permanent economy and are never reset, scaled down, or migrated by this phase — that would directly violate "every existing coin balance, prestige number... must continue to work exactly as it does today." Everything Era-specific is a **new, parallel, resettable set of counters** that only affects a separate Era leaderboard's ranking, never real gameplay math.
### New tables
`game_eras` (uid, era_number int, name str, started_at str, ends_at str, active int default 0, created_at str) — at most one row has `active=1` at a time; **zero active rows means the Era system is completely dormant and every other phase behaves exactly as if Phase 5 was never installed.** This is the backward-compatibility anchor for the whole phase: shipping the code with no era ever started is a no-op.
`game_era_results` (uid, era_number int, user_uid str, rank int, era_score int, era_coins_final int, reward_stars int, reward_cosmetic_key str, created_at str) — the permanent historical record written once, at era-end, before the live counters are zeroed.
### New `game_farms` columns
`era_coins=0`, `era_harvests=0`, `era_joined_at=""`.
`era_coins`/`era_harvests` are incremented through the Phase 0 `credit_farm` choke point (again: one line added there, in the one function every coin/xp credit already flows through) whenever an era is active; when no era is active, these columns simply stay at whatever they were (never touched), keeping the no-op guarantee above exact even mid-development.
### `economy/era.py`
```
ERA_PRESTIGE_CARRYOVER_PCT = 0.4 # veterans keep 40% of their prestige weight on the Era board only
ERA_REWARD_STARS_BY_RANK: tuple[int, ...] = (50, 30, 20, 15, 10, 5, 5, 5, 5, 5) # ranks 1-10
def era_score(era_coins: int, era_harvests: int, prestige: int) -> int:
return era_coins // 20 + era_harvests * 10 + round(prestige * SCORE_PRESTIGE * ERA_PRESTIGE_CARRYOVER_PCT)
```
This is a distinct scoring function from `economy.scoring.farm_score` — it must never replace it. The permanent/global leaderboard (Phase 4) keeps ranking by full lifetime stats; only the Era-specific board uses `era_score`.
### Admin-triggered lifecycle — a deliberate deviation from "automatic every 4-6 weeks"
**Scope decision, stated explicitly:** the critique calls for an automatic timer. This plan makes Era start/end an explicit admin action instead, via a new `routers/admin/game.py` (`require_admin`, mirroring the existing `routers/admin/*.py` package pattern) with `POST /admin/game/era/start` (`{name, duration_days}`) and `POST /admin/game/era/end`. Rationale: a silent, unattended production reset of a live leaderboard is exactly the kind of surprising, hard-to-reverse action this project's own operating principles call for caution on — an admin should be present to handle the support/communication fallout of a reset, not have it fire while nobody is watching. If a genuinely scheduled cadence is wanted later, it can be added as a `BaseService` (the codebase already has the exact machinery for "check a condition once a tick and act" — see `services/CLAUDE.md`) that calls the same admin-triggered functions on a timer; that is a small, separate addition once the manual flow has been exercised at least once in production.
- `store/era.py::start_era(name, duration_days) -> dict` — raises if an era is already active; inserts the `game_eras` row with `active=1`; bulk `UPDATE game_farms SET era_coins=0, era_harvests=0, era_joined_at=:now` (a genuine bulk reset, but of the new shadow counters only, never `coins`/`prestige`).
- `store/era.py::end_era() -> dict` — raises if none active; computes `era_score` for every farm via the same in-memory scan pattern as `leaderboard()`, writes one `game_era_results` row per participating farm (`era_coins_final > 0` or `era_harvests > 0`), awards `ERA_REWARD_STARS_BY_RANK` stars to the top 10 (added to `stars`, which survives forever, exactly like a Refactor's `stars_for_refactor` award) and grants each top-10 farm one Era-exclusive cosmetic (Phase 2c's `Cosmetic.era_key` field, populated for a couple of new catalog entries at this point), sets the era row `active=0`.
### Era-exclusive crops
Reusing the `Crop.era_key` field added in Phase 3: `unlocked_crops` additionally filters `crop.era_key is None or crop.era_key == active_era_name` — when no era is active (the default, forever, until an admin opts in) this filter is a no-op since every existing/new crop from Phases 1-4 has `era_key=None`.
### Route / schema / frontend
`GET /game/leaderboard?board=era` (folds into the Phase 4 dispatch map; a no-op / empty list when no era is active — never a 404 or error, so existing UI code paths that always render the select stay safe). `GameFarmOut` gains `era_active: bool = False`, `era_name: str = ""`, `era_ends_at: str = ""`, `era_coins: int = 0`, `era_harvests: int = 0` (all zero/false when dormant). New admin template `templates/admin/game.html` (era start/end form + live saturation/upkeep stats dashboard, read-only aside from the two era actions) following the existing admin package template conventions.
### Phase 5 tests
Unit: `era_score` formula, `ERA_REWARD_STARS_BY_RANK` payout math. API: full lifecycle — start era, harvest (era_coins increments, real coins also increment normally), end era (results row written, stars awarded, counters zeroed, real coins/prestige unchanged — assert this last point explicitly, it is the whole safety contract of the phase). E2E: admin start/end flow, era banner appears on the game page while active.
---
## Phase 6 — Engagement loops
### Underdog raid bonus (lowest risk, ship first within this phase)
New `game_farms` column `underdog_boost_until=""` (str timestamp).
`economy/rewards.py`: `UNDERDOG_COIN_RATIO = 10`, `UNDERDOG_DURATION_HOURS = 24`, `UNDERDOG_MULTIPLIER = 1.25`.
In `store/actions.py::steal()`, after a successful steal: if `owner_farm["coins"] > thief_farm["coins"] * economy.UNDERDOG_COIN_RATIO`, set the thief's `underdog_boost_until = now + 24h` via the same update call. In `harvest()`/`_auto_harvest()`, compute `underdog = now < parse(farm["underdog_boost_until"])` and pass it into `effective_reward_coins` as one more optional multiplicative factor (same pattern as `market_factor`).
`GameFarmOut` gains `underdog_boost_seconds_remaining: int = 0`; `_game_grid.html`/`GameFarm.js` render a small HUD banner when it's positive. New badge `underdog_raid` -> `[(1, "David vs Goliath")]`, `track_action` at the steal success point when the boost was just granted.
### Weekly contracts (consolidates Section 2's "Legacy Contracts" and Section 6's "weekly contracts" into the existing quest engine, per the Phase 3 DRY note)
Extend `game_quests` (ensure-block in `database/schema.py`) with one new column: `scope="daily"` (str, default `"daily"` — **every existing row and every new daily row keeps this default, so nothing about today's 3-a-day quest behavior changes**).
`economy/quests.py::weekly_contract(user_uid: str, iso_week: str) -> dict` — same deterministic sha256-seeded pick as `daily_quests`, but seeded on `f"{user_uid}:{iso_week}"`, one contract only, rewards Stars instead of coins/xp, plus a `contract_boost_multiplier` similar in shape to the underdog boost.
`store/quests.py::ensure_quests` gains a second call, gated on `farm["mastery_contracts"]`, that materializes a `scope="weekly"` row for the current ISO week if the Mastery upgrade is owned and none exists yet. `claim_quest` already claims by `kind`; extend its lookup to also match `scope` so a weekly and a daily quest of the same `kind` can never collide (add `scope: str = "daily"` to `GameQuestForm`, default preserves existing calls).
### Alliance / Farm Coop — explicitly deferred, sketched only
**Scope decision, stated explicitly:** this is qualitatively different from every other item in this plan — it is a new multiplayer social feature (group membership, shared resource pooling, coop-only buildings, presumably invite/leave/kick UX and its own moderation surface) rather than an economy-tuning change. Cramming it into the same execution pass as five economy systems and a season reset risks under-designing the social/UX side, which deserves its own focused plan and review. It is recommended as a **separate follow-up plan**, not part of this one's "execute at once" scope. For continuity, the minimal future shape: `game_coops` (uid, name, owner_uid, created_at), `game_coop_members` (coop_uid, user_uid, joined_at) — additive, no existing table touched, consistent with every other phase's backward-compatibility posture.
### Notification polish
**"Crops ready" ping:** implemented entirely client-side in `GameFarm.js`'s existing 1-second ticker — when a plot's countdown reaches zero while the tab is open, fire a toast (existing `AppToast`) and, if the user has granted the browser Notification permission (feature-detect, do not prompt automatically), a desktop `Notification`. No backend change, no new scheduled check — this respects the "no background tick" invariant exactly, since nothing server-side needs to detect the transition.
**"Someone is scouting you" ping — explicitly not built.** Scoped out for the same reason as Phase 2's Observability real-time alert: there is no scouting/detection mechanic in the game today (a raid is a single atomic action, not a two-phase "look then strike"), and fabricating one to support a notification is out of scope for this plan. If a scouting mechanic is wanted, it needs its own design pass first.
### Phase 6 tests
Unit: underdog trigger threshold, weekly-contract seeding determinism. API: underdog boost granted/applied/expires; weekly contract claim independent of daily contracts of the same kind. E2E: underdog banner renders; client-side ready notification fires (Playwright can assert the toast, desktop `Notification` mocking is optional).
---
## Scope decisions and deviations — summary
| Critique ask | What ships | Why |
|---|---|---|
| Saturation "from existing harvest logs" | New lightweight `game_market_ticks` table | No harvest log exists today; adding one is unavoidable and kept minimal (hourly buckets, not per-event rows) |
| Saturation counts "Kernel harvests" generally | Only owner harvests/auto-harvests count, not steals | Steals move already-produced value, they don't add new supply — avoids double-counting |
| Observability: "see raid attempts in real time" | Raises the steal-fraction floor instead | No scouting/detection mechanic exists; inventing one is a separate feature |
| Upkeep "proportional to total wealth or prestige" | `max(flat_fee, coins * 0.2%)` per day | Literal implementation of the ask — this is what makes it a real whale sink |
| "Legacy Contracts" (Mastery) + "weekly contracts" (engagement) | One mechanism: a Mastery-gated weekly slot in the existing `game_quests` engine | Avoids two redundant long-goal systems |
| "Best raid efficiency (coins stolen / raids attempted)" | Avg coins per **successful** raid, min 3 raids to qualify | Failed attempts are never persisted today; adding that write is a spam vector for no real value |
| Era reset "every 4-6 weeks" (automatic) | Admin-triggered start/end | An unattended live-leaderboard reset in production is exactly the kind of action this project's operating principles ask to confirm first; the manual flow can be put behind a scheduler later once exercised once |
| "Alliance / Farm Coop" | Deferred to its own follow-up plan; minimal future table shape sketched | Qualitatively a new multiplayer social feature, not an economy change — deserves its own UX design pass |
| "Someone is scouting you" notification | Not built | No scouting mechanic exists anywhere in the game; would require inventing one first |
| "Crops ready" notification | Client-side only, tied to the existing ticker | Respects the "no background tick" hard architectural invariant |
---
## Definition of done, per phase
1. `python -c "from devplacepy.main import app"` imports clean.
2. `grep` every touched file for em-dash (character and HTML entity) — none present, hyphens only.
3. Every new `game_farms`/`game_plots`/`game_*` column has a safe zero-equivalent default and is read through `store/common.py::_lvl()` or an equivalent explicit `or 0`/`or ""` guard, never a bare `farm["new_column"]`.
4. Every new mutating route: appears in `routers/game/index.py` or `farm.py` guarded the same way its neighbors are (`require_user`), returns via the shared `_respond_action`/`GameFarmViewOut` pattern, has a Devii `Action` with matching `requires_auth`, has a `docs_api` entry, and — if it grants an achievement — a `BADGE_CATALOG`/`ACHIEVEMENTS` entry plus a `track_action` call.
5. `devplacepy/services/game/CLAUDE.md` updated with the new mechanic in the same terse, precise style as the existing sections (this file currently documents 5 "extended mechanics" plus the "endgame" section — new phases extend that same running list, they do not replace it).
6. `README.md` updated if the phase adds a user-visible feature.
7. New tests written in the correct tier (`tests/unit/services/game/`, `tests/api/game/`, `tests/e2e/game/`) — not run automatically; the user runs `make test` (or the relevant tier target) before promoting to production.
8. A manual smoke pass on a seeded test farm confirms: a legacy farm (created before the phase shipped) loads without error and behaves exactly as before until it opts into the new mechanic.

875
ref.md
View File

@ -1,875 +0,0 @@
# ref.md - Monster-file split plan (backend)
Author: retoor <retoor@molodetz.nl>
This document is the execution plan for splitting every backend "monster" file in
DevPlace into a package of cohesive modules, without changing behavior and without
breaking any importer or test. It is investigation output only; nothing here has
been executed. Each section is self-contained and can be executed independently in
the recommended order (Section 12).
---
## 1. Scope and definition
"Monster" = first-party backend Python file over 900 lines (excludes vendored JS,
generated artifacts, and the scratch/workspace copies under `data/`, `var/`, `tmp/`,
and the repo-root `maak.py`/`d.py`). The confirmed set:
| # | File | Lines | Split kind |
|---|------|------:|------------|
| 1 | `devplacepy/docs_api.py` | 5659 | package: one module per API group |
| 2 | `devplacepy/database.py` | 3764 | package: layered domain modules |
| 3 | `devplacepy/services/bot/bot.py` | 2999 | in-package: cooperative mixins |
| 4 | `devplacepy/services/containers/files/bot.py` | 2940 | SPECIAL - deployed single-file asset |
| 5 | `devplacepy/services/containers/files/d.py` | 2337 | SPECIAL - deployed single-file asset |
| 6 | `devplacepy/services/devii/actions/catalog.py` | 2036 | same-name package: one module per action domain |
| 7 | `devplacepy/schemas.py` | 1285 | package: domain schema modules |
| 8 | `devplacepy/cli.py` | 1080 | package: one module per command group |
| 9 | `devplacepy/utils.py` | 1024 | package: layered helper modules |
| 10 | `devplacepy/services/news.py` | 986 | package: pipeline-stage modules |
| 11 | `devplacepy/services/devii/session.py` | 962 | same-name package: extract stateless text/helpers |
| 12 | `devplacepy/services/game/store.py` | 948 | subpackage: cohesive store modules |
`services/bot/llm.py` (792) is below threshold and holds two realism mechanics; it is
explicitly **left intact** (Section 3.4).
---
## 2. Global rules for every split (apply to all sections)
These are invariants. Every section assumes them; deviations are called out per file.
1. **Same import path.** Convert `foo.py` into a package `foo/` (directory with
`__init__.py`). Because Python resolves `foo/__init__.py` for `import ...foo`, every
existing `from ...foo import X` and `import ...foo as m; m.X` keeps resolving with no
caller edit. This is the ONLY compatibility mechanism used.
2. **Exhaustive re-export in `__init__.py`.** The package `__init__` must re-export the
complete public surface the codebase and tests currently import - **including
underscore-prefixed names that are imported elsewhere** (they are part of the effective
surface; e.g. `database._now_iso`, `utils._resolve_user`, `docs_api._substitute`,
`store._today`). Prefer **explicit** `from .module import name1, name2` blocks over
`from .module import *` so the surface is auditable and star-import name-drops
(underscore filtering, `__all__` gaps) cannot silently regress it.
3. **No behavior change.** Pure structural moves. No logic edits, no reordering of
ordered data, no renames of public symbols, no signature changes.
4. **Preserve every function-local (lazy) import.** Many files defer imports inside
function bodies specifically to break import cycles (`database` <-> `utils`,
`utils` <-> `services.audit`/`templating`/`content`, `news` AI-key, game `leaderboard`).
These MUST stay function-local in the new modules. Hoisting a lazy import to module
top is the single most likely way this refactor introduces a boot-time cycle.
5. **File header.** Every new file starts on line 1 with `# retoor <retoor@molodetz.nl>`
(project rule). Carry `from __future__ import annotations` into any module that uses
`X | None` style annotations where the original had it.
6. **No em-dash.** Neither the em-dash character nor `&mdash;`/`&#8212;`/`&#x2014;`
anywhere in new files. Hyphen only. Grep every new file before finishing.
7. **No forbidden names / no comments / no docstrings in source** (project rules): do not
introduce `_new`/`_old`/`_temp`/`_v2`/`better_`/`my_`/`the_`, do not add comments or
docstrings.
8. **`_shared.py` is the sanctioned package-private helper module.** The project already
uses `_shared.py` in eight router packages for cross-leaf helpers. Use that exact name
for genuinely package-private construction helpers (docs_api builders, catalog builders,
cli `_audit_cli`). Everything else is a normal named module.
9. **Validation after each split (never run the test suite):**
- `python -c "from devplacepy.main import app"` must import clean.
- `python -m py_compile` every new file.
- A public-surface diff: capture `sorted(n for n in dir(<module>) if not n.startswith('__'))`
BEFORE the split (from git or a saved snapshot) and assert the post-split package
exposes a superset. This is the definitive check that no re-export was missed.
- em-dash grep over every new file.
- The suite is run only if the user explicitly asks; the plan never runs it.
---
## 3. Per-file plans
### 3.1 `docs_api.py` (5659) -> `devplacepy/docs_api/`
**Nature.** A flat declarative module: a few construction helpers, one giant
`API_GROUPS` list literal (~5240 lines, `docs_api.py:143-5384`, 19 group dicts), then
a post-processing/registry/render tail. Endpoints are organized by API group; each group
is a self-contained dict literal built via the `endpoint()` factory.
**Public surface (only these are imported externally):**
`API_GROUPS`, `build_services_group`, `get_group`, `api_doc_pages`, `render_group`, and
`_substitute` (underscore but called by `docs_export.py:119`).
Importers: `docs_search.py:9,87,90`; `docs_export.py:10,118,119,121`;
`services/xmlrpc/registry.py:8,63`; `routers/docs/pages.py:3`; `routers/docs/views.py:14`.
Tests: `tests/e2e/docs.py:9`, `tests/unit/project_files.py:381-385`,
`tests/api/auth/matrix.py:7`, `tests/api/docs/{index,downloadhtml,downloadmd}.py`.
**Target layout:**
```
devplacepy/docs_api/
__init__.py # assembles API_GROUPS in order, runs the negotiation pass,
# re-exports the public surface (incl. _substitute)
_shared.py # field, endpoint, ROLE_LABELS, NON_BODY_ENDPOINTS, and the
# domain constant lists (VOTE_TARGETS, REACTION_TARGETS,
# BOOKMARK_TARGETS, COMMENT_TARGETS, PROJECT_TYPES,
# GIST_LANGUAGES, SERVICE_ACTIONS)
negotiation.py # _PAGE_RESPONSES, _ACTION_RESPONSES, _classify,
# _apply_negotiated_responses(groups) (schemas/docs_examples live here)
services_group.py # _service_control_endpoints, _field_to_param, _service_section,
# build_services_group
render.py # _GROUPS_BY_SLUG, get_group, api_doc_pages, _substitute, render_group
groups/
__init__.py # ORDERED_GROUPS = [conventions.GROUP, auth.GROUP, ... game.GROUP]
conventions.py # docs_api.py:145-293
auth.py # 295-427
lookups.py # 428-484
social_actions.py # 485-674
content.py # 675-1615 (largest group, ~940 lines)
profiles.py # 1616-2294
messaging.py # 2295-2365
notifications.py # 2366-2450
uploads.py # 2451-2552
project_files.py # 2553-3098
containers.py # 3099-3673 (admin: True)
tools.py # 3674-3902
push.py # 3903-3964
issues.py # 3965-4258
gateway.py # 4259-4449 (admin: True)
services.py # 4450-4456 (dynamic placeholder group dict - keep hollow)
admin.py # 4458-5193 (admin: True, ~735 lines)
game.py # 5194-5383
```
Each `groups/*.py` holds one `GROUP = {...}` literal importing `field`, `endpoint`,
`ROLE_LABELS` and the constant lists from `.._shared` (and `constants.TOPICS`/
`REACTION_EMOJI` where the literal uses them - re-scan each block for free names before
cutting).
**`__init__.py` must, in this exact order:**
1. `from .groups import ORDERED_GROUPS as API_GROUPS` (order is load-bearing).
2. `from .negotiation import _apply_negotiated_responses` then call
`_apply_negotiated_responses(API_GROUPS)` - reproduces the current import-time mutation
at `docs_api.py:5494`. Must run AFTER `API_GROUPS` is assembled.
3. Re-export `API_GROUPS`, `build_services_group`, `get_group`, `api_doc_pages`,
`render_group`, `_substitute` (and, for safety, `endpoint`, `field`, `ROLE_LABELS`,
the constant lists).
**Risks / ordering hazards (must be handled):**
- **Import-time negotiation mutation.** `_apply_negotiated_responses()` runs at module top
today (`docs_api.py:5494`) and mutates each endpoint dict in place (fills
`sample_response`, stamps `negotiation`). Centralize this in `__init__` after assembly;
if a group is read before the pass runs, endpoints lack `negotiation`/`sample_response`.
- **Forward reference.** `_classify` (`105-112`) reads `_PAGE_RESPONSES`/`_ACTION_RESPONSES`
defined 5280 lines later. Colocate `_classify` + both maps + the pass in `negotiation.py`
to make the dependency explicit.
- **Assembly-before-render cycle.** `render.py` (`get_group`/`render_group`/`api_doc_pages`/
`_GROUPS_BY_SLUG`) needs the final `API_GROUPS`. Build `API_GROUPS` in `groups/__init__.py`
and import it into both package `__init__` and `render.py`; do NOT let `render.py` import
the package `__init__` at module top (cycle).
- **`build_services_group` reuses core helpers** (`endpoint`, `field` via `_field_to_param`,
`_classify`, `_service_control_endpoints`) and stamps `negotiation` itself for its runtime
group; `services_group.py` imports `endpoint`/`field` from `_shared` and `_classify` from
`negotiation`; do NOT double-apply negotiation to it.
- **`_substitute` name must be preserved** in the re-export (called as
`docs_api._substitute` by `docs_export.py:119`).
- **Two groups slug `services`** (the hollow placeholder in the list + the runtime
`build_services_group` output). Keep the placeholder exactly where it is in the ordered list.
**Test invariant:** `API_GROUPS` order/content, group flags (`slug/title/admin/dynamic/
endpoints`), and every endpoint's post-negotiation shape (`id/method/path/auth/min_role/
params/negotiation/sample_response`) must be identical after the split. `tests/api/auth/
matrix.py` asserts `len(endpoints) >= 50` and drives each endpoint's documented auth against
live behavior; `tests/unit/project_files.py:381-385` asserts the `project-files` group's exact
endpoint id set.
---
### 3.2 `database.py` (3764) -> `devplacepy/database/`
**Nature.** The core data layer, imported nearly everywhere (677 `from ...database import`
statements across 329 files; hottest names `get_table` x404, `refresh_snapshot` x92,
`set_setting` x67, `db` x33). The `__init__` MUST re-export essentially the entire namespace
including underscore helpers (`_now_iso` x6, `_index`, and whatever tests reach for).
**The linchpin.** A single `core.py` owns the `db` singleton, the import-time connection
side effects, the cross-worker cache-version primitives, index helpers, and trivial
accessors. Every other module imports only from `core` (and strictly-lower layers). Because
`core.py` is imported once, Python's module cache guarantees the single `db` object, exactly
as today.
**Target layout (`devplacepy/database/`):**
```
core.py # db, refresh_snapshot; import-time ensure_data_dirs + sqlite mkdir +
# dataset.connect (database.py:20-42); cache-version primitives
# (_ensure_cache_state, get_cache_version, bump_cache_version,
# sync_local_cache, _local_cache_versions, _cache_version_cache,
# _cache_state_ready); _index, _drop_index, _uid_index;
# get_table, _in_clause, _now_iso <- imported by ALL
settings.py # _settings_cache, get_setting, get_int_setting, set_setting,
# clear_settings_cache, internal_gateway_key
users.py # get_users_by_uids, search_users_by_username, set_user_timezone,
# admins cache (_admins_cache, invalidate_admins_cache,
# get_admin_uids, get_primary_admin_uid)
relations.py # user block/mute cache + accessors (get_user_relations, get_blocked_uids,
# get_muted_uids, get_silenced_uids, invalidate_user_relations)
pagination.py # PAGE_SIZE, build_pagination, paginate, paginate_diverse,
# interleave_by_author, get_user_post_count (imports core + relations)
soft_delete.py # SOFT_DELETE_TABLES, ensure_soft_delete_columns, soft_delete,
# soft_delete_in, restore, purge, list_deleted, count_deleted,
# restore_event, purge_event (imports core + pagination.build_pagination)
engagement.py # comment-count/vote/reaction/bookmark batch helpers, polls
comments.py # _drop_blocked, _build_comment_items, load_comments, recent/by-target loaders
attachments_data.py# get_attachments*, get_news_images_by_uids, delete_attachment*,
# _delete_attachment_file, get_user_media, get_deleted_media
usage.py # _add_usage, _get_usage, add/get_*_usage + *_USAGE_KEY constants
seo_meta.py # SEO_META_TYPES + seo_metadata CRUD
activity.py # user_activity counters + calendar/heatmap/streaks
ranking.py # VOTABLE_TARGETS, STAR_TARGETS, authors/leaderboard/stars,
# update_target_stars, get_target_owner_uid, soft_delete_engagement,
# delete_engagement
customization.py # customization prefs + override CRUD
email.py # email account CRUD
notifications.py # notification prefs + mark_notifications_read_by_target
forks.py # record_fork, get_fork_parent, count_forks, *_fork_relations
content.py # resolve_by_slug, resolve_object_url, text_search_clause,
# get_uids_by_username_match, get_daily_topic, get_featured_news
follows.py # get_follow_counts, get_follow_list, get_following_among
stats.py # get_site_stats, get_platform_analytics, get_gist_languages
deepsearch.py # _ds_now + deepsearch session/message/url-cache helpers
schema.py # init_db, _refresh_query_planner_stats, migrate_bug_tables_to_issue_tables,
# BUG_TABLE_RENAMES, migrate_ai_gateway_settings, OLD_GATEWAY_URL,
# backfill_api_keys, _backfill_gamification <- imports EVERYTHING; loaded last
__init__.py # explicit re-export of every module's public + test-touched underscore names
```
**Where shared state lives (critical).** The only truly global object is `db` (+ the
cache-version machinery), which lives in `core.py`. Every domain `TTLCache`/dict singleton
(`_admins_cache`, `_settings_cache`, `_relations_cache`, `_customizations_cache`,
`_notification_prefs_cache`, `_stats_cache`, `_analytics_cache`, `_activity_cache`,
`_gist_languages_cache`, `_authors_cache`, `_comment_count_cache`) is referenced only by its
own domain, so it moves WITH that domain's module - no shared-cache problem. Never re-create
a cache in `__init__`; only re-export. Two import paths to one cache would create two caches.
**Layering (acyclic).** `core` <- everything; `pagination -> relations`;
`soft_delete -> pagination`; `comments -> users + engagement + relations`;
`ranking -> users`; `stats -> ranking`; `attachments_data -> content`; `content -> users`;
`schema` imports any sibling at module top (nothing imports `schema`; loaded last in `__init__`).
**Cycle-avoidance rules:**
- `core.py` imports nothing from siblings.
- Strict layer order: `core -> settings/users/relations -> pagination -> soft_delete ->
domain modules -> schema`.
- **Preserve every existing function-local import verbatim** - the `utils`, `attachments`,
`templating`, `services.audit.store`/`services.backup.store`/`services.openai_gateway.routing`
imports inside functions (e.g. `init_db` at `database.py:951-961`) exist to break the real
`database <-> utils` cycle. Do not hoist them.
- `utils.py` uses `database`'s cache primitive under the `"auth"` name; those primitives are in
`core.py` and remain reachable as `database.bump_cache_version`/`database.sync_local_cache`
via re-export.
**Risks:**
- **`init_db()` (503-1534) is one idempotent ordered routine** - move it WHOLE into `schema.py`;
do not reorder its internal steps (migration-before-index, backfills/ANALYZE last at 1516-1519).
- **Import-time side effects (20-42)** must live in `core.py` and run exactly once; `core` must
be the first thing `__init__` imports.
- **Underscore re-export gap** is the highest-probability regression (`import *` drops `_now_iso`/
`_index`). Use explicit re-exports or per-module `__all__` including the imported underscores.
- **`get_table` (404 importers) and `refresh_snapshot` (92)** - any miss breaks a huge fraction of
the app at import; the public-surface diff (Section 2.9) is the guard.
**Test footprint.** `tests/unit/database.py` is the primary direct consumer (it also hits a live
server, so it is effectively api-tier); `tests/unit/services/gitea.py` and
`tests/unit/services/openai_gateway/analytics.py` import it for setup; dozens of api/e2e files use
`get_table`/`set_setting`/usage helpers indirectly. With a complete re-export, zero test edits are
needed.
---
### 3.3 `services/bot/bot.py` (2999) -> cooperative mixins in `services/bot/`
**Nature.** ONE class `DevPlaceBot` (`bot.py:48`), ~90 methods, every method reads/writes
shared `self` state (`self.state` persisted `BotState`, `self.b` browser, `self.llm`,
`self._session_*` counters, the `_post_cache/_project_cache/_issue_cache/_session_history`
caches). The only zero-behavior-change decomposition is **cooperative mixins** (Python MRO gives
byte-identical `self` semantics; the unbound-method unit test still resolves via inheritance).
There is NO `@tool` registry in this file (the closest analogue is the AI decision/menu engine).
**Target layout (new modules beside the existing `services/bot/` files):**
| New module | Mixin | Moves from bot.py | Contents |
|------------|-------|-------------------|----------|
| `helpers.py` | `BotHelpersMixin` | 103-245 | instrumentation, cost/session banners, `_thread_id`, `_pick_article`, cache warm |
| `auth.py` | `BotAuthMixin` | 246-355 | handles, `_ensure_auth`, account-key adoption |
| `human.py` | `BotHumanMixin` | 356-522 | read/type/fatigue simulation, session type, page inspection |
| `agent.py` | `BotAgentMixin` | 523-757 | AI decision/menu/plan engine (`_build_menu`, `_run_plan`, `_cycle_ai`, dispatch, nav) |
| `engage.py` | `BotEngageMixin` | 753-1436 | voting, commenting, reacting (realism #4b) |
| `posting.py` | `BotPostingMixin` | 1438-1770 | gist + post creation, content open/click (realism #1, #3 call-sites) |
| `social.py` | `BotSocialMixin` | 1771-2378 | profile/project/issue/follow, notifications, mentions (#4b), messaging |
| `browse.py` | `BotBrowseMixin` | 2379-2502 | scroll, nav, browse, search, `_engage_community` |
| `loop.py` | `BotLoopMixin` | 2503-2999 | `_cycle`, `run_forever` (scheduling) |
`bot.py` shrinks to ~110 lines: the mixin imports, `__init__`, and:
```python
class DevPlaceBot(
BotHelpersMixin, BotAuthMixin, BotHumanMixin, BotAgentMixin,
BotEngageMixin, BotPostingMixin, BotSocialMixin, BotBrowseMixin, BotLoopMixin,
):
...
```
`DevPlaceBot` stays importable from `devplacepy.services.bot.bot` (do NOT move the class file;
only lift methods into inherited mixins). Method names are already globally unique across the
90 methods - verify no collision when composing.
**Realism contract (MUST NOT regress) - exact locations to preserve:**
1. Title rewrite: `LLMClient.generate_post`/`generate_post_title` (in `llm.py`, untouched);
call-sites `bot.py:205,211,1571,1577,1597` move into `helpers.py`/`posting.py` unchanged.
2. Persona topic spread: `persona_article_score`/`pick_category` (`config.py`, untouched);
call-sites in `helpers.py`/`posting.py`.
3. Gist quality gate: `LLMClient.gist_quality_check`/`quality_check` (`llm.py`, untouched);
enforced in `posting.py` (`_create_gist`) and `engage.py` (`_comment_on_post`).
4. Inter-bot threads: `ArticleRegistry` (own module, untouched) PLUS in-bot taper
(`state.thread_reply_counts`/`MAX_THREAD_REPLIES` `bot.py:2163-2166,2272-2277`,
`MAX_SIBLING_COMMENTS` `912,997-1020,1155-1164`, `_respond_to_mention` 2143,
`_reply_to_random_comment` 1100, `_spoke_last_in_thread` 2305) - keep this logic together
and unchanged across `engage.py`/`social.py`.
**`__init__.py` - keep exporting ONLY `BotsService` (lazy-Playwright boundary).** `service.py`
lazy-imports `bot.py` and treats `ImportError` as "install the bots extra". `bot.py` imports
`faker` and `BotBrowser` (Playwright) at module top. If the package `__init__` eagerly imported
`DevPlaceBot`, importing `devplacepy.services.bot` (for `BotsService`, used by `main.py`) would
hard-fail on boxes without the extra. So: leave `__init__.py` exporting only `BotsService`; keep
`DevPlaceBot`/`LLMClient` importable from their own modules (all live importers already use those
paths - `service.py:263`, `bot.py:39,59`).
**Stealth rule - already compliant, do not regress.** The only outbound HTTP is
`LLMClient._raw_call` (`llm.py:69`) via `stealth.stealth_sync_client`. `bot.py` makes no bare
`httpx` calls (all page "fetch" is inside Playwright). No new module may introduce a bare
`httpx.Client`.
**Test footprint.** `tests/unit/services/bot/bot.py` imports `bot as bot_module` and
`DevPlaceBot`, calling `DevPlaceBot._adopt_account_api_key(fake)` as an unbound method - safe
under mixins (method stays an attribute via inheritance). Admin fleet api/e2e tests go through
`BotsService`, unaffected.
#### 3.4 `services/bot/llm.py` (792) - LEAVE INTACT (explicit decision)
Below the 900-line threshold, a single cohesive `LLMClient` with shared `_call`/cost counters,
and it holds two realism mechanics (title rewrite, gist gate). A mixin split would be
net-negative and risk the contract. Keep it whole. (If ever reduced later, the only safe cut is
lifting the static text cleaners `clean`/`strip_md`/`strip_label`/`strip_code_fences`/`_parse_json`/
`_clamp01`/`_as_int`/`_as_terms`/`_overlap_ratio` into an `llm_text.py` of module functions that
`LLMClient` staticmethods delegate to, preserving `LLMClient.clean` as a public alias. Not part of
this plan.)
---
### 3.5 `services/devii/actions/catalog.py` (2036) -> same-name package `catalog/`
**Nature.** NOT the `Catalog`/`Action` classes (those live in `actions/spec.py`) and NOT the
live `CATALOG` (that is `registry.py:26-46`, `Catalog(actions=ACTIONS + TASK_ACTIONS + ...)`,
with `_assert_confirm_params(CATALOG)` an import-time side effect at `registry.py:68`). This file
only defines `ACTIONS` (161 `Action` literals, `catalog.py:43-2034`), five private builder helpers
(`path/query/body/upload/confirm`, `8-37`), two description constants (`ATTACHMENTS`, `TARGET_TYPE`,
`40-41`), and `PLATFORM_CATALOG = Catalog(actions=ACTIONS)` (`2036`). No logic.
**Public surface:** `ACTIONS` and `PLATFORM_CATALOG` only. Importers: `actions/__init__.py:3,7`,
`registry.py:9,27`, and tests `tests/unit/project_files.py:79,366,368`, `tests/unit/schemas.py:117,
123,132`, `tests/unit/services/devii/actions/dispatcher.py:6`,
`tests/unit/services/devii/actions/catalog.py:3`, `tests/api/projects/files/{lines,replacelines}.py:79`.
**Target layout (`services/devii/actions/catalog/`):**
```
__init__.py # imports each domain tuple, ACTIONS = (AUTH_ACTIONS + POSTS_ACTIONS + ...),
# PLATFORM_CATALOG = Catalog(actions=ACTIONS); __all__ = ["ACTIONS","PLATFORM_CATALOG"]
_shared.py # path(), query(), body(), upload(), confirm(), ATTACHMENTS, TARGET_TYPE
auth.py # AUTH_ACTIONS (catalog.py:44-105)
posts.py # POSTS_ACTIONS (106-185, incl. view_feed)
comments.py # COMMENTS_ACTIONS (186-216)
projects.py # PROJECTS_ACTIONS (217-342)
project_files.py # PROJECT_FILE_ACTIONS (343-511)
jobs.py # JOB_ACTIONS (zip+fork) (512-578)
tools.py # TOOLS_ACTIONS (579-679)
profile.py # PROFILE_ACTIONS (680-748)
messages.py # MESSAGE_ACTIONS (749-776)
notifications.py # NOTIFICATION_ACTIONS (777-810)
engagement.py # ENGAGEMENT_ACTIONS (811-880)
social.py # SOCIAL_ACTIONS (881-976, incl. leaderboard)
issues.py # ISSUE_ACTIONS (977-1122)
gists.py # GIST_ACTIONS (1123-1190)
news.py # NEWS_ACTIONS (1191-1211)
uploads.py # UPLOAD_ACTIONS (1212-1246)
admin.py # ADMIN_ACTIONS (1247-1661, ~415 lines; optional further sub-split)
dbapi.py # DBAPI_ACTIONS (1662-1778)
gateway.py # GATEWAY_ACTIONS (1779-1874)
game.py # GAME_ACTIONS (1875-2033)
```
Each domain module does `from ._shared import path, query, body, confirm, ATTACHMENTS, TARGET_TYPE`
and `from ..spec import Action`. **`dbapi.py`, `gateway.py`, `game.py` must ALSO
`from ..spec import Param`** - they construct `Param(...)` directly for boolean/integer/number
params. `admin.py` at ~415 lines may optionally be sub-split into `admin_users.py`/`admin_services.py`/
`admin_backups.py`/`admin_analytics.py`, but a single `admin.py` is already a large improvement.
**Risks:**
- **Action ordering is behavior.** `Catalog.tool_schemas()` emits schemas in `self.actions` order
(the LLM tool-list order). The `__init__` concatenation must reproduce the exact source order in
the table above. Verify `[a.name for a in ACTIONS]` is identical before/after (order + set).
- **Import-time singleton + validation.** `PLATFORM_CATALOG` builds at import; `registry.py:68`
runs `_assert_confirm_params(CATALOG)` at import. `ACTIONS` must be fully assembled when the
package `__init__` finishes. Domain modules import only from `.._shared`/`..spec` (leaf-level, no
cycle).
- **`confirm`-param invariant.** `registry._assert_confirm_params` (`49-65`) requires every name in
`dispatcher.CONFIRM_REQUIRED`/`CONDITIONAL_CONFIRM` to declare a `confirm` param; dropping any
`confirm()`/inline-confirm param makes import `RuntimeError`. Watch the inline
`body("confirm", ...)` cases at `catalog.py:310-314,336-340,727-731`.
Nothing outside the package changes; `actions/__init__.py` and `registry.py` keep their exact
imports.
---
### 3.6 `schemas.py` (1285) -> `devplacepy/schemas/`
**Nature.** Pure Pydantic `*Out` models, all subclassing one shared base `_Out`
(`schemas.py:10-11`). One import-time side effect: `CommentItemOut.model_rebuild()` at
`schemas.py:1023` (resolves the self-referential `children` forward ref). No runtime logic.
48 files import from `devplacepy.schemas` (all `from ...import X`), each router pulling the models
it feeds to `respond(..., model=XOut)`; the `__init__` must re-export ALL public model names.
**Model dependency graph (drives module order).** `UserOut` is depended on by nearly every
wrapper; `AttachmentOut/VotesOut/ReactionsOut/PollOut/BadgeOut/CommentItemOut` feed the item
wrappers; cross-module inheritance: `AdminUserOut(UserOut)`, `AdminMediaItemOut(MediaItemOut)`,
`ProjectListItemOut(ProjectOut)`, `ProjectFileRawOut(ProjectFileOut)`. So the shared primitives
live in `content.py`/`profile.py`, imported first by the domain modules.
**Target layout (`devplacepy/schemas/`):**
```
__init__.py # explicit re-export of every *Out (+ ActionResult, ErrorOut, ValidationErrorOut)
base.py # _Out + generic envelopes (ActionResult, ErrorOut, ValidationErrorOut)
content.py # UserOut, AttachmentOut, VotesOut, ReactionsOut, Poll*, BadgeOut, PostOut, GistOut,
# ProjectOut, ProjectFile*, NewsOut, CommentOut, CommentEditOut, CommentItemOut,
# NotificationOut, MessageOut (+ CommentItemOut.model_rebuild() at bottom)
listings.py # FeedItemOut/GistItemOut/ProjectListItemOut/NewsListItemOut, Conversation,
# MessageItemOut, Notification*Out, Saved*, Leaderboard*, and the page-level
# FeedOut/PostDetailOut/ProjectsOut/ProjectDetailOut/Gists*/News*/Messages*/
# Notifications*/Saved*
profile.py # MediaItemOut, ProfileOut, TelegramPairOut
issues.py # IssueItemOut, IssueCommentOut, IssueDetailOut, IssueAttachmentsOut, IssueJobOut, IssuesOut
containers.py # InstanceOut, ScheduleOut, ContainersOut, AdminContainers*, Bot*
jobs.py # ZipJobOut, PlanningJobOut, ForkJobOut, SeoJobOut, SeoReportOut, SeoMetaOut,
# DeepsearchJobOut, DeepsearchSessionOut, DbQueryJobOut
backups.py # BackupStoragePathOut, BackupOut, BackupJobOut, BackupScheduleOut, BackupDashboardOut
admin.py # AdminUserOut, AdminUsersOut, AdminNews*, AdminSettingsOut, AdminNotificationsOut,
# AdminMedia*, AdminTrashOut/TrashItemOut
gateway.py # GatewayUsageOut, UserAiUsageOut
auth.py # AuthPageOut, Landing*, DeviiPageOut
audit.py # AuditLinkOut, AuditEntryOut, AuditLogOut, AuditEventOut
dbapi.py # DbTableOut..NlQueryOut
game.py # GameCropOut..GameLeaderboardOut
```
Every module does `from devplacepy.schemas.base import _Out` and imports the primitive models it
subclasses/references from `content.py`/`profile.py`. Use explicit re-export (not `*`) so ordering
does not matter and the surface is auditable.
**Risks:**
- **Forward-ref rebuild.** Keep `CommentItemOut` and the whole content-primitive cluster in
`content.py` and call `CommentItemOut.model_rebuild()` at the bottom of that module (after all
referencing types are importable).
- **Cross-module inheritance** requires the base imported before the subclass; per-file
`from ...import` handles it since all base classes live in `content.py`/`profile.py`.
- Models reach routers only via `respond(model=...)`; a dropped export silently degrades JSON
responses (Pydantic drops unknown keys). Re-export completeness is the whole risk.
**Test footprint.** `tests/unit/schemas.py` imports `MediaItemOut`, `AdminMediaItemOut`,
`GameLegacyOut` inside test functions - all must survive re-export.
---
### 3.7 `cli.py` (1080) -> `devplacepy/cli/`
**Nature.** Plain argparse. One `main()` builds the parser tree; each subcommand sets
`func=cmd_*`; `main()` dispatches `args.func(args)`. Console-script entry `pyproject.toml:40`
`devplace = "devplacepy.cli:main"`. Heavy imports are lazy inside each command body (containers,
jobs.queue, backup.store) - this keeps CLI import cheap and MUST be preserved.
**Target layout (`devplacepy/cli/`):**
```
__init__.py # from .main import main; re-export EVERY cmd_* (tests call cli.cmd_role_get etc.)
_shared.py # _audit_cli (cli.py:9-23)
roles.py # cmd_role_get/set + register_roles(sub) (26-61)
apikeys.py # cmd_apikey_* + register_apikeys(sub) (64-106)
tokens.py # cmd_token_* + register_tokens(sub) (109-211)
devii.py # cmd_devii_reset_quota + register_devii(sub) (214-254)
news.py # cmd_news_* + register_news(sub) (257-290)
attachments.py # cmd_attachments_prune + register(sub) (293-315)
jobs.py # zips/forks/seo/seo-meta/deepsearch prune-clear + _remove_*_artifacts (318-576)
backups.py # cmd_backups_* + register_backups(sub) (393-445)
containers.py # cmd_containers_* + register_containers(sub) (579-659)
migrate.py # _crc32/_migrate_*/_db helpers + cmd_migrate_data + cmd_emoji_sync (662-870)
main.py # build_parser()/main(): imports each module's register_* and dispatches;
# holds the `if __name__ == "__main__": main()` guard
```
Each leaf exposes `register_<group>(subparsers)` adding its parser(s) + `set_defaults(func=...)`.
`main.py` calls them in order. `__init__.py` re-exports `main` AND every `cmd_*` because
`tests/unit/cli.py` (`from devplacepy import cli`) calls `cli.cmd_role_get/set`, `cli.cmd_news_clear/
sanitize`, `cli.cmd_attachments_prune`, `cli.cmd_apikey_get/reset/backfill`, `cli.main` directly.
**Risks:**
- **Console-script entry** `devplacepy.cli:main` must keep resolving - satisfied by
`from .main import main` in `__init__.py`.
- **Preserve lazy in-function imports** (avoid import-time cycles / heavy deps). Do not hoist.
- Move the `__main__` guard into `main.py` (or `__init__` calling `main()`) so `python -m devplacepy.cli`
behavior is unchanged.
No production module imports `devplacepy.cli` (only tests). Sibling entrypoints
`devplacepy.services.devii.cli:main` and `devplacepy.services.xmlrpc.server:main` are unaffected.
---
### 3.8 `utils.py` (1024) -> `devplacepy/utils/`
**Nature.** The most import-sensitive file: 224 files import from it; source of several Jinja
globals and the notification/rewards/achievement funnels. Most-used: `make_combined_slug` (144),
`generate_uid` (127), `clear_user_cache` (32), `not_found` (22), `get_current_user` (20),
`require_admin` (19), `require_user` (16), `is_admin` (12), `track_action` (10). Several
underscore helpers are part of the effective surface (`_resolve_user`, `_user_from_session`,
`_user_from_api_key`, `_user_from_devrant_token`) and MUST be re-exported.
**Target layout (`devplacepy/utils/`):**
```
__init__.py # explicit re-export of the FULL public API incl. imported underscore helpers
passwords.py # hash_password/verify_password (+async), create_session (40-70)
authcache.py # _user_cache, _sync_auth_cache, clear_user_cache, clear_session_cache (73-90)
auth.py # _UNSET, _user_from_*, _resolve_user, get_current_user (92-249)
guards.py # safe_next, redirect_back, cookie_secure, client_ip, require_user,
# require_admin, require_user_api, not_found, is_admin, is_primary_admin (252-376)
text.py # strip_html, plain_float, _mark_floats, pretty_json, slugify, generate_uid,
# make_combined_slug, time_ago, format_date, extract_mentions (379-459, 1015-1024)
notifications.py # PUSH_ICON, DEFAULT_PUSH_URL, _push_tasks, _safe_notify, _schedule_push,
# _schedule_telegram, create_notification, _deliver_notification,
# create_mention_notifications, _deliver_mention_notifications (462-575, 989-1013)
badges.py # BADGE_CATALOG, BADGE_GROUPS, LEVEL_BADGES, get_badge, build_achievements,
# award_badge, notify_badge (671-745, 806-848)
rewards.py # LEVEL_XP, XP_*, level_for_xp, award_xp, _COUNT_MILESTONES, _milestone_metrics,
# check_milestone_badges, award_rewards, _apply_rewards, ACHIEVEMENTS,
# UNIQUE_ACTIONS, track_action, _apply_achievements (654-987)
accounts.py # _create_account, register_account, register_account_async (578-641)
```
**Circular-import sensitivity (must preserve exactly):**
- `clear_user_cache` is lazily imported by `services/correction.py:183` and by 32 other files +
`cli.py:74`; it MUST stay importable as `devplacepy.utils.clear_user_cache` via re-export.
- Every cross-cutting lazy import inside utils stays function-local: `is_primary_admin` ->
`database.get_primary_admin_uid` (`328`); `require_user`/`require_admin` -> `responses.wants_json`
+ `services.audit.record` (`303-304,336-337`); `_deliver_notification` -> `templating.clear_unread_cache`
(`531`) + `services.audit` (`557`); `award_xp`/`check_milestone_badges`/`award_badge`/`_create_account`
-> `services.audit`; `_milestone_metrics` -> `database.get_streaks`; `_schedule_telegram` ->
`services.telegram.store`; `_safe_notify` -> `devplacepy.push`; `_user_from_devrant_token`/
`_user_from_access_token` -> `services.devrant.tokens`/`services.access_tokens`. Do NOT hoist any.
- **Cache-name coupling:** the auth cache uses `sync_local_cache("auth", ...)`/`bump_cache_version("auth")`
- do not rename the `"auth"` key.
**Funnel pairing (do not split apart, do not double-wrap):** each public funnel and its
`background.submit` worker must live in the same module: `create_notification`/`_deliver_notification`
and `create_mention_notifications`/`_deliver_mention_notifications` in `notifications.py`;
`award_rewards`/`_apply_rewards` and `track_action`/`_apply_achievements` in `rewards.py`. Both
`notifications.py` and `rewards.py` need the module-level `from devplacepy.services.background import
background` import.
**Jinja globals (register-once, from `templating.py`).** `templating.py:13-15,62-63,112,132-136`
imports `format_date`, `time_ago`, `get_badge`, `is_admin`, `is_primary_admin`, `pretty_json` and
registers `is_admin`/`is_primary_admin`/`format_date` as callable globals (load-bearing per the
`respond()` collision rule). The registration stays in `templating.py`; the split must only keep
those import paths resolving via the package.
**Cache invalidation invariant.** `_create_account` calls `database.invalidate_admins_cache()` on the
first user (`640`) - keep that import in `accounts.py`.
**Test footprint.** `tests/unit/utils.py` imports `award_badge, award_xp, get_badge,
check_milestone_badges, create_mention_notifications, extract_mentions, generate_uid, hash_password,
is_primary_admin, level_for_xp, safe_next, slugify, strip_html, time_ago, verify_password` - every one
must survive re-export. After the split, diff `dir(devplacepy.utils)` public names against the
pre-split snapshot.
---
### 3.9 `services/news.py` (986) -> `devplacepy/services/news/`
**Nature.** `NewsService(BaseService)` plus a set of pure free functions and constants for the
import pipeline. `main.py:88,236` imports and registers `NewsService`. The CLI does NOT import this
module (its `news clear`/`news sanitize` operate on raw tables via dataset + `utils.strip_html`).
**Target layout (`devplacepy/services/news/`):**
```
__init__.py # re-exports the full public + test surface; also re-binds `net_guard` and `httpx`
constants.py # all tuning constants + JUNK_PATTERNS + WHITESPACE_PATTERN + prose blocks
# (NEWS_SUMMARY, GRADE_PROMPT_SPEC, FORMAT_PROMPT_SPEC, GRADING_RULES_DESCRIPTION);
# numeric constants MUST precede GRADING_RULES_DESCRIPTION (it is an f-string over them)
models.py # ImageCandidate, ArticleGrade
clean.py # clean_news_text, _caps_ratio, _is_valid_url, reliability_reason, effective_score,
# _extract_grade, _strip_md_fence
images.py # _decode_phash, _get_article_images, _fetch_image_candidate,
# _flag_shared_placeholders, _primary_image
service.py # NewsService (run_once/collect_metrics/_store_*/_apply_landing_selection/
# _grade_article/_format_article/_grade_article_full/_resolve_uid) + _get_ai_key
```
`__init__.py` re-export must include at minimum: `NewsService`, `ImageCandidate`, `clean_news_text`,
`reliability_reason`, `effective_score`, `_extract_grade`, `_get_ai_key`, `_get_article_images`,
`_flag_shared_placeholders`, `_primary_image`, `_strip_md_fence`, and constants `LANDING_MIN_SCORE`,
`LANDING_MAX`, `UNIQUE_IMAGE_BONUS`, `THIN_CONTENT_PENALTY`; plus re-bind `net_guard` and `httpx` as
package attributes.
**Load-bearing risk (dictates where `_get_ai_key` lives).** `tests/unit/services/news.py`
monkeypatches names in the news-module namespace: `monkeypatch.setattr(news_mod, "get_setting", ...)`
(x10) where `news_mod = devplacepy.services.news`, and `setattr(news_mod.net_guard, "guard_public_url",
...)`, and reads `news_mod.httpx`. `_get_ai_key` resolves `get_setting` from ITS OWN module globals. To
keep these tests passing with NO test edits, keep `_get_ai_key` and its `from ...database import
get_setting` in the SAME module the test patches - i.e. keep them in `service.py` and ensure the test's
`monkeypatch.setattr(news_mod, "get_setting", ...)` still targets the name `_get_ai_key` reads. The safe
form: keep `_get_ai_key` + the `get_setting` import together in `service.py`, and have `__init__` do
`from .service import get_setting` re-binding so the package-namespace patch and the resolving namespace
are the same object. Simplest conservative variant: peel off only the genuinely pure modules
(`constants.py`, `models.py`, `clean.py`, `images.py`) and keep `NewsService` + `_get_ai_key` +
`get_setting`/`net_guard`/`httpx` imports in `service.py` (which `__init__` re-exports wholesale). This
removes most of the size while eliminating the namespace hazard.
**Other risks:**
- `GRADING_RULES_DESCRIPTION` f-string ordering (constants first).
- `_decode_phash` runs via `run_in_executor` - pure, safe to relocate to `images.py`.
- BaseService contract: `run_once`/`collect_metrics`/`config_fields`/`interval_key`/`default_enabled`/
`__init__(name="news", interval_seconds=3600)` stay on `NewsService`. Methods using `self.log`/
`self.get_config` cannot become free functions.
- `_grade_article_full` is pure but called as `self._grade_article_full(...)`; leave it a method
(lowest risk).
---
### 3.10 `services/devii/session.py` (962) -> same-name package `session/` (extract, do not decompose)
**Nature.** ONE heavily-shared-mutable-state class `DeviiSession` (`73-851`) plus ~150 lines of
stateless constants/pure helpers. Constructed by `hub.py:70-85` with an exact 11-positional +
3-keyword arg signature. `_parse_topic` is imported by `tests/unit/services/devii/session.py:6`.
**Recommendation: extract the stateless text/helpers into a package, keep `DeviiSession` whole.**
A true class decomposition would require threading `_lock`/`_send_lock`/`_conns`/`_conn_meta`/
`agent._messages`/`self.tools`/`_pending`/`_buffer`/`_turn_epoch` across objects and risks the
`self.tools[:]` aliasing and `_turn_epoch` interrupt guard. Not worth it.
**Target layout (`services/devii/session/`):**
```
__init__.py # from .core import DeviiSession; from .prompts import _parse_topic (+ optional constants)
prompts.py # DOCS_SYSTEM_PROMPT, DOCS_GREETING, DOCS_TOOLS, TOPIC_CLASSIFIER_PROMPT,
# NON_ADMIN_COST_RULE, LOGIN_REQUEST, BEHAVIOR_HEADER, _parse_topic(), _system_prompt_for()
# (session.py:854-947 + 40 + 67)
_helpers.py # _repair_history(), _now_iso(), _format_offset() (43-65, 950-962)
core.py # class DeviiSession + the numeric constants only it uses (27-31, 39)
```
This lifts ~150 lines out of the class file with zero behavior risk. `__init__.py` re-exports
`DeviiSession` (used by `hub.py:15`) and `_parse_topic` (test). `prompts.py`/`_helpers.py` import
nothing from `core.py` (no cycle). All prompt constants are used only inside the class today, so they
move freely.
Optional (secondary, only if further reduction is wanted): a mixin split of `DeviiSession` into
`ConnectionMixin`/`PromptComposition`/`TurnControl`/`SpendTracking`/`Scheduling` operating on one
shared `self`. Given the density of cross-group `self.` references (e.g. `_run_turn` touches
`_refresh_tools`, `_refresh_system_prompt`, `_docs_topic_gate`, `_cost_snapshot`, `_record_spend`,
`_persist_history`, `_emit`, `_turn_epoch`), this trades one long file for cross-mixin coupling - do it
only if the primary extraction is not enough.
**Invariants that must not break (whichever depth is chosen):**
- **`self.tools` mutated in place, never reassigned.** `_refresh_tools` uses `self.tools[:] = ...`
because the same list object is bound into `AgenticController`/`Agent`/`react_loop`. Reassigning
freezes the tool list (virtual tools stop taking effect). Same in-place discipline for
`self.agent._messages` rewrites (`_refresh_system_prompt` `544-547`, `reset_conversation` `357-375`,
`_docs_topic_gate` `549-577`, `stop`/`_repair_history` `408-426`).
- **Turn-epoch interrupt guard** (`_turn_epoch` at `184`, incremented in `cancel_turns:384`, checked
in `_run_turn:451,459,463`) - keep `spawn_turn`/`cancel_turns`/`_run_turn` with the shared
`_turn_epoch`/`_turns` state (MEMORY: devii-browser-and-interrupt; do not regress).
- **`__init__` positional-arg contract** with `hub.py:70-85` must be preserved exactly.
- `session.py` imports `CATALOG` from `.registry` (import-time `_assert_confirm_params`); the package
rename must not introduce a cycle - `prompts.py`/`_helpers.py` import nothing from `core.py`.
- `from __future__ import annotations` in every extracted module (annotations use `X | None`).
---
### 3.11 `services/game/store.py` (948) -> subpackage `services/game/store/`
**Nature.** The DB/stateful layer of the already-package `services/game/` (siblings: `economy.py`
pure rules - untouched; `__init__.py` does `from . import economy, store; from .store import GameError`).
Only `store.py` needs splitting. Every function funnels through four `get_table` accessors
(`_farms/_plots/_quests/_steals`) and `_update_farm`; the only module-level state is `PERK_COLUMN`
(built from `economy.PERKS` at import). The game tables carry no `deleted_at` (outside
`SOFT_DELETE_TABLES`) - no soft-delete concerns.
**Target layout (convert `store.py` -> `store/` package, keeping path `services.game.store`):**
```
store/
__init__.py # re-exports EVERY name currently on store.py, incl. private helpers tests call
common.py # GameError, _now, _iso, _parse, _today, _parse_date, _farms, _plots, _quests,
# _steals, _lvl, PERK_COLUMN, _update_farm, last_steal_at, steal_cooldown_remaining
farm.py # get_farm, ensure_farm, _create_plot, get_plots, _plot_at, buy_plot, upgrade_ci, leaderboard
serialize.py # _watered_by, _plot_state, serialize_plot, serialize_farm, _serialize_perks,
# _serialize_legacy, _serialize_quests, _daily_available
quests.py # ensure_quests, advance_quests
actions.py # plant, harvest, water, steal, fertilize, prestige, upgrade_perk, upgrade_legacy,
# claim_daily, claim_quest, _auto_harvest
```
`__init__.py` re-export must include the router surface (`ensure_farm`, `serialize_farm`, `plant`,
`harvest`, `buy_plot`, `upgrade_ci`, `water`, `steal`, `leaderboard`, `claim_daily`, `claim_quest`,
`upgrade_perk`, `upgrade_legacy`, `prestige`, `fertilize`, `GameError`) **plus the private symbols the
unit test calls directly**: `get_farm`, `get_plots`, `ensure_quests`, `advance_quests`, `_plot_at`,
`_today`. `services/game/__init__.py` keeps working unchanged.
Importers: `routers/game/index.py:18`, `routers/game/farm.py:11`, `routers/game/_shared.py:6` (all
`from devplacepy.services.game import GameError, store` then `store.X`). No CLI, no `main.py`.
**Risks:**
- **Re-export private helpers** (`_plot_at`, `_today`, + `get_farm`/`get_plots`/`ensure_quests`/
`advance_quests`) or the unit test breaks. No monkeypatch-of-module-name coupling here (unlike news),
so the only hazard is a missing re-export.
- **`PERK_COLUMN`** built from `economy.PERKS` at import must live in the low-level shared module
(`common.py`) imported by `serialize.py`/`actions.py`; `common.py` imports `economy`, higher modules
import `common` + `economy` (no cycle).
- **Dense intra-module call graph** (`serialize_farm -> _auto_harvest -> advance_quests`;
`actions -> ensure_farm/_update_farm/advance_quests`) - group so lower-level modules
(`common`/`quests`/`farm`) are imported by higher-level (`serialize`/`actions`), not vice-versa.
- **`leaderboard` lazy import** `from devplacepy.database import get_users_by_uids` at `store.py:593` -
keep function-local when relocating.
---
## 4. SPECIAL CASE - container standalone scripts (files 4 and 5)
`services/containers/files/bot.py` (2940, `COPY bot.py /usr/bin/botje.py`) and
`services/containers/files/d.py` (2337, `COPY d.py /usr/bin/d.py`) are NOT server backend. They are
standalone single-file programs baked into the `ppy` image (`ppy.Dockerfile:47-48`, build context
`devplacepy/services/containers/files`, invoked `python /usr/bin/botje.py`). Their only hard
third-party dep is `httpx`. **Nothing in `devplacepy/` or `tests/` imports either file** (confirmed
grep) - they are not on `sys.path` as modules and have no test coverage. The only breakage surface is
the Docker build's COPY path and the documented invocation.
**Binding constraint.** A normal package split (like the server-side files) is **incompatible** with
the single-file COPY + bare-`python` exec contract: a package leaf COPY'd to `/usr/bin/botje.py` would
fail at the first cross-module `from botpkg... import` (no package root on the container's path). So the
in-place package approach used for every other monster does not apply here.
**Options (pick per user direction; this plan does not choose for you):**
- **(a) Keep as single-file monoliths.** Zero Dockerfile/Makefile change; the contract stays intact.
Trade-off: the two files remain large and duplicate ~2000 lines of agent code between them
(`files/bot.py` = `files/d.py`'s agent + a DevPlace XML-RPC bot loop; `files/d.py`'s stealth half is
a pure-httpx sibling of `devplacepy/stealth.py`). This is the lowest-risk option.
- **(b) Split into a source package + add a build step** that re-stitches the single deployed artifact.
Two sub-options, each with concrete required edits:
- **zipapp:** author `files/agent/` as a package, build with
`python -m zipapp files/agent -m "agent.cli:main" -o build/botje.pyz`. Requires changing
`ppy.Dockerfile:47-48` COPY source + extension (`.pyz`), the `chmod`/`ln` lines (`:52`), the
invocation in docs (`templates/docs/getting-started-vibing.html:128-162`) from `python /usr/bin/
botje.py` to `python /usr/bin/botje.pyz`, and the `make ppy` target (`Makefile:129-130`) to run the
bundler before `docker build`. Downside: breaks the documented `cp /usr/bin/botje.py /app/mybot.py`
copy-and-edit single-file flow.
- **concatenation/stitch:** a `make` recipe concatenates the package modules (deduped imports, in
dependency order) into one `build/bot.py`, which the Dockerfile then COPYs. Keeps the `.py`
extension and `python /usr/bin/botje.py` intact; users edit generated output, not source. Requires
`Makefile:129` (add the bundler step + widen the rebuild-trigger set) and `ppy.Dockerfile:47-48`
(COPY from `build/`).
**Note (not a recommendation, for context):** the largest issue with these two files is duplication,
not size - `files/bot.py` and `files/d.py` share the entire agent (tool framework, `react_loop`,
`CorpusIndex`, embedded `ChromeStealthClient`). If the user later wants these deduplicated rather than
split, that is a different task; this plan records the fact only.
---
## 5. Cross-file execution order (recommended)
Order by blast radius (lowest first) so a mistake surfaces on a small surface before the wide ones,
and so the widely-imported files are split last against a known-green tree:
1. **`services/game/store.py`** - self-contained subpackage, 3 importers, no cycles. Warm-up.
2. **`services/devii/actions/catalog.py`** - pure data, 2 exported symbols, ordering-only risk.
3. **`schemas.py`** - pure models, mechanical, wide but simple re-export.
4. **`services/news.py`** - watch the monkeypatch namespace; conservative variant recommended.
5. **`services/devii/session.py`** - extract stateless helpers only (keep class whole).
6. **`services/bot/bot.py`** - mixin lift; keep `__init__` exporting only `BotsService`.
7. **`cli.py`** - package with per-group modules; keep the console-script entry.
8. **`docs_api.py`** - large but declarative; handle the three ordering hazards.
9. **`utils.py`** - 224 importers, funnel + lazy-import + Jinja-global sensitivity.
10. **`database.py`** - 329 importers, the core layer; do it last against a green tree.
11. **Container `files/bot.py` / `files/d.py`** - only if the user opts into option (b); otherwise
leave as-is per Section 4.
Each step is independent: split one file, run the Section-2.9 validation, commit, move on. Do not batch
multiple monster splits into one unvalidated change.
---
## 6. Convention alignment (why packages, not `_`-suffixed files)
This plan uses the project's existing structural idiom, verified against the current tree:
- Router packages (`routers/issues/`, `routers/admin/`, `routers/profile/`, `routers/projects/`) are
directories whose `__init__.py` aggregates leaf modules and re-exports the public symbol, each new
file headed `# retoor <retoor@molodetz.nl>`. Every split above mirrors that.
- `_shared.py` is the sanctioned package-private helper module (already used in eight router packages);
it is used here only for genuine cross-leaf construction helpers (`docs_api/_shared.py`,
`catalog/_shared.py`, `cli/_shared.py`), never as a dumping ground.
- No `_v2`/`_refactored`/`_new` suffixes anywhere (forbidden by project rules). The compatibility
mechanism is the same-name package + re-export, not a parallel file.
- `services/` already contains packages that expose a `store` submodule (`services/audit/`,
`services/backup/`, `services/containers/`, `services/game/`), so `services/news/` and
`services/game/store/` and `services/devii/session/` and `services/devii/actions/catalog/` all follow
precedent.
---
## 7. Documentation follow-up (after any split is executed)
Per the project feature workflow, once a split is executed the structural change is documented in the
three canonical files - but ONLY if a rule/convention changes. A pure module move that preserves every
import path introduces no new architectural rule, so:
- `README.md`: no change (no user-visible/route/env/dep change).
- `AGENTS.md`: add a short note under the relevant domain only if the new module boundary is worth
navigational guidance (e.g. "docs_api is now a package, one module per API group").
- `CLAUDE.md`: no change unless the split establishes a NEW convention (it does not - it follows the
existing package idiom).
This section is a reminder, not a task; the plan itself does not edit docs.
---
## 8. Summary table - target packages
| Source file | Target | Split style | Key invariant to protect |
|-------------|--------|-------------|--------------------------|
| `docs_api.py` | `docs_api/` (+`groups/`) | one module per API group | import-time negotiation pass; group order; `_substitute` re-export |
| `database.py` | `database/` | layered domain modules on `core.py` | `db` singleton in `core`; underscore re-exports; keep lazy imports |
| `services/bot/bot.py` | mixins in `services/bot/` | cooperative mixins | realism contract; `__init__` exports only `BotsService`; stealth rule |
| `services/devii/actions/catalog.py` | `catalog/` (+`_shared.py`) | one module per action domain | `ACTIONS` order; `confirm`-param invariant; `Param` import in dbapi/gateway/game |
| `schemas.py` | `schemas/` | domain schema modules | `CommentItemOut.model_rebuild()`; cross-module inheritance order |
| `cli.py` | `cli/` (+`_shared.py`) | one module per command group | console-script `cli:main`; re-export all `cmd_*`; lazy imports |
| `utils.py` | `utils/` | layered helper modules | funnel pairing; lazy imports; Jinja-global paths; `"auth"` cache key |
| `services/news.py` | `services/news/` | pipeline-stage modules | monkeypatch namespace (`_get_ai_key`/`get_setting`); f-string const order |
| `services/devii/session.py` | `session/` | extract stateless text/helpers | `self.tools[:]` in-place; `_turn_epoch` guard; `__init__` arg contract |
| `services/game/store.py` | `services/game/store/` | cohesive store modules | re-export private test helpers; `PERK_COLUMN` placement |
| `services/containers/files/bot.py` | SPECIAL (Section 4) | single-file OR build step | COPY-single-file + bare-exec contract |
| `services/containers/files/d.py` | SPECIAL (Section 4) | single-file OR build step | COPY-single-file + bare-exec contract |

65
scripts/check_database_imports.py
View File

@ -1,65 +0,0 @@
#!/usr/bin/env python3
import re
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
ALLOW_PREFIXES = (
ROOT / "devplacepy" / "database",
ROOT / "devplacepy" / "db_client.py",
ROOT / "devplacepy_services" / "database",
ROOT / "scripts" / "migrate_db_imports.py",
ROOT / "scripts" / "check_database_imports.py",
)
PATTERNS = [
re.compile(r"from devplacepy\.database(?:\.[a-z_]+)? import"),
re.compile(r"import devplacepy\.database\b"),
]
# Only flagged outside tests/: unit tests may legitimately use `local_db` /
# `from devplacepy import database` directly (Section 13.4). Source code must
# always go through `devplacepy.db_client` so the remote broker patch applies.
SOURCE_ONLY_PATTERNS = [
re.compile(r"from devplacepy import(?:\s+\w+,)*\s+database\b"),
]
def allowed(path: Path) -> bool:
for prefix in ALLOW_PREFIXES:
if path == prefix or str(path).startswith(str(prefix) + "/"):
return True
return False
def main() -> int:
errors: list[str] = []
for pattern in ("devplacepy/**/*.py", "devplacepy_services/**/*.py", "tests/**/*.py"):
is_source = not pattern.startswith("tests/")
for path in ROOT.glob(pattern):
if not path.is_file() or allowed(path):
continue
try:
text = path.read_text(encoding="utf-8", errors="ignore")
except OSError:
continue
active_patterns = PATTERNS + SOURCE_ONLY_PATTERNS if is_source else PATTERNS
for lineno, line in enumerate(text.splitlines(), 1):
for rx in active_patterns:
if rx.search(line):
errors.append(f"{path.relative_to(ROOT)}:{lineno}: {line.strip()}")
break
if errors:
print("FORBIDDEN database imports (use devplacepy.db_client):")
for line in sorted(errors)[:80]:
print(f" {line}")
if len(errors) > 80:
print(f" ... and {len(errors) - 80} more")
return 1
print("check_database_imports: ok")
return 0
if __name__ == "__main__":
raise SystemExit(main())

85
scripts/check_no_monolith.py
View File

@ -1,85 +0,0 @@
#!/usr/bin/env python3
import re
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
ALLOWLIST = {
ROOT / "splitup.md",
ROOT / "scripts/check_no_monolith.py",
}
PATTERNS = [
(r"uvicorn\s+devplacepy\.main:app", "uvicorn monolith boot"),
(r"from devplacepy\.main import", "main.py import"),
(r"import devplacepy\.main", "main.py import"),
(r"devplacepy\.main:app", "main app string"),
(r"DEVPLACE_DB_MODE", "hybrid DB mode flag"),
(r"make dev-monolith", "dev-monolith target"),
(r"make prod-monolith", "prod-monolith target"),
(r"DEVPLACE_TEST_PORT=10501", "legacy test port default"),
(r"monolith-dev", "monolith port profile"),
]
PHASE = sys.argv[1] if len(sys.argv) > 1 else "m7"
def scan_file(path: Path) -> list[str]:
if path in ALLOWLIST:
return []
try:
text = path.read_text(encoding="utf-8", errors="ignore")
except OSError:
return []
hits = []
for pattern, label in PATTERNS:
if re.search(pattern, text):
hits.append(f"{path.relative_to(ROOT)}: {label}")
return hits
def main() -> int:
import subprocess
db_check = subprocess.run(
[sys.executable, str(ROOT / "scripts" / "check_database_imports.py")],
capture_output=True,
text=True,
check=False,
)
if db_check.returncode != 0:
print(db_check.stdout or db_check.stderr)
return 1
globs = [
"Makefile",
"Dockerfile",
"docker-compose*.yml",
".gitea/**/*.yaml",
"CLAUDE.md",
"README.md",
"pyproject.toml",
"devplacepy/**/*.py",
"tests/**/*.py",
"orchestrator.py",
]
errors = []
for pattern in globs:
for path in ROOT.glob(pattern):
if path.is_file():
errors.extend(scan_file(path))
if PHASE in ("m3", "m6", "m7") and (ROOT / "devplacepy/main.py").exists():
if PHASE == "m7":
errors.append("devplacepy/main.py: file must be deleted at M7")
elif PHASE == "m3":
pass
if errors:
print("MONOLITH ARTIFACTS FOUND:")
for line in sorted(set(errors)):
print(f" {line}")
return 1
print("check_no_monolith: ok")
return 0
if __name__ == "__main__":
raise SystemExit(main())

60
scripts/migrate_db_imports.py
View File

@ -1,60 +0,0 @@
#!/usr/bin/env python3
import re
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SKIP_PREFIXES = (
ROOT / "devplacepy" / "database",
ROOT / "devplacepy_services" / "database",
ROOT / "devplacepy" / "db_client.py",
ROOT / "scripts",
ROOT / "splitup.md",
)
REPLACEMENTS = [
(re.compile(r"\bfrom devplacepy\.database import\b"), "from devplacepy.db_client import"),
(re.compile(r"\bfrom devplacepy\.database\.([a-z_]+) import\b"), r"from devplacepy.db_client import"),
(re.compile(r"\bimport devplacepy\.database\b"), "import devplacepy.db_client"),
]
def should_skip(path: Path) -> bool:
for prefix in SKIP_PREFIXES:
if path == prefix or str(path).startswith(str(prefix) + "/"):
return True
return False
def migrate_file(path: Path) -> bool:
text = path.read_text(encoding="utf-8")
original = text
for pattern, repl in REPLACEMENTS:
text = pattern.sub(repl, text)
if text != original:
path.write_text(text, encoding="utf-8")
return True
return False
def main() -> int:
changed = []
for pattern in ("devplacepy/**/*.py", "tests/**/*.py", "devplacepy_services/**/*.py"):
for path in ROOT.glob(pattern):
if not path.is_file() or should_skip(path):
continue
if migrate_file(path):
changed.append(path.relative_to(ROOT))
if changed:
print(f"migrated {len(changed)} files")
for item in sorted(changed)[:40]:
print(f" {item}")
if len(changed) > 40:
print(f" ... and {len(changed) - 40} more")
else:
print("no files changed")
return 0
if __name__ == "__main__":
raise SystemExit(main())

92
scripts/refactor_to_microservices.py
View File

@ -1,92 +0,0 @@
#!/usr/bin/env python3
import argparse
import importlib
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SERVICES_DIR = ROOT / "devplacepy_services"
SERVICES = [
"web",
"database",
"pubsub",
"gateway",
"jobs",
"devii",
"bot",
"backup",
"containers",
"telegram",
"email",
"news",
"gitea",
"audit",
"xmlrpc",
]
def list_services() -> int:
for name in SERVICES:
main_py = SERVICES_DIR / name / "main.py"
status = "ok" if main_py.exists() else "missing"
print(f"{name:12} {status}")
return 0
def verify_stubs() -> int:
sys.path.insert(0, str(ROOT))
errors: list[str] = []
for name in SERVICES:
main_py = SERVICES_DIR / name / "main.py"
if not main_py.exists():
errors.append(f"{name}: missing main.py")
continue
module_name = f"devplacepy_services.{name}.main"
try:
module = importlib.import_module(module_name)
except Exception as exc:
errors.append(f"{name}: import failed ({exc})")
continue
app = getattr(module, "app", None)
if app is None:
errors.append(f"{name}: no app export")
continue
try:
from fastapi.testclient import TestClient
client = TestClient(app)
resp = client.get("/health")
if resp.status_code != 200:
errors.append(f"{name}: /health returned {resp.status_code}")
continue
data = resp.json()
for key in ("service", "status", "uptime_s", "version", "deps", "stats"):
if key not in data:
errors.append(f"{name}: health missing key {key}")
except Exception as exc:
errors.append(f"{name}: health check failed ({exc})")
if errors:
print("verify: failed")
for err in errors:
print(f" {err}")
return 1
print("verify: ok")
return 0
def main() -> int:
parser = argparse.ArgumentParser(prog="refactor_to_microservices.py")
parser.add_argument("--list", action="store_true")
parser.add_argument("--verify", action="store_true")
args = parser.parse_args()
if args.list:
return list_services()
if args.verify:
return verify_stubs()
parser.print_help()
return 0
if __name__ == "__main__":
raise SystemExit(main())

62
test_api.json
View File

@ -1,62 +0,0 @@
{
"tests": [
{
"name": "Landing page returns 200",
"method": "GET",
"path": "/",
"expect_status": 200
},
{
"name": "Signup page returns 200",
"method": "GET",
"path": "/auth/signup",
"expect_status": 200
},
{
"name": "Login page returns 200",
"method": "GET",
"path": "/auth/login",
"expect_status": 200
},
{
"name": "Feed redirects to login when not authenticated",
"method": "GET",
"path": "/feed",
"expect_status": 303
},
{
"name": "Static CSS is served",
"method": "GET",
"path": "/static/css/variables.css",
"expect_status": 200
},
{
"name": "Static JS is served",
"method": "GET",
"path": "/static/js/Application.js",
"expect_status": 200
},
{
"name": "User signup works",
"method": "POST",
"path": "/auth/signup",
"form": {
"username": "testuser",
"email": "test@devplace.com",
"password": "test123",
"confirm_password": "test123"
},
"expect_status": 303
},
{
"name": "User login works",
"method": "POST",
"path": "/auth/login",
"form": {
"email": "test@devplace.com",
"password": "test123"
},
"expect_status": 303
}
]
}

2
tests/api/admin/awards.py
View File

@ -6,7 +6,7 @@ from datetime import datetime, timezone
from PIL import Image from PIL import Image
from tests.conftest import BASE_URL from tests.conftest import BASE_URL
from devplacepy.attachments import store_attachment from devplacepy.attachments import store_attachment
from devplacepy.db_client import get_table from devplacepy.database import get_table
from devplacepy.utils import generate_uid, make_combined_slug from devplacepy.utils import generate_uid, make_combined_slug
JSON = {"Accept": "application/json"} JSON = {"Accept": "application/json"}

2
tests/api/awards/index.py
View File

@ -6,7 +6,7 @@ from datetime import datetime, timezone
from PIL import Image from PIL import Image
from tests.conftest import BASE_URL from tests.conftest import BASE_URL
from devplacepy.attachments import store_attachment from devplacepy.attachments import store_attachment
from devplacepy.db_client import get_table from devplacepy.database import get_table
from devplacepy.utils import generate_uid, make_combined_slug from devplacepy.utils import generate_uid, make_combined_slug

2
tests/api/profile/award.py
View File

@ -3,7 +3,7 @@
import time import time
import requests import requests
from tests.conftest import BASE_URL from tests.conftest import BASE_URL
from devplacepy.db_client import get_table from devplacepy.database import get_table
JSON = {"Accept": "application/json"} JSON = {"Accept": "application/json"}
_counter = [0] _counter = [0]

2
tests/api/profile/awards_tab.py
View File

@ -6,7 +6,7 @@ from datetime import datetime, timezone
from PIL import Image from PIL import Image
from tests.conftest import BASE_URL from tests.conftest import BASE_URL
from devplacepy.attachments import store_attachment from devplacepy.attachments import store_attachment
from devplacepy.db_client import get_table from devplacepy.database import get_table
from devplacepy.utils import generate_uid, make_combined_slug from devplacepy.utils import generate_uid, make_combined_slug
JSON = {"Accept": "application/json"} JSON = {"Accept": "application/json"}

5
tests/unit/database/awards.py
View File

@ -3,12 +3,13 @@
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
import pytest import pytest
from devplacepy.db_client import get_table, init_db from devplacepy.database import (
from devplacepy.db_client import (
can_give_award, can_give_award,
can_receive_award, can_receive_award,
get_table,
has_giver_cooldown, has_giver_cooldown,
has_receiver_cooldown, has_receiver_cooldown,
init_db,
recompute_user_award_stats, recompute_user_award_stats,
revoke_award, revoke_award,
) )

8
tests/unit/services/jobs/award_service.py
View File

@ -6,8 +6,12 @@ from datetime import datetime, timezone
import pytest import pytest
from PIL import Image from PIL import Image
from devplacepy.db_client import get_award_usage, get_table, init_db from devplacepy.database import (
from devplacepy.db_client import recompute_user_award_stats get_award_usage,
get_table,
init_db,
recompute_user_award_stats,
)
from devplacepy.services.jobs.award_service import AwardService from devplacepy.services.jobs.award_service import AwardService
from devplacepy.utils import generate_uid, make_combined_slug from devplacepy.utils import generate_uid, make_combined_slug
from tests.conftest import run_async from tests.conftest import run_async

212
tests/unit/services/test_compounds.py
View File

@ -1,212 +0,0 @@
# retoor <retoor@molodetz.nl>
from datetime import datetime, timezone
import pytest
from devplacepy.db_client import get_table
from devplacepy.utils import generate_uid
def _now():
return datetime.now(timezone.utc).isoformat()
def _make_user(local_db, **overrides):
uid = generate_uid()
row = {
"uid": uid,
"username": f"cpd_{uid[:8]}",
"email": f"cpd_{uid[:8]}@t.dev",
"api_key": generate_uid(),
"role": "Member",
"is_active": True,
"created_at": _now(),
"bio": "",
}
row.update(overrides)
get_table("users").insert(row)
return row
def _make_post(local_db, user, **overrides):
from devplacepy.utils import make_combined_slug
uid = generate_uid()
row = {
"uid": uid,
"user_uid": user["uid"],
"title": "Compound test post",
"content": "Body of the compound test post.",
"topic": "devlog",
"slug": make_combined_slug("Compound test post", uid),
"stars": 0,
"created_at": _now(),
"deleted_at": None,
"deleted_by": None,
}
row.update(overrides)
get_table("posts").insert(row)
return row
def _make_project(local_db, user, **overrides):
from devplacepy.utils import make_combined_slug
uid = generate_uid()
row = {
"uid": uid,
"user_uid": user["uid"],
"title": "Compound test project",
"description": "",
"slug": make_combined_slug("Compound test project", uid),
"stars": 0,
"is_private": 0,
"created_at": _now(),
"deleted_at": None,
"deleted_by": None,
}
row.update(overrides)
get_table("projects").insert(row)
return row
def _make_gist(local_db, user, **overrides):
from devplacepy.utils import make_combined_slug
uid = generate_uid()
row = {
"uid": uid,
"user_uid": user["uid"],
"title": "Compound test gist",
"description": "",
"source_code": "print('hi')",
"language": "python",
"slug": make_combined_slug("Compound test gist", uid),
"stars": 0,
"created_at": _now(),
"deleted_at": None,
"deleted_by": None,
}
row.update(overrides)
get_table("gists").insert(row)
return row
# Section 5.3 page compounds actually wired to a caller (db_client.build_*)
# and consumed by a real web route. Each assertion checks the template
# context keys the corresponding route/schema depends on.
def test_build_feed_page_shape(local_db):
from devplacepy_services.database.compounds_page import build_feed_page
user = _make_user(local_db)
post = _make_post(local_db, user)
result = build_feed_page(user=user)
for key in ("posts", "next_cursor", "stats", "top_authors", "daily_topic", "current_tab"):
assert key in result
uids = [item["post"]["uid"] for item in result["posts"]]
assert post["uid"] in uids
item = next(i for i in result["posts"] if i["post"]["uid"] == post["uid"])
for key in ("attachments", "recent_comments", "reactions", "bookmarked", "poll"):
assert key in item
def test_build_post_detail_shape(local_db):
from devplacepy_services.database.compounds_page import build_post_detail
user = _make_user(local_db)
post = _make_post(local_db, user)
result = build_post_detail(post["uid"], user=user)
assert result["post"]["uid"] == post["uid"]
assert result["author"]["uid"] == user["uid"]
assert isinstance(result["comments"], list)
assert isinstance(result["attachments"], list)
def test_build_post_detail_missing_post(local_db):
from devplacepy_services.database.compounds_page import build_post_detail
result = build_post_detail("does-not-exist")
assert result["post"] is None
def test_build_profile_bundle_shape(local_db):
from devplacepy_services.database.compounds_page import build_profile_bundle
user = _make_user(local_db)
result = build_profile_bundle(user["uid"])
assert result["user"]["uid"] == user["uid"]
assert "followers" in result["follow_counts"]
assert "following" in result["follow_counts"]
def test_build_leaderboard_page_shape(local_db):
from devplacepy_services.database.compounds_page import build_leaderboard_page
user = _make_user(local_db)
result = build_leaderboard_page(viewer_uid=user["uid"])
assert "leaderboard" in result
assert isinstance(result["leaderboard"], list)
def test_build_project_detail_shape(local_db):
from devplacepy_services.database.compounds_page import build_project_detail
user = _make_user(local_db)
project = _make_project(local_db, user)
result = build_project_detail(project["uid"])
assert result["project"]["uid"] == project["uid"]
assert result["owner"]["uid"] == user["uid"]
def test_build_gist_detail_shape(local_db):
from devplacepy_services.database.compounds_page import build_gist_detail
user = _make_user(local_db)
gist = _make_gist(local_db, user)
result = build_gist_detail(gist["uid"], user=user)
assert result["gist"]["uid"] == gist["uid"]
assert result["author"]["uid"] == user["uid"]
assert isinstance(result["comments"], list)
# Section 5.3 rows registered as HTTP endpoints (devplacepy_services/database/routes.py)
# but not yet called by any web route (routers/messages.py, notifications.py,
# game/, admin/index.py all still do their own direct db_client calls) - these
# four builders are unfinished stubs, tracked here so a real implementation
# lands as a visible test change rather than silently.
@pytest.mark.parametrize(
"fn_name,args,expected_keys",
[
("build_messages_page", ("u1",), {"threads", "unread", "partners"}),
("build_notifications_page", ("u1",), {"notifications", "actors", "unread_count"}),
("build_game_state", (), {"store", "leaderboard"}),
("build_admin_dashboard", (), {"stats", "service_states"}),
],
)
def test_stub_compounds_documented(local_db, fn_name, args, expected_keys):
import devplacepy_services.database.compounds_page as compounds_page
fn = getattr(compounds_page, fn_name)
result = fn(*args)
assert set(result.keys()) == expected_keys, (
f"{fn_name} shape changed - if this is a real implementation now, "
"replace this stub-tracking test with a real assertion and update "
"splitup.md Section 17.2"
)

Some files were not shown because too many files have changed in this diff Show More