From 6c161401deea39f16df0b32c6d4a7509a2ef8cac Mon Sep 17 00:00:00 2001 From: retoor Date: Sat, 1 Aug 2026 01:14:24 +0200 Subject: [PATCH] Update --- .claude/settings.local.json | 20 - .gitignore | 3 + CHANGELOG.md | 313 ------- CLAUDE.md | 2 +- devplacepy/database/remote.py | 348 ------- devplacepy/db_client.py | 29 - devplacepy/services/audit/CLAUDE.md | 2 +- devplacepy/services/containers/CLAUDE.md | 1 + devplacepy/services/registry.py | 2 +- devplacepy_services/__init__.py | 1 - devplacepy_services/audit/SERVICE.md | 1 - devplacepy_services/audit/__init__.py | 0 devplacepy_services/audit/main.py | 23 - devplacepy_services/backup/SERVICE.md | 1 - devplacepy_services/backup/__init__.py | 0 devplacepy_services/backup/main.py | 23 - devplacepy_services/base/__init__.py | 5 - devplacepy_services/base/auth.py | 61 -- devplacepy_services/base/bootstrap.py | 4 - devplacepy_services/base/cache.py | 67 -- devplacepy_services/base/compound.py | 153 --- devplacepy_services/base/config.py | 70 -- devplacepy_services/base/db_client.py | 236 ----- devplacepy_services/base/db_codec.py | 126 --- devplacepy_services/base/errors.py | 63 -- devplacepy_services/base/health.py | 57 -- devplacepy_services/base/http.py | 77 -- devplacepy_services/base/manifest.py | 215 ----- devplacepy_services/base/middleware.py | 121 --- devplacepy_services/base/proxy.py | 37 - devplacepy_services/base/pubsub_client.py | 13 - devplacepy_services/base/schemas.py | 29 - devplacepy_services/base/service.py | 134 --- devplacepy_services/base/sqlite_broker.py | 110 --- devplacepy_services/base/stats.py | 26 - devplacepy_services/base/testing.py | 40 - devplacepy_services/bot/SERVICE.md | 1 - devplacepy_services/bot/__init__.py | 0 devplacepy_services/bot/main.py | 23 - devplacepy_services/containers/SERVICE.md | 1 - devplacepy_services/containers/__init__.py | 0 devplacepy_services/containers/main.py | 24 - devplacepy_services/database/SERVICE.md | 1 - devplacepy_services/database/__init__.py | 0 devplacepy_services/database/broker_setup.py | 52 -- .../database/compounds_page.py | 135 --- .../database/compounds_primitive.py | 138 --- devplacepy_services/database/db_patch.py | 119 --- .../database/invoke_registry.py | 40 - devplacepy_services/database/main.py | 52 -- devplacepy_services/database/routes.py | 421 --------- devplacepy_services/devii/SERVICE.md | 1 - devplacepy_services/devii/__init__.py | 0 devplacepy_services/devii/main.py | 40 - devplacepy_services/devii/store_brokers.py | 27 - devplacepy_services/devii/store_routes.py | 63 -- devplacepy_services/email/SERVICE.md | 1 - devplacepy_services/email/__init__.py | 0 devplacepy_services/email/main.py | 20 - devplacepy_services/email/routes.py | 9 - devplacepy_services/gateway/SERVICE.md | 1 - devplacepy_services/gateway/__init__.py | 0 devplacepy_services/gateway/main.py | 24 - devplacepy_services/gitea/SERVICE.md | 1 - devplacepy_services/gitea/__init__.py | 0 devplacepy_services/gitea/main.py | 23 - devplacepy_services/jobs/SERVICE.md | 1 - devplacepy_services/jobs/__init__.py | 0 devplacepy_services/jobs/main.py | 49 - devplacepy_services/news/SERVICE.md | 1 - devplacepy_services/news/__init__.py | 0 devplacepy_services/news/main.py | 23 - devplacepy_services/pubsub/SERVICE.md | 1 - devplacepy_services/pubsub/__init__.py | 0 devplacepy_services/pubsub/main.py | 31 - devplacepy_services/telegram/SERVICE.md | 1 - devplacepy_services/telegram/__init__.py | 0 devplacepy_services/telegram/main.py | 24 - devplacepy_services/web/SERVICE.md | 1 - devplacepy_services/web/__init__.py | 0 devplacepy_services/web/factory.py | 616 ------------ devplacepy_services/web/ingress.py | 216 ----- devplacepy_services/web/main.py | 21 - devplacepy_services/xmlrpc/SERVICE.md | 1 - devplacepy_services/xmlrpc/__init__.py | 0 devplacepy_services/xmlrpc/main.py | 24 - fplan.md | 589 ------------ ref.md | 875 ------------------ scripts/check_database_imports.py | 65 -- scripts/check_no_monolith.py | 85 -- scripts/migrate_db_imports.py | 60 -- scripts/refactor_to_microservices.py | 92 -- test_api.json | 62 -- tests/api/admin/awards.py | 2 +- tests/api/awards/index.py | 2 +- tests/api/profile/award.py | 2 +- tests/api/profile/awards_tab.py | 2 +- tests/unit/database/awards.py | 5 +- tests/unit/services/jobs/award_service.py | 8 +- tests/unit/services/test_compounds.py | 212 ----- tests/unit/services/test_conformance.py | 56 -- 101 files changed, 20 insertions(+), 6737 deletions(-) delete mode 100644 .claude/settings.local.json delete mode 100644 CHANGELOG.md delete mode 100644 devplacepy/database/remote.py delete mode 100644 devplacepy/db_client.py delete mode 100644 devplacepy_services/__init__.py delete mode 100644 devplacepy_services/audit/SERVICE.md delete mode 100644 devplacepy_services/audit/__init__.py delete mode 100644 devplacepy_services/audit/main.py delete mode 100644 devplacepy_services/backup/SERVICE.md delete mode 100644 devplacepy_services/backup/__init__.py delete mode 100644 devplacepy_services/backup/main.py delete mode 100644 devplacepy_services/base/__init__.py delete mode 100644 devplacepy_services/base/auth.py delete mode 100644 devplacepy_services/base/bootstrap.py delete mode 100644 devplacepy_services/base/cache.py delete mode 100644 devplacepy_services/base/compound.py delete mode 100644 devplacepy_services/base/config.py delete mode 100644 devplacepy_services/base/db_client.py delete mode 100644 devplacepy_services/base/db_codec.py delete mode 100644 devplacepy_services/base/errors.py delete mode 100644 devplacepy_services/base/health.py delete mode 100644 devplacepy_services/base/http.py delete mode 100644 devplacepy_services/base/manifest.py delete mode 100644 devplacepy_services/base/middleware.py delete mode 100644 devplacepy_services/base/proxy.py delete mode 100644 devplacepy_services/base/pubsub_client.py delete mode 100644 devplacepy_services/base/schemas.py delete mode 100644 devplacepy_services/base/service.py delete mode 100644 devplacepy_services/base/sqlite_broker.py delete mode 100644 devplacepy_services/base/stats.py delete mode 100644 devplacepy_services/base/testing.py delete mode 100644 devplacepy_services/bot/SERVICE.md delete mode 100644 devplacepy_services/bot/__init__.py delete mode 100644 devplacepy_services/bot/main.py delete mode 100644 devplacepy_services/containers/SERVICE.md delete mode 100644 devplacepy_services/containers/__init__.py delete mode 100644 devplacepy_services/containers/main.py delete mode 100644 devplacepy_services/database/SERVICE.md delete mode 100644 devplacepy_services/database/__init__.py delete mode 100644 devplacepy_services/database/broker_setup.py delete mode 100644 devplacepy_services/database/compounds_page.py delete mode 100644 devplacepy_services/database/compounds_primitive.py delete mode 100644 devplacepy_services/database/db_patch.py delete mode 100644 devplacepy_services/database/invoke_registry.py delete mode 100644 devplacepy_services/database/main.py delete mode 100644 devplacepy_services/database/routes.py delete mode 100644 devplacepy_services/devii/SERVICE.md delete mode 100644 devplacepy_services/devii/__init__.py delete mode 100644 devplacepy_services/devii/main.py delete mode 100644 devplacepy_services/devii/store_brokers.py delete mode 100644 devplacepy_services/devii/store_routes.py delete mode 100644 devplacepy_services/email/SERVICE.md delete mode 100644 devplacepy_services/email/__init__.py delete mode 100644 devplacepy_services/email/main.py delete mode 100644 devplacepy_services/email/routes.py delete mode 100644 devplacepy_services/gateway/SERVICE.md delete mode 100644 devplacepy_services/gateway/__init__.py delete mode 100644 devplacepy_services/gateway/main.py delete mode 100644 devplacepy_services/gitea/SERVICE.md delete mode 100644 devplacepy_services/gitea/__init__.py delete mode 100644 devplacepy_services/gitea/main.py delete mode 100644 devplacepy_services/jobs/SERVICE.md delete mode 100644 devplacepy_services/jobs/__init__.py delete mode 100644 devplacepy_services/jobs/main.py delete mode 100644 devplacepy_services/news/SERVICE.md delete mode 100644 devplacepy_services/news/__init__.py delete mode 100644 devplacepy_services/news/main.py delete mode 100644 devplacepy_services/pubsub/SERVICE.md delete mode 100644 devplacepy_services/pubsub/__init__.py delete mode 100644 devplacepy_services/pubsub/main.py delete mode 100644 devplacepy_services/telegram/SERVICE.md delete mode 100644 devplacepy_services/telegram/__init__.py delete mode 100644 devplacepy_services/telegram/main.py delete mode 100644 devplacepy_services/web/SERVICE.md delete mode 100644 devplacepy_services/web/__init__.py delete mode 100644 devplacepy_services/web/factory.py delete mode 100644 devplacepy_services/web/ingress.py delete mode 100644 devplacepy_services/web/main.py delete mode 100644 devplacepy_services/xmlrpc/SERVICE.md delete mode 100644 devplacepy_services/xmlrpc/__init__.py delete mode 100644 devplacepy_services/xmlrpc/main.py delete mode 100644 fplan.md delete mode 100644 ref.md delete mode 100644 scripts/check_database_imports.py delete mode 100644 scripts/check_no_monolith.py delete mode 100644 scripts/migrate_db_imports.py delete mode 100644 scripts/refactor_to_microservices.py delete mode 100644 test_api.json delete mode 100644 tests/unit/services/test_compounds.py delete mode 100644 tests/unit/services/test_conformance.py diff --git a/.claude/settings.local.json b/.claude/settings.local.json deleted file mode 100644 index 2b94f98d..00000000 --- a/.claude/settings.local.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "permissions": { - "allow": [ - "Bash(python *)", - "Bash(DEVPLACE_DISABLE_SERVICES=1 python -)", - "Bash(command -v hawk)", - "Bash(export DEVPLACE_DISABLE_SERVICES=1)", - "Bash(export DEVPLACE_DATABASE_URL=\"sqlite:///tmp/devplace_verify.db\")", - "Bash(rm -f /tmp/devplace_verify.db)", - "Bash(export DEVPLACE_DATABASE_URL=\"sqlite:////tmp/devplace_verify.db\")", - "Bash", - "Edit(/home/retoor/projects/devplacepy/devplacepy/routers/projects/containers/instances.py)", - "Edit(/home/retoor/projects/devplacepy/devplacepy/static/js/components/ContainerTerminal.js)", - "Edit(/home/retoor/projects/devplacepy/devplacepy/services/containers/store.py)", - "Verify", - "Edit(/home/retoor/projects/devplacepy/devplacepy/static/js/MessagesLayout.js)", - "Write(/home/retoor/projects/devplacepy/devplacepy/static/css/messages.css)" - ] - } -} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 876dabca..8ee19372 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,10 @@ notification-private.pem notification-private.pkcs8.pem notification-public.pem .pytest_cache/ +.ruff_cache/ .opencode +.dpc/ +.claude/settings.local.json devii_*.db devii_*.db-shm devii_*.db-wal diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 01bea4e2..00000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,313 +0,0 @@ -## 2026-06-19 🟢 - -- Block and mute user relations with API endpoints, CLI emoji-sync command, and content filtering - - -## 2026-06-18 🟢 - -- News service with image dedup, AI grading, featured/landing auto-rotation and admin lock -- Server-rendered content pipeline with Telegram pairing, response timing, and admin user index -- AI Markdown reformatting for news articles with usage metering and sidebar cleanup - - -## 2026-06-17 🟢 - -- Backup download restricted to primary admin, admin-hidden projects invisible to other admins -- Access token system with CLI management and wildcard file type support -- Access token issuance with JSON/form login endpoint and token lifecycle management - - -## 2026-06-16 🔥 Big day! - -- Gateway admin UI with provider and model routing for OpenAI gateway -- Backup management CLI commands and service layer with configurable data directories -- Audit logging for admin trash restore/purge and notification clear, plus SEO noindex for private projects and sitemap docs page refactor -- Instance lookup by name in addition to uid and slug, new terminal session service -- Keyboard-aware input visibility with ResizeObserver fallback for mobile message layout -- E2E comment hierarchy seed helpers for gists, news and projects -- Optimistic message insertion disabled to prevent duplicate bubbles -- Router tree documented in AGENTS.md with 14 new route entries -- Add dpc binary to container image and set executable permissions -- Remove .html and .svg from allowed upload types and MIME mappings - - -## 2026-06-15 🟢 - -- ASGI lifespan handler with background service orchestration and lock-based worker coordination -- Message chunking with sentence-aware splitting and configurable character limits -- Enforce hard test-coverage standard across DevPlace workflows and agents -- Audio file support with inline player and expanded allowed upload types -- Gist comment form integration with card-scoped comment targeting -- Bot account API key adoption for per-user gateway spend attribution - - -## 2026-06-14 🔥 Massive day! - -- Admin/internal database API with CRUD, natural-language query, and read-only SQL execution -- DeepSearch research job queue with CLI prune/clear, Chroma vector store, and date-aware system message composition -- DeepSearch multi-agent researcher with grounded RAG chat and per-session vector store -- SEO Diagnostics tool with CLI management, live WebSocket progress, and static asset cache-busting -- OpenAI-compatible embeddings endpoint with model mapping and usage tracking -- Stealth HTTP client with curl_cffi transport adapter replacing raw httpx for outbound requests -- Bot monitor with live age badges and zoomable screenshots -- Author-interleaved feed ordering across all feed views and tabs -- Author diversity via interleaving (no per-author cap) for home and feed -- devRant API client library and example scripts in Python and JavaScript -- TTLCache-backed cache version reads with invalidation on bump -- Random client IP spoofing for load-testing traffic -- Deepsearch chat component attribute naming from data-* to direct properties -- Replace `python -m agents.validator` with `hawk` across all agent markdown files - - -## 2026-06-13 🔥 Massive day! - -- Three-tier test suite with unit, API, and E2E directories mirroring source and endpoint paths -- Soft-delete audit for bookmarks, comments, follows, polls, project files, reactions, and bug create request event -- Notification preferences with per-user per-channel toggles and admin defaults -- Author diversity enforcement across home page and feed with personalized landing for authenticated users -- Shared free-text search across feed, gists, and projects listings -- Docs search with agent-powered Docii chat and admin-configurable search mode -- Devii agent audit log query action with filterable paginated endpoint -- Router directory-tree convention with admin audit log, AI quota, and container management endpoints -- Initial maintenance agent fleet with per-dimension code quality enforcers -- Unified blob sharding on uuid7 random tail across attachments, project files, and zip service -- Consolidated runtime data directory layout with migration CLI command -- Context-aware window control button visibility with font size boundary detection and minimize/normalize size presets -- Overflow-managed profile tabs with a "more" dropdown for narrow screens -- Startup jitter, randomized browser fingerprinting, and short comment styles for bot realism -- Sidebar search form with hidden field support and configurable placeholder -- Prevent titlebar double-click maximize when clicking buttons in FloatingWindow and DeviiTerminal -- Pin test server to single worker and use upsert for rate-limit settings to prevent spurious 429s -- DEVPLACE_DISABLE_RATE_LIMIT env var to bypass rate limiter in tests and middleware -- Fallback to location.origin when DEVPLACE_DOCS.base is missing -- Add claude-manual task-oriented guide page with cross-reference from claude.html -- Remove PWA install button and associated installer module -- Removed stale test files and fixed Gitea env teardown and ingress proxy test cleanup -- Locustfile seed data expansion and route exclusion documentation - - -## 2026-06-12 🔥 Massive day! - -- Agent report system with codenames, timestamped output streams, and write-budget enforcement -- Tool-scoped payload filtering for worker agents with orchestration tool isolation -- Agent isolation and result caching in Maestro review sweep -- Concurrent read-only fleet check mode with per-agent cost tracking and contextvar-isolated findings -- Admin analytics and AI usage API response keys renamed, password change toggle added -- Gitea-backed bug tracker with list/detail/comment/status and AI-enhanced filing -- Bug detail page with admin/member role rendering and viewer_is_admin context flag -- Changed-files fast mode for maintenance agents with write-allowlist guard -- Partial config save with error reporting and password manager suppression -- Admin route cache-disabling headers via Cache-Control, Pragma and Expires -- Dirty-field tracking and server-side value sync for service config forms -- Rename `is_admin` to `viewer_is_admin` in bug detail schema, router, and template -- Bug tracker unavailable page with JSON and HTML 503 error responses -- Bot comments avoid repeating sibling opinions via thread-aware distinctness prompt -- Default Gitea repository changed from pydevplace to devplacepy -- Remove pytest-xdist parallel test execution, switch to serial single-process test runner - - -## 2026-06-11 🔥 Massive day! - -- Platform-wide soft delete with deleted_at/deleted_by columns and admin trash management -- Owner-or-admin soft-delete enforcement on all content endpoints -- Unified image lightbox with attribute-wired opening and per-user media tab with soft delete -- Autonomous maintenance agent fleet with CLI entry point, Makefile targets, and dependency-free validator -- Seed-finding guided fix mode for maintenance agents with incomplete report tracking -- Audit log tables with CLI recording hooks -- Resolve merge conflict in pagination template and add admin-audit-log endpoint to docs API -- Bots documentation pages and session stop/reset commands -- Reduced nested comment indentation from 1.5rem to 0.25rem per depth level -- Reduce comment indentation multiplier and padding for nested replies -- Switch to dynamic viewport height and remove autofocus from message input -- Inline message layout with responsive height and auto-scroll -- Optional label attribute with hidden empty state for dp-upload component -- Mandatory retoor header added to all devplacepy source files - - -## 2026-06-10 🟢 - -- Port conflict detection and test isolation hardening across admin, avatar, bugs, landing, messages, and customization tests -- Container proxy routing via container IP instead of host port, with fake backend network simulation -- XDG-compliant devii tasks database path with DEVII_HOME override -- Project editing endpoint with 125k char body limit and remote URL attachment guard - - -## 2026-06-09 🔥 Big day! - -- Container manager with Dockerfile CRUD, image builds, instance lifecycle, ingress proxy, and CLI commands -- Async project fork service with job queue, CLI management, and shared container image build -- Parallel test execution with per-worker isolated databases, data dirs, and uvicorn subprocesses via pytest-xdist -- Per-user customization suppression toggles with profile UI and Devii tool -- Customization toggle UI with enable/disable state management -- Unified shared Http and Poller utilities across all frontend modules, replacing inline fetch and setInterval patterns -- Responsive refinements for sub-360px screens, touch targets, safe-area insets, and mobile window controls -- Click-to-open profile dropdown with keyboard and outside-click dismissal -- Migrate hardcoded spacing values to CSS custom properties across multiple stylesheets -- Unicode escape normalization for emoji constants across codebase -- Consolidated upload ignore rules into a single directory-level gitignore entry -- Removed unused imports across routers, database, and services -- Remove project_set_private from confirmation-required actions and fix async test helpers - - -## 2026-06-08 🟢 - -- Admin AI quota management with CLI and admin panel reset controls -- API key management CLI with backfill command and auth support across session, API key, and HTTP Basic -- Per-project filesystem with directory and file CRUD, upload, and inline editing -- Async zip job framework with CLI management and zip archive download endpoints -- Add mistune dependency to project - - -## 2026-06-06 🟢 - -- Reactions, bookmarks, polls, extended sessions, and operational settings - - -## 2026-06-05 🔥 Massive day! - -- Batch attachment linking, deduplicated mention notifications, and idempotent badge milestone checks -- Cursor-based load-more pagination across feed, gists, news and projects -- Canonical slug redirects, cursor-based next-page links, and OG image extraction across feed, gists, news, posts, projects, and profile -- TTLCache with LRU eviction, CLI role management, content unit helpers, database query functions, follow API with XP rewards, and news service with AI grading -- Unified comment form component with mobile touch optimizations across all CSS -- Inline comment previews on post cards with per-comment reply forms -- Comment template with threaded voting, author display, and attachment support -- Post-login redirect with `next` parameter and unauthenticated comment redirect to login -- Login redirect for unauthenticated admin, next parameter support with external URL rejection, and inline comment reply forms -- Seed comments created for all posts instead of only the first -- Replace uuid4 with uuid7 via uuid_utils for push notification JWT jti claims -- Coverage instrumentation for CI and local test runs with HTML report artifact -- Coverage configuration with subprocess measurement support -- Sitemap TTL configurable via environment variable and news_images schema migration -- Kill stale server process and add startup failure detection for Locust targets - - -## 2026-06-02 🟢 - -- Multi-worker service lock with cascading vote/comment cleanup on content deletion - - -## 2026-05-30 🟢 - -- Leaderboard route with gamification system (XP, levels, badges, stars) and content creation refactor - - -## 2026-05-28 🟢 - -- Cursor-based pagination for feed, notifications, and votes with thumbnail extension fallback -- Push registration returns creation flag and only sends welcome notification on first registration - - -## 2026-05-27 🟢 - -- AJAX vote buttons with live count updates across posts, gists, projects, and comments -- CSS-only card-link overlay replacing JS-driven data-href navigation - - -## 2026-05-25 🟢 - -- Unified notification click-to-navigate with comment anchor highlighting and dismiss refactor - - -## 2026-05-23 🔥 Massive day! - -- Web push notifications with PWA manifest and service worker registration -- Web push notifications with PWA offline shell and install prompt -- Unified badge, notification, and content enrichment system with star tracking helpers -- Aggregate star counts across posts, projects and gists for profile and top-author ranking -- Content editing and deletion with cascading cleanup, avatar image helper, HTTP form POST, text input cursor management, and toast flash utility -- Share button with clipboard copy across detail pages, structured data schemas for gists and news articles, configurable site URL and rate limit, and production proxy headers support -- Production deployment workflow via git merge master into production -- Automatic production deployment on successful master push -- Removed automatic production deployment from CI pipeline -- Admin settings form with Pydantic validation and model-driven save -- Pydantic form models with validation for signup, login, password reset, comments, bugs, admin actions, and posts -- Type-safe integer settings with empty-value skip on admin save -- Input validation tests for votes, posts, profile, and signup endpoints -- Rate-limit environment variable and expanded Locust seed data for gists, notifications, and uploads -- TTLCache with ETag-based HTTP caching for avatar endpoint -- Dynamic language sidebar filtering based on existing gist language codes -- Vendor static assets for CodeMirror, highlight.js, marked, and emoji picker -- Test server log capture via tempfile with reduced log verbosity -- DOMPurify XSS sanitization for client-side rendered markdown content -- Add mobile-web-app-capable meta tag for PWA support -- Topnav notification bell selector scoped to /notifications href -- Fix notification bell icon locator to use explicit href selector instead of first match -- Remove stale import of get_users_by_uids from project_detail endpoint - - -## 2026-05-22 🟢 - -- News article HTML sanitization CLI command and database migration - - -## 2026-05-19 🟢 - -- Avatar generation exception logging with full traceback -- Fix multiavatar import path and add required arguments to function call - - -## 2026-05-16 🟢 - -- Clickable post titles and content with downvote support on feed and detail pages -- Interactive vote buttons and clickable post titles on profile page -- Handle @-mention with preceding text in content rendering -- Unread notification cache invalidation across comments, follows, messages, votes, and mentions -- Compact send button, attachment upload container, and auto-scroll on message thread load -- GistEditor lazy init with modal observer, CodeMirror Rust mode removed, emoji picker module type, source textarea required removed, projects tab spacing and settings button removed -- Add space between icon and label in feed navigation tabs - - -## 2026-05-15 🟢 - -- Python 3.13 base image, default port 10500, and nginx template to conf.d migration -- Responsive mobile navigation and messages layout with hamburger menu and back button -- Responsive breakpoint widened from 768px to 1024px for topnav, breadcrumb and page layouts - - -## 2026-05-14 🟢 - -- Migrate from deprecated `datetime.utcnow()` to timezone-aware `datetime.now(timezone.utc)` across the entire codebase -- Wait-for-url stabilization in noindex tests for messages and notifications pages -- Disable parallel test execution in CI pipeline - - -## 2026-05-13 🟢 - -- Migrate all TemplateResponse calls to pass request as first positional argument -- Attachment linking and deletion refactored into dedicated module with batch support -- Parallelised integration test suite with xdist worker port isolation -- Remove deprecated imghdr dependency and fix icon spacing in bug report buttons -- Replace hardcoded pytest.BASE_URL with conftest BASE_URL in attachment tests -- CI trigger branch from main to master - - -## 2026-05-12 🟢 - -- News service with admin curation, landing page articles, and comment support -- Mention notification system across bugs, comments, messages, posts, and projects with user search API -- Gists page with code snippet sharing, voting, and comment integration - - -## 2026-05-11 🔥 Big day! - -- Unified threaded comment system with polymorphic target support across posts, projects, and bugs -- News management system with admin panel, pagination, and SEO sitemap integration -- News background service framework with CLI management, bug reports router, and admin services monitoring -- Admin panel with user management CLI, SEO metadata, and production deployment config -- Multiavatar local SVG generation with WAL mode SQLite and Locust load testing -- CI branch target renamed from main to master and test fixtures refactored for explicit login and seeded database -- Test fixture improvements with debug logging, stderr capture, and extended startup timeout -- Remove hawk static analysis step from CI test workflow - - -## 2026-05-10 🚀 First commit! - -- Initial project scaffold with FastAPI SSR app, auth, feed, posts, comments, projects, profile, messages, notifications, and voting -- DiceBear avatar proxy with style picker on signup and profile, threaded comments -- Image upload support for posts with daily topic display on landing and feed - -──────────────────────────────────────────────────────────── - -Summary: 194 commits over 29 active days. The project launched on May 10 with the initial FastAPI scaffold, auth, feed, and core content features. The biggest pushes came on June 13 (23 commits) delivering the three-tier test suite, soft-delete audit system, notification preferences, and author diversity enforcement; June 23 (23 commits) adding web push notifications, PWA support, content editing/deletion, and production deployment workflows; and June 14 (14 commits) introducing the admin database API, DeepSearch research system, SEO diagnostics, and the stealth HTTP client. - diff --git a/CLAUDE.md b/CLAUDE.md index 59a57557..932c21cd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -159,7 +159,7 @@ Nested `CLAUDE.md` files (loaded automatically by Claude Code only when a file i | `devplacepy/templates/CLAUDE.md` | Modal system, CDN libraries, shared template partials | | `tests/CLAUDE.md` | Detailed testing patterns and pitfalls | -`isslop/` at the repo root is a separate, standalone sibling project (own `pyproject.toml`, `Makefile`, port 18732) with its own `isslop/CLAUDE.md` - it is not a nested subsystem of the `devplacepy` package. The integrated engine that DevPlace actually runs (`devplace isslop analyze`, the `/tools/isslop` job service) is a distinct implementation documented in `devplacepy/services/jobs/CLAUDE.md`. +The AI usage analyzer (`devplace isslop analyze`, the `/tools/isslop` job service) lives entirely inside the package at `devplacepy/services/jobs/isslop/` and is documented in `devplacepy/services/jobs/CLAUDE.md`. There is no repo-root `isslop/` project. ## Architecture diff --git a/devplacepy/database/remote.py b/devplacepy/database/remote.py deleted file mode 100644 index 29a8e609..00000000 --- a/devplacepy/database/remote.py +++ /dev/null @@ -1,348 +0,0 @@ -# retoor - -import inspect -import os - -import httpx - -from devplacepy.cache import TTLCache -from devplacepy_services.base.db_codec import ( - decode_value, - encode_args, - is_write, - is_write_sql, -) - -_SERVICE_URL = os.environ.get("DEVPLACE_DB_SERVICE_URL", "http://127.0.0.1:10601").rstrip("/") -_INTERNAL_KEY = os.environ.get("DEVPLACE_GATEWAY_INTERNAL_KEY", "").strip() -_CLIENT: httpx.Client | None = None - -# Section 7.3: settings reads tolerate up to 5s staleness. patch_module() -# generically RPCs every devplacepy.database call, bypassing the local -# TTL cache get_setting/get_int_setting had in-process - without this, -# every settings read (rate limiting, maintenance mode, admin dashboards) -# pays a full HTTP round trip to the database broker. -_SETTINGS_CACHE_TTL_SECONDS = 5 -_SETTINGS_CACHE = TTLCache(ttl=_SETTINGS_CACHE_TTL_SECONDS, max_size=512) -_CACHED_SETTINGS_FNS = frozenset({"get_setting", "get_int_setting"}) - - -def _service_url() -> str: - return os.environ.get("DEVPLACE_DB_SERVICE_URL", _SERVICE_URL).rstrip("/") - - -def _headers() -> dict[str, str]: - headers: dict[str, str] = {} - key = os.environ.get("DEVPLACE_GATEWAY_INTERNAL_KEY", _INTERNAL_KEY).strip() - if key: - headers["X-Internal-Key"] = key - return headers - - -def _client() -> httpx.Client: - global _CLIENT - if _CLIENT is None: - _CLIENT = httpx.Client(timeout=30.0) - return _CLIENT - - -def _post(path: str, body: dict) -> object: - response = _client().post( - f"{_service_url()}/{path.lstrip('/')}", - json=body, - headers=_headers(), - ) - if response.status_code >= 400: - payload = response.json() if response.content else {} - message = payload.get("error", "Database service request failed") - raise RuntimeError(message) - if not response.content: - return None - return decode_value(response.json()) - - -def _invoke_cached(fn_name: str, args, kwargs): - cache_key = f"{fn_name}:{args!r}:{sorted(kwargs.items())!r}" - cached = _SETTINGS_CACHE.get(cache_key) - if cached is not None: - return cached - value = _invoke(fn_name, args, kwargs, write=False) - _SETTINGS_CACHE.set(cache_key, value) - return value - - -def _invoke(fn_name: str, args, kwargs, *, write: bool = False): - encoded_args, encoded_kwargs = encode_args(args, kwargs) - payload = { - "fn": fn_name, - "args": encoded_args, - "kwargs": encoded_kwargs, - "write": write, - } - result = _post("internal/invoke", payload) - if isinstance(result, dict) and "result" in result: - return result["result"] - return result - - -class RemoteSearchClause: - def __init__(self, term, fields, author_field=None): - self.term = term.strip() - self.fields = tuple(fields) - self.author_field = author_field - - -class RemoteUidInClause: - def __init__(self, field, uids): - self.field = field - self.uids = frozenset(uids) - - -class RemoteTable: - def __init__(self, db: "RemoteDb", name: str) -> None: - self._db = db - self._name = name - self._column_cache = None - - def __getattr__(self, name: str): - def caller(*args, **kwargs): - return self._db._table_op(self._name, name, args, kwargs) - - return caller - - def has_column(self, name: str) -> bool: - cache = self._column_cache - if cache is None: - sample = self.find(_limit=1) - row = next(iter(sample), None) - cache = set(row.keys()) if row else set() - self._column_cache = cache - return name in cache - - def count(self, **kwargs): - return self._db._table_op(self._name, "count", [], kwargs) - - @property - def table(self): - return self - - @property - def exists(self) -> bool: - return self._name in self._db.tables - -class RemoteDb: - def __init__(self) -> None: - self._tables_cache: list[str] | None = None - - @property - def tables(self) -> list[str]: - if self._tables_cache is None: - result = _post("internal/db-op", {"op": "tables"}) - self._tables_cache = list(result or []) - return self._tables_cache - - def __getitem__(self, name: str) -> RemoteTable: - return RemoteTable(self, name) - - def query(self, sql: str, **params): - encoded_args, encoded_kwargs = encode_args((sql,), params) - result = _post( - "internal/db-op", - { - "op": "query", - "args": encoded_args, - "kwargs": encoded_kwargs, - "write": is_write_sql(sql), - }, - ) - return result or [] - - def _table_op(self, table: str, method: str, args, kwargs, *, write: bool = False): - encoded_args, encoded_kwargs = encode_args(args, kwargs) - result = _post( - "internal/db-op", - { - "op": "table_op", - "table": table, - "method": method, - "args": encoded_args, - "kwargs": encoded_kwargs, - "write": write, - }, - ) - if method in {"insert", "update", "delete"}: - self._tables_cache = None - return result - - @property - def executable(self): - return self - - @property - def in_transaction(self) -> bool: - return False - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc, tb): - return False - - -_LOCAL_REMOTE = frozenset( - { - "get_table", - "refresh_snapshot", - "_in_clause", - "_now_iso", - "text_search_clause", - } -) - - -def _remote_text_search_clause( - table, search, fields=("title", "description"), author_field=None -): - term = (search or "").strip() - if not term: - return None - if type(table).__name__ == "RemoteTable": - return RemoteSearchClause(term, fields, author_field) - from devplacepy.database.content import text_search_clause as local_clause - - return local_clause(table, search, fields, author_field=author_field) - - -def _remote_get_table(name: str): - import devplacepy.database.core as core - - return core.db[name] - - -def _remote_refresh_snapshot() -> None: - return None - - -def patch_module(module) -> None: - import devplacepy.database as db_module - - for name in db_module.__all__: - if name in _LOCAL_REMOTE: - continue - target = getattr(module, name, None) - if target is None or not callable(target): - continue - if inspect.isclass(target): - continue - - def make_wrapper(fn_name: str, fn_write: bool): - if fn_name in _CACHED_SETTINGS_FNS: - - def wrapper(*args, **kwargs): - return _invoke_cached(fn_name, args, kwargs) - - wrapper.__name__ = fn_name - return wrapper - - def wrapper(*args, **kwargs): - return _invoke(fn_name, args, kwargs, write=fn_write) - - wrapper.__name__ = fn_name - return wrapper - - setattr(module, name, make_wrapper(name, is_write(name))) - - -def activate() -> None: - import devplacepy.database.core as core - - core.db = RemoteDb() - import devplacepy.database as db_module - - patch_module(db_module) - for submodule_name in ( - "settings", - "users", - "relations", - "pagination", - "soft_delete", - "engagement", - "usage", - "awards", - "seo_meta", - "activity", - "customization", - "email", - "notifications", - "forks", - "follows", - "deepsearch", - "ranking", - "comments", - "content", - "attachments_data", - "stats", - "schema", - ): - try: - submodule = __import__( - f"devplacepy.database.{submodule_name}", - fromlist=[submodule_name], - ) - except ImportError: - continue - patch_module(submodule) - for external_name in ( - "devplacepy.services.statistics.tracking", - "devplacepy.services.base", - "devplacepy.attachments", - "devplacepy.project_files", - ): - try: - external = __import__(external_name, fromlist=[external_name.split(".")[-1]]) - except ImportError: - continue - if hasattr(external, "db"): - external.db = RemoteDb() - db_module.db = core.db - db_module.get_table = _remote_get_table - core.get_table = _remote_get_table - db_module.refresh_snapshot = _remote_refresh_snapshot - core.refresh_snapshot = _remote_refresh_snapshot - db_module.text_search_clause = _remote_text_search_clause - import devplacepy.database.content as content_module - - content_module.text_search_clause = _remote_text_search_clause - for submodule_name in ( - "settings", - "users", - "relations", - "pagination", - "soft_delete", - "engagement", - "usage", - "awards", - "seo_meta", - "activity", - "customization", - "email", - "notifications", - "forks", - "follows", - "deepsearch", - "ranking", - "comments", - "content", - "attachments_data", - "stats", - "schema", - ): - try: - submodule = __import__( - f"devplacepy.database.{submodule_name}", - fromlist=[submodule_name], - ) - except ImportError: - continue - if hasattr(submodule, "db"): - submodule.db = core.db \ No newline at end of file diff --git a/devplacepy/db_client.py b/devplacepy/db_client.py deleted file mode 100644 index d8b3f997..00000000 --- a/devplacepy/db_client.py +++ /dev/null @@ -1,29 +0,0 @@ -# retoor - -import os - - -def _activate() -> None: - if os.environ.get("DEVPLACE_DB_SERVICE") == "1": - return - if os.environ.get("DEVPLACE_REMOTE_DB") == "1": - from devplacepy.database.remote import activate - - activate() - - -_activate() - -import devplacepy.database as _database - - -def _remote_table(table) -> bool: - return type(table).__name__ == "RemoteTable" - - -def __getattr__(name: str): - return getattr(_database, name) - - -def __dir__(): - return sorted(name for name in dir(_database) if not name.startswith("_")) \ No newline at end of file diff --git a/devplacepy/services/audit/CLAUDE.md b/devplacepy/services/audit/CLAUDE.md index 989115fc..2f5b4ba2 100644 --- a/devplacepy/services/audit/CLAUDE.md +++ b/devplacepy/services/audit/CLAUDE.md @@ -2,7 +2,7 @@ This file documents the audit log subsystem. Claude Code auto-loads it when a fi ## Audit log (`services/audit/`) -**Admin-only, append-only** record of every state-changing action. The authoritative event catalogue is `events.md` (its storage model, relation vocabulary, and per-event specs are authoritative); the catalogue currently spans 223 keys across 38 domains. +**Admin-only, append-only** record of every state-changing action. The authoritative event catalogue is `events.md` (its storage model, relation vocabulary, and per-event specs are authoritative); the catalogue currently spans 288 keys across 42 domains. ### Package layout diff --git a/devplacepy/services/containers/CLAUDE.md b/devplacepy/services/containers/CLAUDE.md index f37fde4d..f462ebaa 100644 --- a/devplacepy/services/containers/CLAUDE.md +++ b/devplacepy/services/containers/CLAUDE.md @@ -120,6 +120,7 @@ The security hotpatch that used to run per build is now baked into `ppy.Dockerfi - `COPY`s the **sudo superclone** (`files/sudo`) over `/usr/local/bin/sudo` (+ symlink `/usr/bin/sudo`; the real `sudo` package is not installed). - `COPY`s the **`aptroot` fakeroot wrapper** (`files/aptroot`, symlinked over `apt`/`apt-get`/`dpkg` in `/usr/local/bin` so pravda installs system packages without root). - `COPY`s **`pagent`** (`files/pagent`, the stdlib AI agent; reads `DEVPLACE_OPENAI_URL`+`DEVPLACE_API_KEY`, falling back to its public endpoint + `DEEPSEEK_API_KEY`) to `/usr/bin/pagent.py`, plus `files/.vimrc` to `/home/pravda/.vimrc` (whose AI helper - `AiEditSelection` - targets the same gateway as pagent via `DEVPLACE_OPENAI_URL`/`DEVPLACE_API_KEY`, with a public fallback, never `api.openai.com`). +- `COPY`s **`dpc`** (`files/dpc`, DevPlace Code, the Claude-Code-class coding agent) to `/usr/bin/dpc`, and **`bot.py`** to `/usr/bin/botje.py`. **`dpc` is the one prebuilt binary in this repository**: a stripped ELF 64-bit x86-64 executable, 3578664 bytes, sha256 `24f7fbb068e8461e085c5d49ea92556afc10452e4610de7784b87650f96377dd`, linked against GCC (Ubuntu 15.2.0-4ubuntu4) 15.2. Its source is NOT in this repository and there is no build recipe here, so unlike every other file in `files/` it cannot be reviewed before it is installed root-owned onto `PATH` in every user container. Record a new checksum here whenever it is replaced. - Evicts any pre-existing uid-1000 user, creates user **`pravda` at `1000:1000`**. - Hands pravda ownership of the toolchain AND the OS package trees (`chown -R pravda` over `/usr/local/lib`, `/usr/local/bin`, `/usr/lib/python3`, `/opt`, `/app`, `/home/pravda`, plus `/usr/lib`, `/usr/bin`, `/usr/sbin`, `/usr/share`, `/usr/include`, `/etc`, `/var/lib`, `/var/cache`, `/var/log`, `/srv` so `apt`/`dpkg` can write; `~/.local/bin` on `PATH`). - Ends on `USER pravda`. diff --git a/devplacepy/services/registry.py b/devplacepy/services/registry.py index ad47d2a3..bc5df60f 100644 --- a/devplacepy/services/registry.py +++ b/devplacepy/services/registry.py @@ -42,7 +42,7 @@ def admin_shell_manager() -> ServiceManager: Never supervised (no set_lock_owner/supervise call) - used only for describe()/config metadata/key derivation, all of which read or write - through db_client to the shared service_state / site_settings tables. + through the database layer to the shared service_state / site_settings tables. The owning Tier 3 process is the only one that ever ticks these services. """ global _SHELL_MANAGER diff --git a/devplacepy_services/__init__.py b/devplacepy_services/__init__.py deleted file mode 100644 index b8f73fd5..00000000 --- a/devplacepy_services/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# retoor \ No newline at end of file diff --git a/devplacepy_services/audit/SERVICE.md b/devplacepy_services/audit/SERVICE.md deleted file mode 100644 index 0e2f1d67..00000000 --- a/devplacepy_services/audit/SERVICE.md +++ /dev/null @@ -1 +0,0 @@ -# Audit Service (stub) \ No newline at end of file diff --git a/devplacepy_services/audit/__init__.py b/devplacepy_services/audit/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/devplacepy_services/audit/main.py b/devplacepy_services/audit/main.py deleted file mode 100644 index 5d4377c3..00000000 --- a/devplacepy_services/audit/main.py +++ /dev/null @@ -1,23 +0,0 @@ -import devplacepy_services.base.bootstrap - -from devplacepy.services.audit import AuditService -from devplacepy_services.base.service import BaseMicroservice, build_standard_app - - -class AuditMicroservice(BaseMicroservice): - name = "audit" - title = "Audit" - default_port = 10639 - workers = 1 - stateful = True - depends_on = ["database"] - managed_services = [AuditService()] - use_background = True - run_supervisor = True - - def build_app(self): - return build_standard_app(self) - - -_service = AuditMicroservice() -app = _service.build_app() \ No newline at end of file diff --git a/devplacepy_services/backup/SERVICE.md b/devplacepy_services/backup/SERVICE.md deleted file mode 100644 index 9091af55..00000000 --- a/devplacepy_services/backup/SERVICE.md +++ /dev/null @@ -1 +0,0 @@ -# Backup Service (stub) \ No newline at end of file diff --git a/devplacepy_services/backup/__init__.py b/devplacepy_services/backup/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/devplacepy_services/backup/main.py b/devplacepy_services/backup/main.py deleted file mode 100644 index c4887186..00000000 --- a/devplacepy_services/backup/main.py +++ /dev/null @@ -1,23 +0,0 @@ -import devplacepy_services.base.bootstrap - -from devplacepy.services.backup.service import BackupService -from devplacepy_services.base.service import BaseMicroservice, build_standard_app - - -class BackupMicroservice(BaseMicroservice): - name = "backup" - title = "Backup" - default_port = 10633 - workers = 1 - stateful = True - depends_on = ["database"] - managed_services = [BackupService()] - use_background = True - run_supervisor = True - - def build_app(self): - return build_standard_app(self) - - -_service = BackupMicroservice() -app = _service.build_app() \ No newline at end of file diff --git a/devplacepy_services/base/__init__.py b/devplacepy_services/base/__init__.py deleted file mode 100644 index 4f9f2712..00000000 --- a/devplacepy_services/base/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# retoor - -from devplacepy_services.base.service import BaseMicroservice - -__all__ = ["BaseMicroservice"] \ No newline at end of file diff --git a/devplacepy_services/base/auth.py b/devplacepy_services/base/auth.py deleted file mode 100644 index d96d5150..00000000 --- a/devplacepy_services/base/auth.py +++ /dev/null @@ -1,61 +0,0 @@ -# retoor - -from __future__ import annotations - -import os -from typing import Annotated - -from fastapi import Header, Request - -from devplacepy_services.base.errors import http_error -from devplacepy_services.base.schemas import InternalUser - -_KEY_CACHE = "" - - -def internal_gateway_key() -> str: - global _KEY_CACHE - if _KEY_CACHE: - return _KEY_CACHE - env_key = os.environ.get("DEVPLACE_GATEWAY_INTERNAL_KEY", "").strip() - if env_key: - _KEY_CACHE = env_key - return env_key - try: - from devplacepy.db_client import internal_gateway_key as load_key - - loaded = load_key().strip() - if loaded: - _KEY_CACHE = loaded - return loaded - except ImportError: - return "" - - -def set_internal_gateway_key(value: str) -> None: - global _KEY_CACHE - _KEY_CACHE = value.strip() - - -def validate_internal_key(presented: str | None) -> bool: - expected = internal_gateway_key() - if not expected: - return True - if not presented: - return False - return presented.strip() == expected - - -def require_internal_key(request: Request) -> None: - if validate_internal_key(request.headers.get("X-Internal-Key")): - return - raise http_error(401, "Unauthorized", "unauthorized") - - -async def internal_user( - request: Request, - x_authenticated_user: Annotated[str | None, Header()] = None, -) -> InternalUser: - require_internal_key(request) - uid = x_authenticated_user.strip() if x_authenticated_user else None - return InternalUser(uid=uid or None) \ No newline at end of file diff --git a/devplacepy_services/base/bootstrap.py b/devplacepy_services/base/bootstrap.py deleted file mode 100644 index d88f16d4..00000000 --- a/devplacepy_services/base/bootstrap.py +++ /dev/null @@ -1,4 +0,0 @@ -import os - -os.environ["DEVPLACE_REMOTE_DB"] = "1" -import devplacepy.db_client \ No newline at end of file diff --git a/devplacepy_services/base/cache.py b/devplacepy_services/base/cache.py deleted file mode 100644 index 9e29d9aa..00000000 --- a/devplacepy_services/base/cache.py +++ /dev/null @@ -1,67 +0,0 @@ -# retoor - -from __future__ import annotations - -import time -from collections import OrderedDict - - -class TTLCache: - def __init__(self, ttl: int, max_size: int = 0): - self.ttl = ttl - self.max_size = max_size - self._store: OrderedDict[str, tuple[object, float]] = OrderedDict() - - def get(self, key: str): - entry = self._store.get(key) - if entry is None: - return None - value, expiry = entry - if time.time() >= expiry: - self._store.pop(key, None) - return None - self._store.move_to_end(key) - return value - - def set(self, key: str, value) -> None: - self._store[key] = (value, time.time() + self.ttl) - self._store.move_to_end(key) - if self.max_size and len(self._store) > self.max_size: - self._store.popitem(last=False) - - def pop(self, key: str) -> None: - self._store.pop(key, None) - - def clear(self) -> None: - self._store.clear() - - -class CacheStateGate: - def __init__(self, *, ttl: int = 60, max_staleness: float = 5.0): - self.ttl = ttl - self.max_staleness = max_staleness - self._cache = TTLCache(ttl=ttl) - self._versions: dict[str, int] = {} - self._bumped_at: dict[str, float] = {} - - def get_version(self, name: str) -> int: - return int(self._versions.get(name, 0)) - - def bump(self, name: str) -> int: - next_version = self.get_version(name) + 1 - self._versions[name] = next_version - self._bumped_at[name] = time.time() - self._cache.clear() - return next_version - - def get(self, key: str, version_name: str, loader): - bumped_at = self._bumped_at.get(version_name, 0.0) - if bumped_at and (time.time() - bumped_at) < self.max_staleness: - self._cache.pop(key) - version_key = f"{version_name}:{self.get_version(version_name)}" - cached = self._cache.get(version_key) - if cached is not None: - return cached - value = loader() - self._cache.set(version_key, value) - return value \ No newline at end of file diff --git a/devplacepy_services/base/compound.py b/devplacepy_services/base/compound.py deleted file mode 100644 index 54c5b6f4..00000000 --- a/devplacepy_services/base/compound.py +++ /dev/null @@ -1,153 +0,0 @@ -# retoor - -from __future__ import annotations - -from typing import Any - -from pydantic import BaseModel, Field - - -class InvokeIn(BaseModel): - fn: str - args: list[Any] = Field(default_factory=list) - kwargs: dict[str, Any] = Field(default_factory=dict) - write: bool = False - - -class CacheBumpIn(BaseModel): - name: str - - -class UsersByUidsIn(BaseModel): - uids: list[str] - - -class CommentCountsIn(BaseModel): - post_uids: list[str] - - -class VoteCountsIn(BaseModel): - target_uids: list[str] - - -class ReactionsIn(BaseModel): - target_type: str - target_uids: list[str] - user: dict[str, Any] | None = None - - -class BookmarksIn(BaseModel): - user_uid: str - target_type: str - target_uids: list[str] - - -class PollsIn(BaseModel): - post_uids: list[str] - user: dict[str, Any] | None = None - - -class RecentCommentsIn(BaseModel): - post_uids: list[str] - limit: int = 3 - user: dict[str, Any] | None = None - - -class CommentsIn(BaseModel): - target_type: str - target_uid: str - user: dict[str, Any] | None = None - - -class FollowBundleIn(BaseModel): - user_uid: str - target_uids: list[str] | None = None - - -class RelationsIn(BaseModel): - viewer_uid: str | None = None - - -class OnlineUsersIn(BaseModel): - cutoff_iso: str - limit: int = 30 - - -class NotificationPrefsIn(BaseModel): - user_uid: str - - -class LeaderboardBundleIn(BaseModel): - limit: int = 50 - offset: int = 0 - viewer_uid: str | None = None - - -class SiteSidebarIn(BaseModel): - authors_limit: int = 5 - - -class AwardsBundleIn(BaseModel): - receiver_uid: str - page: int = 1 - per_page: int = 12 - profile_user: dict[str, Any] | None = None - - -class AttachmentsIn(BaseModel): - resource_type: str - resource_uids: list[str] - - -class UserMediaIn(BaseModel): - user_uid: str - page: int = 1 - per_page: int = 24 - - -class SeoMetaIn(BaseModel): - target_type: str - uids: list[str] - - -class FeedPageIn(BaseModel): - user: dict[str, Any] | None = None - tab: str = "all" - topic: str | None = None - search: str = "" - before: str | None = None - - -class PostDetailIn(BaseModel): - post_uid: str - user: dict[str, Any] | None = None - - -class ProfileBundleIn(BaseModel): - profile_uid: str - viewer: dict[str, Any] | None = None - - -class MessagesPageIn(BaseModel): - user_uid: str - - -class NotificationsPageIn(BaseModel): - user_uid: str - - -class LeaderboardPageIn(BaseModel): - viewer_uid: str | None = None - - -class ProjectDetailIn(BaseModel): - project_uid: str - - -class GistDetailIn(BaseModel): - gist_uid: str - user: dict[str, Any] | None = None - - -class GameStateIn(BaseModel): - user_uid: str | None = None \ No newline at end of file diff --git a/devplacepy_services/base/config.py b/devplacepy_services/base/config.py deleted file mode 100644 index 4c5ffc89..00000000 --- a/devplacepy_services/base/config.py +++ /dev/null @@ -1,70 +0,0 @@ -# retoor - -from __future__ import annotations - -import os -from pathlib import Path - -from devplacepy_services.base.manifest import ( - FORBIDDEN_GAPS, - PORT_HINTS, - SERVICE_URL_ENV, - SERVICES, - build_profiles, -) - -PROFILES = build_profiles() - -_BROKER_URL_ENV = { - "main": "DEVPLACE_DB_SERVICE_URL", - "devii_tasks": "DEVPLACE_DEVII_TASKS_STORE_URL", - "devii_lessons": "DEVPLACE_DEVII_LESSONS_STORE_URL", -} - - -def port_profile() -> str: - return os.environ.get("DEVPLACE_PORT_PROFILE", "micro-dev") - - -def data_dir() -> Path: - root = Path(__file__).resolve().parents[2] - return Path(os.environ.get("DEVPLACE_DATA_DIR", str(root / "data"))) - - -def service_url(name: str) -> str: - env_key = SERVICE_URL_ENV.get(name) - if env_key: - override = os.environ.get(env_key, "").strip().rstrip("/") - if override: - return override - profile = port_profile() - host = os.environ.get("DEVPLACE_SERVICE_HOST", "127.0.0.1") - port = PROFILES[profile][name] - return f"http://{host}:{port}" - - -def broker_url(name: str) -> str: - env_key = _BROKER_URL_ENV.get(name) - if env_key: - override = os.environ.get(env_key, "").strip().rstrip("/") - if override: - return override - if name == "main": - return service_url("database") - if name == "devii_tasks": - return f"{service_url('devii')}/internal/store/devii_tasks" - if name == "devii_lessons": - return f"{service_url('devii')}/internal/store/devii_lessons" - raise KeyError(name) - - -def sqlite_read_pool_size() -> int: - raw = os.environ.get("DEVPLACE_SQLITE_READ_POOL", "8").strip() - try: - return max(1, int(raw)) - except ValueError: - return 8 - - -def service_spec(name: str): - return SERVICES[name] \ No newline at end of file diff --git a/devplacepy_services/base/db_client.py b/devplacepy_services/base/db_client.py deleted file mode 100644 index 4c643765..00000000 --- a/devplacepy_services/base/db_client.py +++ /dev/null @@ -1,236 +0,0 @@ -# retoor - -from __future__ import annotations - -from typing import Any - -from devplacepy_services.base.compound import ( - AttachmentsIn, - AwardsBundleIn, - BookmarksIn, - CacheBumpIn, - CommentCountsIn, - CommentsIn, - FeedPageIn, - FollowBundleIn, - LeaderboardBundleIn, - NotificationPrefsIn, - OnlineUsersIn, - PollsIn, - ReactionsIn, - RecentCommentsIn, - RelationsIn, - SeoMetaIn, - SiteSidebarIn, - UserMediaIn, - UsersByUidsIn, - VoteCountsIn, -) -from devplacepy_services.base.errors import http_error -from devplacepy_services.base.http import internal_request - - -class BrokerClient: - def __init__(self, name: str, url: str | None = None) -> None: - self.name = name - if url is not None: - self.url = url.rstrip("/") - else: - from devplacepy_services.base.config import broker_url - - self.url = broker_url(name).rstrip("/") - - async def post(self, path: str, body: dict[str, Any] | None = None) -> Any: - target = f"{self.url}/{path.lstrip('/')}" - response = await internal_request("POST", target, json=body or {}) - if response.status_code >= 400: - payload = response.json() if response.content else {} - message = payload.get("error", "Broker request failed") - code = payload.get("code", "broker_error") - raise http_error(response.status_code, message, code) - if not response.content: - return None - return response.json() - - async def get(self, path: str, *, params: dict[str, Any] | None = None) -> Any: - target = f"{self.url}/{path.lstrip('/')}" - response = await internal_request("GET", target, params=params) - if response.status_code >= 400: - payload = response.json() if response.content else {} - message = payload.get("error", "Broker request failed") - code = payload.get("code", "broker_error") - raise http_error(response.status_code, message, code) - if not response.content: - return None - return response.json() - - -main = BrokerClient("main") -devii_tasks = BrokerClient("devii_tasks") -devii_lessons = BrokerClient("devii_lessons") - - -async def get_setting(key: str, default: str = "") -> str: - payload = await main.get(f"settings/{key}", params={"default": default}) - return payload.get("value", default) if isinstance(payload, dict) else default - - -async def get_int_setting(key: str, default: int) -> int: - raw = await get_setting(key, str(default)) - try: - return int(raw) - except (TypeError, ValueError): - return default - - -async def bump_cache_version(name: str) -> None: - await main.post("/cache/bump", CacheBumpIn(name=name).model_dump()) - - -async def get_users_by_uids(uids: list[str]) -> dict: - return await main.post("/compound/users-by-uids", UsersByUidsIn(uids=uids).model_dump()) - - -async def get_comment_counts_by_post_uids(post_uids: list[str]) -> dict: - return await main.post( - "/compound/comment-counts", CommentCountsIn(post_uids=post_uids).model_dump() - ) - - -async def get_vote_counts(target_uids: list[str]) -> tuple[dict, dict]: - payload = await main.post( - "/compound/vote-counts", VoteCountsIn(target_uids=target_uids).model_dump() - ) - return payload.get("ups", {}), payload.get("downs", {}) - - -async def get_reactions_by_targets( - target_type: str, target_uids: list[str], user: dict | None = None -) -> dict: - return await main.post( - "/compound/reactions", - ReactionsIn(target_type=target_type, target_uids=target_uids, user=user).model_dump(), - ) - - -async def get_user_bookmarks( - user_uid: str, target_type: str, target_uids: list[str] -) -> set[str]: - payload = await main.post( - "/compound/bookmarks", - BookmarksIn( - user_uid=user_uid, target_type=target_type, target_uids=target_uids - ).model_dump(), - ) - return set(payload or []) - - -async def get_polls_by_post_uids(post_uids: list[str], user: dict | None = None) -> dict: - return await main.post( - "/compound/polls", PollsIn(post_uids=post_uids, user=user).model_dump() - ) - - -async def get_recent_comments_by_post_uids( - post_uids: list[str], limit: int = 3, user: dict | None = None -) -> dict: - return await main.post( - "/compound/recent-comments", - RecentCommentsIn(post_uids=post_uids, limit=limit, user=user).model_dump(), - ) - - -async def load_comments( - target_type: str, target_uid: str, user: dict | None = None -) -> list: - return await main.post( - "/compound/comments", - CommentsIn(target_type=target_type, target_uid=target_uid, user=user).model_dump(), - ) - - -async def get_follow_bundle( - user_uid: str, target_uids: list[str] | None = None -) -> dict: - return await main.post( - "/compound/follow-bundle", - FollowBundleIn(user_uid=user_uid, target_uids=target_uids).model_dump(), - ) - - -async def get_user_relations_bundle(viewer_uid: str | None = None) -> dict: - return await main.post( - "/compound/relations", RelationsIn(viewer_uid=viewer_uid).model_dump() - ) - - -async def get_online_users(cutoff_iso: str, limit: int = 30) -> list: - return await main.post( - "/compound/online-users", - OnlineUsersIn(cutoff_iso=cutoff_iso, limit=limit).model_dump(), - ) - - -async def get_notification_prefs(user_uid: str) -> list: - return await main.post( - "/compound/notification-prefs", - NotificationPrefsIn(user_uid=user_uid).model_dump(), - ) - - -async def get_leaderboard_bundle( - limit: int = 50, offset: int = 0, viewer_uid: str | None = None -) -> dict: - return await main.post( - "/compound/leaderboard", - LeaderboardBundleIn(limit=limit, offset=offset, viewer_uid=viewer_uid).model_dump(), - ) - - -async def get_site_sidebar(authors_limit: int = 5) -> dict: - return await main.post( - "/compound/site-sidebar", SiteSidebarIn(authors_limit=authors_limit).model_dump() - ) - - -async def get_awards_bundle( - receiver_uid: str, - page: int = 1, - per_page: int = 12, - profile_user: dict | None = None, -) -> dict: - return await main.post( - "/compound/awards", - AwardsBundleIn( - receiver_uid=receiver_uid, - page=page, - per_page=per_page, - profile_user=profile_user, - ).model_dump(), - ) - - -async def get_attachments_batch(resource_type: str, resource_uids: list[str]) -> dict: - return await main.post( - "/compound/attachments", - AttachmentsIn(resource_type=resource_type, resource_uids=resource_uids).model_dump(), - ) - - -async def get_user_media( - user_uid: str, page: int = 1, per_page: int = 24 -) -> dict: - return await main.post( - "/compound/user-media", - UserMediaIn(user_uid=user_uid, page=page, per_page=per_page).model_dump(), - ) - - -async def get_seo_metadata_batch(target_type: str, uids: list[str]) -> dict: - return await main.post( - "/compound/seo-meta", SeoMetaIn(target_type=target_type, uids=uids).model_dump() - ) - - -async def build_feed_page(**kwargs) -> dict: - return await main.post("/compound/feed-page", FeedPageIn(**kwargs).model_dump()) \ No newline at end of file diff --git a/devplacepy_services/base/db_codec.py b/devplacepy_services/base/db_codec.py deleted file mode 100644 index a7788185..00000000 --- a/devplacepy_services/base/db_codec.py +++ /dev/null @@ -1,126 +0,0 @@ -# retoor - -from __future__ import annotations - -from datetime import date, datetime -from pathlib import Path -from typing import Any - -WRITE_PREFIXES = ( - "set_", - "add_", - "delete_", - "create_", - "update_", - "upsert_", - "record_", - "migrate_", - "backfill_", - "mark_", - "invalidate_", - "ensure_", - "restore", - "purge", - "revoke_", - "recompute_", - "soft_delete", - "bump_", - "init_", - "clear_settings", -) - -WRITE_EXACT = frozenset( - { - "soft_delete_in", - "restore_event", - "purge_event", - "delete_engagement", - "soft_delete_engagement", - "delete_fork_relations", - "soft_delete_fork_relations", - "delete_attachments", - "delete_attachment_record", - "_delete_attachment_file", - "_index", - "_drop_index", - "_uid_index", - "_ensure_cache_state", - "_refresh_query_planner_stats", - "_backfill_gamification", - } -) - -REMOTE_TABLE_MARKER = "__remote_table__" - - -def is_write(name: str) -> bool: - if name in WRITE_EXACT: - return True - return any(name.startswith(prefix) for prefix in WRITE_PREFIXES) - - -SQL_WRITE_KEYWORDS = frozenset( - {"INSERT", "UPDATE", "DELETE", "REPLACE", "CREATE", "ALTER", "DROP"} -) - - -def is_write_sql(sql: str) -> bool: - first_word = sql.strip().split(None, 1)[0].upper() if sql.strip() else "" - return first_word in SQL_WRITE_KEYWORDS - - -def encode_value(value: Any) -> Any: - if value is None or isinstance(value, (bool, int, float, str)): - return value - if type(value).__name__ == "RemoteTable": - return {REMOTE_TABLE_MARKER: value._name} - if isinstance(value, (datetime, date)): - return value.isoformat() - if isinstance(value, Path): - return str(value) - if isinstance(value, frozenset): - return [encode_value(item) for item in value] - if isinstance(value, set): - return [encode_value(item) for item in value] - if isinstance(value, tuple): - return [encode_value(item) for item in value] - if isinstance(value, list): - return [encode_value(item) for item in value] - if isinstance(value, dict): - return {str(key): encode_value(item) for key, item in value.items()} - if hasattr(value, "items") and callable(value.items): - try: - return {str(key): encode_value(item) for key, item in value.items()} - except TypeError: - pass - return str(value) - - -def encode_args(args: tuple | list, kwargs: dict) -> tuple[list, dict]: - return [encode_value(item) for item in args], { - str(key): encode_value(value) for key, value in kwargs.items() - } - - -def decode_value(value: Any) -> Any: - if isinstance(value, list): - return [decode_value(item) for item in value] - if isinstance(value, dict): - return {key: decode_value(item) for key, item in value.items()} - return value - - -def decode_arg(value: Any) -> Any: - if isinstance(value, list): - return [decode_arg(item) for item in value] - if isinstance(value, dict): - if set(value) == {REMOTE_TABLE_MARKER}: - from devplacepy.db_client import get_table - - return get_table(value[REMOTE_TABLE_MARKER]) - return {key: decode_arg(item) for key, item in value.items()} - return value - - -def encode_result(value: Any) -> Any: - return encode_value(value) \ No newline at end of file diff --git a/devplacepy_services/base/errors.py b/devplacepy_services/base/errors.py deleted file mode 100644 index c16f527e..00000000 --- a/devplacepy_services/base/errors.py +++ /dev/null @@ -1,63 +0,0 @@ -# retoor - -from fastapi import HTTPException -from fastapi.responses import JSONResponse - -from devplacepy_services.base.schemas import ErrorOut - - -def error_response(status_code: int, message: str, code: str) -> JSONResponse: - return JSONResponse( - ErrorOut(error=message, code=code).model_dump(), - status_code=status_code, - ) - - -def http_error(status_code: int, message: str, code: str) -> HTTPException: - return HTTPException( - status_code=status_code, - detail=ErrorOut(error=message, code=code).model_dump(), - ) - - -def error_code_for_status(status_code: int) -> str: - if status_code == 404: - return "not_found" - if status_code == 401: - return "unauthorized" - if status_code == 403: - return "forbidden" - if status_code == 422: - return "validation_error" - if status_code >= 500: - return "internal_error" - return "error" - - -def error_message_for_status(status_code: int, detail: str | None = None) -> str: - if status_code == 404: - return "Not found" - if status_code == 401: - return "Unauthorized" - if status_code == 403: - return "Forbidden" - if status_code >= 500: - return "Internal server error" - if detail: - return detail - return "Request failed" - - -def sanitize_detail(detail: object, status_code: int = 400) -> tuple[str, str]: - if isinstance(detail, dict): - if "error" in detail and "code" in detail: - return str(detail["error"]), str(detail["code"]) - if "message" in detail: - return str(detail["message"]), error_code_for_status(status_code) - if isinstance(detail, list): - return "Validation failed", "validation_error" - if isinstance(detail, str): - return error_message_for_status(status_code, detail), error_code_for_status(status_code) - if detail is None: - return error_message_for_status(status_code), error_code_for_status(status_code) - return error_message_for_status(status_code, str(detail)), error_code_for_status(status_code) \ No newline at end of file diff --git a/devplacepy_services/base/health.py b/devplacepy_services/base/health.py deleted file mode 100644 index e9c18625..00000000 --- a/devplacepy_services/base/health.py +++ /dev/null @@ -1,57 +0,0 @@ -# retoor - -from __future__ import annotations - -import asyncio -import time - -from fastapi import APIRouter - -from devplacepy_services.base.config import service_url -from devplacepy_services.base.http import internal_request -from devplacepy_services.base.schemas import HealthOut, HealthStatsOut -from devplacepy_services.base.service import BaseMicroservice -from devplacepy_services.base.stats import COUNTERS - - -async def _dep_status(name: str) -> str: - url = f"{service_url(name)}/health" - try: - response = await internal_request("GET", url, timeout=2.0) - if response.status_code == 200: - payload = response.json() - if payload.get("status") == "ok": - return "ok" - return "degraded" - return "down" - except Exception: - return "down" - - -def health_router(service: BaseMicroservice) -> APIRouter: - router = APIRouter() - - @router.get("/health", response_model=HealthOut) - async def health() -> HealthOut: - deps: dict[str, str] = {} - if service.depends_on: - results = await asyncio.gather( - *[_dep_status(dep) for dep in service.depends_on] - ) - for dep_name, status in zip(service.depends_on, results, strict=True): - deps[dep_name] = status - overall = "ok" - if deps and any(status != "ok" for status in deps.values()): - overall = "degraded" - uptime_s = int(time.monotonic() - service.started_at) - stats = COUNTERS.snapshot() - return HealthOut( - service=service.name, - status=overall, - uptime_s=uptime_s, - version=service.version, - deps=deps, - stats=HealthStatsOut(requests=stats["requests"], errors=stats["errors"]), - ) - - return router \ No newline at end of file diff --git a/devplacepy_services/base/http.py b/devplacepy_services/base/http.py deleted file mode 100644 index 7462bc90..00000000 --- a/devplacepy_services/base/http.py +++ /dev/null @@ -1,77 +0,0 @@ -# retoor - -from __future__ import annotations - -import contextvars -import os -from typing import Any - -import httpx - -from devplacepy import stealth - -INTERNAL_CALLS: contextvars.ContextVar[int] = contextvars.ContextVar( - "internal_calls", default=0 -) -REQUEST_ID: contextvars.ContextVar[str] = contextvars.ContextVar("request_id", default="") - -_POOL: httpx.AsyncClient | None = None - - -def bump_internal_calls() -> None: - INTERNAL_CALLS.set(INTERNAL_CALLS.get() + 1) - - -def current_internal_calls() -> int: - return INTERNAL_CALLS.get() - - -def current_request_id() -> str: - return REQUEST_ID.get() - - -async def startup_pool() -> None: - global _POOL - if _POOL is None: - _POOL = httpx.AsyncClient( - limits=httpx.Limits(max_connections=32, max_keepalive_connections=32), - timeout=httpx.Timeout(30.0), - http2=False, - ) - - -async def shutdown_pool() -> None: - global _POOL - if _POOL is not None: - await _POOL.aclose() - _POOL = None - - -def stealth_async_client(**kwargs: Any) -> httpx.AsyncClient: - return stealth.stealth_async_client(**kwargs) - - -async def internal_request( - method: str, - url: str, - *, - json: dict | None = None, - params: dict | None = None, - headers: dict[str, str] | None = None, - timeout: float = 30.0, -) -> httpx.Response: - await startup_pool() - assert _POOL is not None - bump_internal_calls() - merged: dict[str, str] = {} - request_id = current_request_id() - if request_id: - merged["X-Request-Id"] = request_id - internal_key = os.environ.get("DEVPLACE_GATEWAY_INTERNAL_KEY", "").strip() - if internal_key: - merged["X-Internal-Key"] = internal_key - if headers: - merged.update(headers) - return await _POOL.request( - method, url, json=json, params=params, headers=merged, timeout=timeout - ) \ No newline at end of file diff --git a/devplacepy_services/base/manifest.py b/devplacepy_services/base/manifest.py deleted file mode 100644 index ed02e786..00000000 --- a/devplacepy_services/base/manifest.py +++ /dev/null @@ -1,215 +0,0 @@ -# retoor - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Literal - -WorkersSpec = int | Literal["auto"] - - -@dataclass(frozen=True) -class ServiceSpec: - name: str - module: str - port: int - tier: int - workers: WorkersSpec - stateful: bool - depends_on: tuple[str, ...] - health_client_timeout: float = 2.0 - - -@dataclass(frozen=True) -class IngressRoute: - prefix: str - service: str - websocket: bool = False - - -SERVICE_SPECS: tuple[ServiceSpec, ...] = ( - ServiceSpec("database", "devplacepy_services.database.main:app", 10601, 1, 1, True, ()), - ServiceSpec("pubsub", "devplacepy_services.pubsub.main:app", 10602, 1, 1, True, ("database",)), - ServiceSpec( - "web", - "devplacepy_services.web.main:app", - 10500, - 2, - "auto", - False, - ("database", "pubsub"), - health_client_timeout=4.0, - ), - ServiceSpec( - "gateway", - "devplacepy_services.gateway.main:app", - 10620, - 2, - "auto", - False, - ("database",), - health_client_timeout=4.0, - ), - ServiceSpec( - "jobs", - "devplacepy_services.jobs.main:app", - 10630, - 3, - 1, - True, - ("database", "pubsub", "gateway"), - health_client_timeout=8.0, - ), - ServiceSpec( - "devii", - "devplacepy_services.devii.main:app", - 10631, - 3, - 2, - True, - ("database", "pubsub", "gateway"), - health_client_timeout=4.0, - ), - ServiceSpec("bot", "devplacepy_services.bot.main:app", 10632, 3, 1, True, ("database", "gateway")), - ServiceSpec("backup", "devplacepy_services.backup.main:app", 10633, 3, 1, True, ("database",)), - ServiceSpec( - "containers", - "devplacepy_services.containers.main:app", - 10634, - 3, - 1, - True, - ("database",), - health_client_timeout=4.0, - ), - ServiceSpec("telegram", "devplacepy_services.telegram.main:app", 10635, 3, 1, True, ("database", "pubsub")), - ServiceSpec("email", "devplacepy_services.email.main:app", 10636, 3, 1, True, ("database",)), - ServiceSpec( - "news", - "devplacepy_services.news.main:app", - 10637, - 3, - 1, - True, - ("database", "gateway"), - ), - ServiceSpec("gitea", "devplacepy_services.gitea.main:app", 10638, 3, 1, True, ("database", "gateway")), - ServiceSpec("audit", "devplacepy_services.audit.main:app", 10639, 3, 1, True, ("database",)), - ServiceSpec("xmlrpc", "devplacepy_services.xmlrpc.main:app", 10649, 3, 1, True, ("database",)), -) - -SERVICES: dict[str, ServiceSpec] = {spec.name: spec for spec in SERVICE_SPECS} - -TIER_ORDER: dict[int, tuple[str, ...]] = { - 1: ("database", "pubsub"), - 2: ("web", "gateway"), - 3: ( - "jobs", - "devii", - "bot", - "backup", - "containers", - "telegram", - "email", - "news", - "gitea", - "audit", - "xmlrpc", - ), -} - -SERVICE_URL_ENV: dict[str, str] = { - "web": "DEVPLACE_WEB_URL", - "database": "DEVPLACE_DB_SERVICE_URL", - "pubsub": "DEVPLACE_PUBSUB_URL", - "gateway": "DEVPLACE_GATEWAY_URL", - "jobs": "DEVPLACE_JOBS_URL", - "devii": "DEVPLACE_DEVII_URL", - "bot": "DEVPLACE_BOT_URL", - "backup": "DEVPLACE_BACKUP_URL", - "containers": "DEVPLACE_CONTAINERS_URL", - "telegram": "DEVPLACE_TELEGRAM_URL", - "email": "DEVPLACE_EMAIL_URL", - "news": "DEVPLACE_NEWS_URL", - "gitea": "DEVPLACE_GITEA_URL", - "audit": "DEVPLACE_AUDIT_URL", - "xmlrpc": "DEVPLACE_XMLRPC_URL", -} - -INGRESS_ROUTES: tuple[IngressRoute, ...] = ( - IngressRoute("/openai", "gateway"), - IngressRoute("/devii", "devii", websocket=True), - IngressRoute("/pubsub", "pubsub", websocket=True), - IngressRoute("/zips", "jobs"), - IngressRoute("/forks", "jobs"), - IngressRoute("/tools", "jobs", websocket=True), - IngressRoute("/xmlrpc", "xmlrpc"), -) - -INGRESS_WS_PREFIXES: tuple[tuple[str, str], ...] = ( - ( - "/projects/", - "containers", - ), -) - -FORBIDDEN_GAPS: tuple[tuple[int, int], ...] = ( - (10520, 10599), - (10610, 10619), - (10650, 10699), - (10750, 10799), - (20550, 20599), - (20650, 20699), -) - -PORT_HINTS: dict[int, str] = { - 10500: "web service", - 20500: "test orchestrator", - 10502: "locust", -} - -_MICRO_DEV_PORTS: dict[str, int] = {spec.name: spec.port for spec in SERVICE_SPECS} - -XMLRPC_RAW_SOCKET_PORT_DEV = 10648 - - -def xmlrpc_raw_socket_port(profile: str) -> int: - if profile == "micro-test": - return XMLRPC_RAW_SOCKET_PORT_DEV + 10000 - if profile == "micro-prod": - return XMLRPC_RAW_SOCKET_PORT_DEV + 100 - return XMLRPC_RAW_SOCKET_PORT_DEV - - -def build_profiles() -> dict[str, dict[str, int]]: - return { - "micro-dev": dict(_MICRO_DEV_PORTS), - "micro-test": { - k: (20500 if k == "web" else v + 10000) for k, v in _MICRO_DEV_PORTS.items() - }, - "micro-prod": { - k: (10500 if k == "web" else v + 100) for k, v in _MICRO_DEV_PORTS.items() - }, - "locust": {"app": 10502, "ui": 10503}, - } - - -def orchestrator_services_dict() -> dict[str, dict]: - return { - spec.name: { - "module": spec.module, - "port": spec.port, - "tier": spec.tier, - "workers": spec.workers, - "stateful": spec.stateful, - "depends_on": list(spec.depends_on), - } - for spec in SERVICE_SPECS - } - - -def health_client_timeout(name: str) -> float: - spec = SERVICES.get(name) - if spec is None: - return 2.0 - return spec.health_client_timeout \ No newline at end of file diff --git a/devplacepy_services/base/middleware.py b/devplacepy_services/base/middleware.py deleted file mode 100644 index 0ab66c06..00000000 --- a/devplacepy_services/base/middleware.py +++ /dev/null @@ -1,121 +0,0 @@ -# retoor - -from __future__ import annotations - -import json -import logging -import time -import uuid_utils -from starlette.middleware.base import BaseHTTPMiddleware -from starlette.requests import Request -from starlette.responses import Response - -from devplacepy_services.base.auth import validate_internal_key -from devplacepy_services.base.errors import error_response, sanitize_detail -from devplacepy_services.base.http import INTERNAL_CALLS, REQUEST_ID, current_internal_calls -from devplacepy_services.base.stats import COUNTERS - -logger = logging.getLogger(__name__) - -PUBLIC_PATHS = {"/health"} - - -class RequestStatsMiddleware(BaseHTTPMiddleware): - async def dispatch(self, request: Request, call_next): - COUNTERS.record_request() - response = await call_next(request) - if response.status_code >= 500: - COUNTERS.record_error() - return response - - -class InternalCallCounterMiddleware(BaseHTTPMiddleware): - async def dispatch(self, request: Request, call_next): - token_calls = INTERNAL_CALLS.set(0) - response = await call_next(request) - service_name = getattr(request.app.state, "service_name", "") - if service_name in {"web", "gateway"}: - response.headers["X-Internal-Calls"] = str(current_internal_calls()) - INTERNAL_CALLS.reset(token_calls) - return response - - -class InternalAuthMiddleware(BaseHTTPMiddleware): - async def dispatch(self, request: Request, call_next): - path = request.url.path - if path in PUBLIC_PATHS: - return await call_next(request) - if not validate_internal_key(request.headers.get("X-Internal-Key")): - return error_response(401, "Unauthorized", "unauthorized") - return await call_next(request) - - -class SanitizeErrorsMiddleware(BaseHTTPMiddleware): - async def dispatch(self, request: Request, call_next): - request_id = request.headers.get("X-Request-Id") or uuid_utils.uuid7().hex - token_id = REQUEST_ID.set(request_id) - started = time.perf_counter() - try: - response = await call_next(request) - except Exception: - COUNTERS.record_error() - logger.exception( - "unhandled service error", - extra={ - "request_id": request_id, - "service": getattr(request.app.state, "service_name", ""), - "path": request.url.path, - }, - ) - response = error_response(500, "Internal server error", "internal_error") - finally: - REQUEST_ID.reset(token_id) - duration_ms = int((time.perf_counter() - started) * 1000) - service_name = getattr(request.app.state, "service_name", "") - logger.info( - "request", - extra={ - "request_id": request_id, - "service": service_name, - "path": request.url.path, - "status": response.status_code, - "duration_ms": duration_ms, - "internal_calls": current_internal_calls(), - }, - ) - if response.status_code < 400: - return response - if not hasattr(response, "body_iterator"): - return response - content_type = response.headers.get("content-type", "") - if "application/json" not in content_type: - if response.status_code == 404: - return error_response(404, "Not found", "not_found") - if response.status_code == 401: - return error_response(401, "Unauthorized", "unauthorized") - if response.status_code >= 500: - return error_response(500, "Internal server error", "internal_error") - return error_response(response.status_code, "Request failed", "error") - body = b"" - async for chunk in response.body_iterator: - body += chunk - try: - parsed = json.loads(body) - except json.JSONDecodeError: - return error_response(response.status_code, "Request failed", "error") - if isinstance(parsed, dict) and "error" in parsed and "code" in parsed: - return Response( - content=body, - status_code=response.status_code, - media_type="application/json", - ) - if isinstance(parsed, dict) and isinstance(parsed.get("error"), dict) and "message" in parsed["error"]: - return Response( - content=body, - status_code=response.status_code, - media_type="application/json", - ) - if isinstance(parsed, dict) and "detail" in parsed: - message, code = sanitize_detail(parsed["detail"], response.status_code) - return error_response(response.status_code, message, code) - return error_response(response.status_code, "Request failed", "error") \ No newline at end of file diff --git a/devplacepy_services/base/proxy.py b/devplacepy_services/base/proxy.py deleted file mode 100644 index 14a7c6c4..00000000 --- a/devplacepy_services/base/proxy.py +++ /dev/null @@ -1,37 +0,0 @@ -# retoor - -from __future__ import annotations - -from starlette.requests import Request - -HOP_HEADERS = frozenset( - { - "connection", - "keep-alive", - "proxy-authenticate", - "proxy-authorization", - "te", - "trailers", - "transfer-encoding", - "upgrade", - "host", - "content-length", - "content-encoding", - } -) - - -def forward_headers(request: Request, prefix: str) -> dict[str, str]: - headers = { - k: v for k, v in request.headers.items() if k.lower() not in HOP_HEADERS - } - headers["X-Forwarded-Prefix"] = prefix - headers["X-Script-Name"] = prefix - headers["X-Forwarded-Host"] = request.headers.get( - "host", request.url.hostname or "" - ) - headers["X-Forwarded-Proto"] = request.headers.get( - "x-forwarded-proto", request.url.scheme - ) - headers["Accept-Encoding"] = "identity" - return headers \ No newline at end of file diff --git a/devplacepy_services/base/pubsub_client.py b/devplacepy_services/base/pubsub_client.py deleted file mode 100644 index ce9abeda..00000000 --- a/devplacepy_services/base/pubsub_client.py +++ /dev/null @@ -1,13 +0,0 @@ -# retoor - -from __future__ import annotations - -from typing import Any - -from devplacepy_services.base.config import service_url -from devplacepy_services.base.http import internal_request - - -async def publish(topic: str, data: dict[str, Any]) -> None: - url = f"{service_url('pubsub')}/internal/publish" - await internal_request("POST", url, json={"topic": topic, "data": data}) \ No newline at end of file diff --git a/devplacepy_services/base/schemas.py b/devplacepy_services/base/schemas.py deleted file mode 100644 index 0f34f03d..00000000 --- a/devplacepy_services/base/schemas.py +++ /dev/null @@ -1,29 +0,0 @@ -# retoor - -from __future__ import annotations - -from pydantic import BaseModel, ConfigDict - - -class ErrorOut(BaseModel): - error: str - code: str - - -class HealthStatsOut(BaseModel): - requests: int - errors: int - - -class HealthOut(BaseModel): - service: str - status: str - uptime_s: int - version: str - deps: dict[str, str] - stats: HealthStatsOut - - -class InternalUser(BaseModel): - model_config = ConfigDict(frozen=True) - uid: str | None = None \ No newline at end of file diff --git a/devplacepy_services/base/service.py b/devplacepy_services/base/service.py deleted file mode 100644 index c5fdc00b..00000000 --- a/devplacepy_services/base/service.py +++ /dev/null @@ -1,134 +0,0 @@ -# retoor - -from __future__ import annotations - -import os -import subprocess -import time -from contextlib import asynccontextmanager -from typing import Literal - -from fastapi import APIRouter, FastAPI -from fastapi.exceptions import HTTPException - -from devplacepy_services.base.config import PROFILES, port_profile -from devplacepy_services.base.errors import error_response, sanitize_detail -from devplacepy_services.base.http import shutdown_pool, startup_pool -from devplacepy_services.base.middleware import ( - InternalAuthMiddleware, - InternalCallCounterMiddleware, - RequestStatsMiddleware, - SanitizeErrorsMiddleware, -) - - -class BaseMicroservice: - name: str = "" - title: str = "" - default_port: int = 0 - workers: int | Literal["auto"] = 1 - stateful: bool = True - depends_on: list[str] = [] - managed_services: list = [] - use_background: bool = False - run_supervisor: bool = False - - def __init__(self) -> None: - self.started_at = time.monotonic() - self.version = self._resolve_version() - - def _resolve_version(self) -> str: - env_version = os.environ.get("DEVPLACE_VERSION", "").strip() - if env_version: - return env_version - try: - result = subprocess.run( - ["git", "rev-parse", "--short", "HEAD"], - capture_output=True, - text=True, - timeout=2, - check=False, - ) - if result.returncode == 0: - return result.stdout.strip() or "unknown" - except (OSError, subprocess.TimeoutExpired): - pass - return "unknown" - - def resolved_port(self) -> int: - if self.default_port: - return self.default_port - profile = port_profile() - return PROFILES[profile][self.name] - - def apply_base_middleware(self, app: FastAPI, *, internal_auth: bool = False) -> None: - app.add_middleware(SanitizeErrorsMiddleware) - if internal_auth: - app.add_middleware(InternalAuthMiddleware) - app.add_middleware(InternalCallCounterMiddleware) - app.add_middleware(RequestStatsMiddleware) - - @asynccontextmanager - async def lifespan(self, app: FastAPI): - app.state.service_name = self.name - self.started_at = time.monotonic() - await startup_pool() - background = None - if self.use_background: - from devplacepy.services.background import background - - await background.start() - background = background - if self.managed_services: - from devplacepy.services.manager import service_manager - - for svc in self.managed_services: - service_manager.register(svc) - service_manager.set_lock_owner(True) - if self.run_supervisor: - import asyncio - - asyncio.get_running_loop().call_soon(service_manager.supervise) - yield - if self.managed_services: - from devplacepy.services.manager import service_manager - - await service_manager.shutdown_all() - if background is not None: - await background.stop() - await shutdown_pool() - - def create_app(self) -> FastAPI: - app = FastAPI(title=self.title, lifespan=self.lifespan) - app.state.service_name = self.name - - @app.exception_handler(HTTPException) - async def http_exception_handler(_request, exc: HTTPException): - if isinstance(exc.detail, dict) and "error" in exc.detail and "code" in exc.detail: - return error_response(exc.status_code, exc.detail["error"], exc.detail["code"]) - message, code = sanitize_detail(exc.detail, exc.status_code) - return error_response(exc.status_code, message, code) - - return app - - def build_app(self) -> FastAPI: - raise NotImplementedError - - -def build_standard_app( - service: BaseMicroservice, - *, - routers: list[tuple[APIRouter, str] | tuple[APIRouter]] | None = None, - internal_auth: bool = False, -) -> FastAPI: - from devplacepy_services.base.health import health_router - - app = service.create_app() - service.apply_base_middleware(app, internal_auth=internal_auth) - app.include_router(health_router(service)) - for entry in routers or []: - if len(entry) == 2: - app.include_router(entry[0], prefix=entry[1]) - else: - app.include_router(entry[0]) - return app \ No newline at end of file diff --git a/devplacepy_services/base/sqlite_broker.py b/devplacepy_services/base/sqlite_broker.py deleted file mode 100644 index c41030aa..00000000 --- a/devplacepy_services/base/sqlite_broker.py +++ /dev/null @@ -1,110 +0,0 @@ -# retoor - -from __future__ import annotations - -import asyncio -from collections.abc import Callable -from pathlib import Path -from typing import Any, TypeVar - -import dataset - -from devplacepy_services.base.config import sqlite_read_pool_size - -T = TypeVar("T") - -PRAGMAS = [ - "PRAGMA journal_mode=WAL", - "PRAGMA synchronous=NORMAL", - "PRAGMA busy_timeout=30000", - "PRAGMA cache_size=-8000", - "PRAGMA temp_store=MEMORY", - "PRAGMA mmap_size=268435456", -] - - -class SQLiteFileBroker: - name: str - path: Path - read_pool_size: int - - def __init__(self, name: str, path: Path, read_pool_size: int | None = None) -> None: - self.name = name - self.path = Path(path) - self.read_pool_size = read_pool_size or sqlite_read_pool_size() - self._write_db: Any = None - self._read_pool: asyncio.Queue[Any] | None = None - self._write_queue: asyncio.Queue[Any] | None = None - self._write_worker_task: asyncio.Task | None = None - - def _connect_rw(self): - return dataset.connect( - f"sqlite:///{self.path}", - engine_kwargs={ - "connect_args": { - "timeout": 30, - "check_same_thread": False, - }, - }, - on_connect_statements=PRAGMAS, - ) - - def _connect_ro(self): - abs_path = self.path.resolve().as_posix() - return dataset.connect( - f"sqlite:///file:{abs_path}?uri=true&mode=ro", - engine_kwargs={ - "connect_args": { - "timeout": 30, - "check_same_thread": False, - "uri": True, - }, - }, - on_connect_statements=PRAGMAS, - ) - - async def startup(self) -> None: - self.path.parent.mkdir(parents=True, exist_ok=True) - self._write_db = self._connect_rw() - self._read_pool = asyncio.Queue() - for _ in range(self.read_pool_size): - await self._read_pool.put(self._connect_ro()) - self._write_queue = asyncio.Queue() - self._write_worker_task = asyncio.create_task(self._write_worker()) - - async def shutdown(self) -> None: - if self._write_worker_task is not None: - self._write_worker_task.cancel() - try: - await self._write_worker_task - except asyncio.CancelledError: - pass - self._write_worker_task = None - - async def read(self, fn: Callable[..., T], *args, **kwargs) -> T: - assert self._read_pool is not None - db = await self._read_pool.get() - try: - return await asyncio.to_thread(fn, db, *args, **kwargs) - finally: - await self._read_pool.put(db) - - async def write(self, fn: Callable[..., T], *args, **kwargs) -> T: - assert self._write_queue is not None - loop = asyncio.get_running_loop() - future = loop.create_future() - await self._write_queue.put((fn, args, kwargs, future)) - return await future - - async def _write_worker(self) -> None: - assert self._write_queue is not None - while True: - fn, args, kwargs, future = await self._write_queue.get() - try: - result = fn(self._write_db, *args, **kwargs) - future.set_result(result) - except Exception as exc: - future.set_exception(exc) - - async def read_after_write(self, fn: Callable[..., T], *args, **kwargs) -> T: - return await asyncio.to_thread(fn, self._write_db, *args, **kwargs) \ No newline at end of file diff --git a/devplacepy_services/base/stats.py b/devplacepy_services/base/stats.py deleted file mode 100644 index f4020e0c..00000000 --- a/devplacepy_services/base/stats.py +++ /dev/null @@ -1,26 +0,0 @@ -# retoor - -from dataclasses import dataclass, field -import threading - - -@dataclass -class RequestCounters: - requests: int = 0 - errors: int = 0 - _lock: threading.Lock = field(default_factory=threading.Lock, repr=False) - - def record_request(self) -> None: - with self._lock: - self.requests += 1 - - def record_error(self) -> None: - with self._lock: - self.errors += 1 - - def snapshot(self) -> dict[str, int]: - with self._lock: - return {"requests": self.requests, "errors": self.errors} - - -COUNTERS = RequestCounters() \ No newline at end of file diff --git a/devplacepy_services/base/testing.py b/devplacepy_services/base/testing.py deleted file mode 100644 index 5b253d93..00000000 --- a/devplacepy_services/base/testing.py +++ /dev/null @@ -1,40 +0,0 @@ -# retoor - -from __future__ import annotations - -from typing import Any - -from fastapi import FastAPI -from starlette.testclient import TestClient - -from devplacepy_services.base.auth import set_internal_gateway_key - - -class ServiceTestClient: - def __init__( - self, - app: FastAPI, - *, - internal_key: str = "test-internal-key", - authenticated_user: str | None = None, - ) -> None: - set_internal_gateway_key(internal_key) - headers: dict[str, str] = {"X-Internal-Key": internal_key} - if authenticated_user: - headers["X-Authenticated-User"] = authenticated_user - self._client = TestClient(app, headers=headers) - - def get(self, path: str, **kwargs: Any): - return self._client.get(path, **kwargs) - - def post(self, path: str, **kwargs: Any): - return self._client.post(path, **kwargs) - - def put(self, path: str, **kwargs: Any): - return self._client.put(path, **kwargs) - - def delete(self, path: str, **kwargs: Any): - return self._client.delete(path, **kwargs) - - def close(self) -> None: - self._client.close() \ No newline at end of file diff --git a/devplacepy_services/bot/SERVICE.md b/devplacepy_services/bot/SERVICE.md deleted file mode 100644 index ceaf1e3a..00000000 --- a/devplacepy_services/bot/SERVICE.md +++ /dev/null @@ -1 +0,0 @@ -# Bot Service (stub) \ No newline at end of file diff --git a/devplacepy_services/bot/__init__.py b/devplacepy_services/bot/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/devplacepy_services/bot/main.py b/devplacepy_services/bot/main.py deleted file mode 100644 index 0f44e24f..00000000 --- a/devplacepy_services/bot/main.py +++ /dev/null @@ -1,23 +0,0 @@ -import devplacepy_services.base.bootstrap - -from devplacepy.services.bot.service import BotsService -from devplacepy_services.base.service import BaseMicroservice, build_standard_app - - -class BotService(BaseMicroservice): - name = "bot" - title = "Bot" - default_port = 10632 - workers = 1 - stateful = True - depends_on = ["database", "gateway"] - managed_services = [BotsService()] - use_background = True - run_supervisor = True - - def build_app(self): - return build_standard_app(self) - - -_service = BotService() -app = _service.build_app() \ No newline at end of file diff --git a/devplacepy_services/containers/SERVICE.md b/devplacepy_services/containers/SERVICE.md deleted file mode 100644 index cb64b7e1..00000000 --- a/devplacepy_services/containers/SERVICE.md +++ /dev/null @@ -1 +0,0 @@ -# Containers Service (stub) \ No newline at end of file diff --git a/devplacepy_services/containers/__init__.py b/devplacepy_services/containers/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/devplacepy_services/containers/main.py b/devplacepy_services/containers/main.py deleted file mode 100644 index 2749c85b..00000000 --- a/devplacepy_services/containers/main.py +++ /dev/null @@ -1,24 +0,0 @@ -import devplacepy_services.base.bootstrap - -from devplacepy.routers.projects import containers -from devplacepy.services.containers.service import ContainerService -from devplacepy_services.base.service import BaseMicroservice, build_standard_app - - -class ContainersService(BaseMicroservice): - name = "containers" - title = "Containers" - default_port = 10634 - workers = 1 - stateful = True - depends_on = ["database"] - managed_services = [ContainerService()] - use_background = True - run_supervisor = True - - def build_app(self): - return build_standard_app(self, routers=[(containers.router, "/projects")]) - - -_service = ContainersService() -app = _service.build_app() \ No newline at end of file diff --git a/devplacepy_services/database/SERVICE.md b/devplacepy_services/database/SERVICE.md deleted file mode 100644 index 48f53008..00000000 --- a/devplacepy_services/database/SERVICE.md +++ /dev/null @@ -1 +0,0 @@ -# Database Service (stub) \ No newline at end of file diff --git a/devplacepy_services/database/__init__.py b/devplacepy_services/database/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/devplacepy_services/database/broker_setup.py b/devplacepy_services/database/broker_setup.py deleted file mode 100644 index a88a83cb..00000000 --- a/devplacepy_services/database/broker_setup.py +++ /dev/null @@ -1,52 +0,0 @@ -# retoor - -from __future__ import annotations - -import os -from pathlib import Path - -from devplacepy.config import DATA_DIR, DATABASE_URL -from devplacepy_services.base.sqlite_broker import SQLiteFileBroker - - -def _db_path() -> Path: - # DEVPLACE_DATABASE_URL is the documented override (tests use a dedicated - # tempfile for isolation - see tests/conftest.py). DATABASE_URL already - # falls back to DATA_DIR / "devplace.db" when the env var is unset, so - # deriving the broker's path from it (instead of hardcoding DATA_DIR - # directly) keeps the broker and every other DEVPLACE_DATABASE_URL - # consumer (devplacepy/database/core.py) pointed at the same file. - if DATABASE_URL.startswith("sqlite:///"): - raw = DATABASE_URL[len("sqlite:///") :] - if raw and raw != ":memory:": - return Path(raw) - return DATA_DIR / "devplace.db" - - -_broker = SQLiteFileBroker("main", _db_path()) - - -def get_broker() -> SQLiteFileBroker: - return _broker - - -async def startup() -> None: - await _broker.startup() - from devplacepy_services.database.db_patch import patch_all_db, set_fallback_db - - patch_all_db() - set_fallback_db(_broker._write_db) - os.environ["DEVPLACE_DB_SERVICE"] = "1" - from devplacepy.database import init_db - - init_db() - from devplacepy.database.settings import internal_gateway_key - - key = (internal_gateway_key() or "").strip() - if key: - os.environ["DEVPLACE_GATEWAY_INTERNAL_KEY"] = key - (DATA_DIR / ".internal_key").write_text(key + "\n") - - -async def shutdown() -> None: - await _broker.shutdown() \ No newline at end of file diff --git a/devplacepy_services/database/compounds_page.py b/devplacepy_services/database/compounds_page.py deleted file mode 100644 index b420de35..00000000 --- a/devplacepy_services/database/compounds_page.py +++ /dev/null @@ -1,135 +0,0 @@ -# retoor - -from __future__ import annotations - -from devplacepy.database import ( - get_daily_topic, - get_polls_by_post_uids, - get_reactions_by_targets, - get_recent_comments_by_post_uids, - get_site_stats, - get_top_authors, - get_user_bookmarks, - get_users_by_uids, -) -from devplacepy.database.attachments_data import get_attachments_by_type -from devplacepy.routers.feed import get_feed_posts - - -def build_feed_page( - user: dict | None = None, - tab: str = "all", - topic: str | None = None, - search: str = "", - before: str | None = None, -) -> dict: - posts, next_cursor = get_feed_posts(user, tab, topic, search, before) - stats = get_site_stats() - top_authors = get_top_authors(5) - daily_topic = get_daily_topic() - post_uids = [item["post"]["uid"] for item in posts] - attachments_map = get_attachments_by_type("post", post_uids) - recent_comments = get_recent_comments_by_post_uids(post_uids, 3, user) - reactions_map = get_reactions_by_targets("post", post_uids, user) - bookmark_set = ( - get_user_bookmarks(user["uid"], "post", post_uids) if user else set() - ) - polls_map = get_polls_by_post_uids(post_uids, user) - for item in posts: - uid = item["post"]["uid"] - item["attachments"] = attachments_map.get(uid, []) - item["recent_comments"] = recent_comments.get(uid, []) - item["reactions"] = reactions_map.get(uid, {"counts": {}, "mine": []}) - item["bookmarked"] = uid in bookmark_set - item["poll"] = polls_map.get(uid) - return { - "posts": posts, - "next_cursor": next_cursor, - "stats": stats, - "top_authors": top_authors, - "daily_topic": daily_topic, - "online_users": [], - "current_tab": tab, - "current_topic": topic, - "search": search, - } - - -def build_post_detail(post_uid: str, user: dict | None = None) -> dict: - from devplacepy.database import get_table, load_comments - - post = get_table("posts").find_one(uid=post_uid, deleted_at=None) - if not post: - return {"post": None, "author": None, "comments": [], "attachments": []} - author = get_users_by_uids([post["user_uid"]]).get(post["user_uid"]) - comments = load_comments("post", post_uid, user) - attachments = get_attachments_by_type("post", [post_uid]).get(post_uid, []) - return { - "post": post, - "author": author, - "comments": comments, - "attachments": attachments, - } - - -def build_profile_bundle(profile_uid: str, viewer: dict | None = None) -> dict: - from devplacepy.database import get_table - from devplacepy.database.follows import get_follow_counts - - user = get_table("users").find_one(uid=profile_uid) - if not user: - return {"user": None, "follow_counts": {}, "awards": [], "posts_count": 0} - follow_counts = get_follow_counts(profile_uid) - return { - "user": user, - "follow_counts": follow_counts, - "awards": [], - "posts_count": 0, - "viewer_relation": {}, - } - - -def build_messages_page(user_uid: str) -> dict: - return {"threads": [], "unread": 0, "partners": {}} - - -def build_notifications_page(user_uid: str) -> dict: - return {"notifications": [], "actors": {}, "unread_count": 0} - - -def build_leaderboard_page(viewer_uid: str | None = None) -> dict: - from devplacepy.database import get_leaderboard, get_user_rank - - return { - "leaderboard": get_leaderboard(), - "viewer_rank": get_user_rank(viewer_uid) if viewer_uid else None, - } - - -def build_project_detail(project_uid: str) -> dict: - from devplacepy.database import get_table - - project = get_table("projects").find_one(uid=project_uid, deleted_at=None) - owner = None - if project: - owner = get_users_by_uids([project["user_uid"]]).get(project["user_uid"]) - return {"project": project, "owner": owner, "files": [], "containers": []} - - -def build_gist_detail(gist_uid: str, user: dict | None = None) -> dict: - from devplacepy.database import get_table, load_comments - - gist = get_table("gists").find_one(uid=gist_uid, deleted_at=None) - author = None - if gist: - author = get_users_by_uids([gist["user_uid"]]).get(gist["user_uid"]) - comments = load_comments("gist", gist_uid, user) if gist else [] - return {"gist": gist, "author": author, "comments": comments} - - -def build_game_state(user_uid: str | None = None) -> dict: - return {"store": {}, "leaderboard": []} - - -def build_admin_dashboard() -> dict: - return {"stats": get_site_stats(), "service_states": []} \ No newline at end of file diff --git a/devplacepy_services/database/compounds_primitive.py b/devplacepy_services/database/compounds_primitive.py deleted file mode 100644 index 570268ce..00000000 --- a/devplacepy_services/database/compounds_primitive.py +++ /dev/null @@ -1,138 +0,0 @@ -# retoor - -from __future__ import annotations - -from devplacepy.database import ( - get_attachments_by_type, - get_comment_counts_by_post_uids, - get_follow_counts, - get_following_among, - get_leaderboard, - get_muted_uids, - get_blocked_uids, - get_user_relations, - get_notification_prefs, - get_online_users, - get_polls_by_post_uids, - get_prominent_award, - get_reactions_by_targets, - get_recent_comments_by_post_uids, - get_seo_metadata_batch, - get_site_stats, - get_top_authors, - get_user_awards, - get_user_bookmarks, - get_user_media, - get_user_rank, - get_users_by_uids, - get_vote_counts, - load_comments, -) - - -def users_by_uids(uids: list[str]) -> dict: - return get_users_by_uids(uids) - - -def comment_counts(post_uids: list[str]) -> dict: - return get_comment_counts_by_post_uids(post_uids) - - -def vote_counts(target_uids: list[str]) -> dict: - ups, downs = get_vote_counts(target_uids) - return {"ups": ups, "downs": downs} - - -def reactions(target_type: str, target_uids: list[str], user: dict | None = None) -> dict: - return get_reactions_by_targets(target_type, target_uids, user) - - -def bookmarks(user_uid: str, target_type: str, target_uids: list[str]) -> list[str]: - result = get_user_bookmarks(user_uid, target_type, target_uids) - return sorted(result) - - -def polls(post_uids: list[str], user: dict | None = None) -> dict: - return get_polls_by_post_uids(post_uids, user) - - -def recent_comments( - post_uids: list[str], limit: int = 3, user: dict | None = None -) -> dict: - return get_recent_comments_by_post_uids(post_uids, limit, user) - - -def comments(target_type: str, target_uid: str, user: dict | None = None) -> list: - return load_comments(target_type, target_uid, user) - - -def follow_bundle(user_uid: str, target_uids: list[str] | None = None) -> dict: - payload = { - "counts": get_follow_counts(user_uid), - "following_among": sorted(get_following_among(user_uid, target_uids or [])), - } - return payload - - -def relations_bundle(viewer_uid: str | None) -> dict: - relations = get_user_relations(viewer_uid) - return { - "relations": { - "block": sorted(relations["block"]), - "mute": sorted(relations["mute"]), - }, - "blocked_uids": sorted(get_blocked_uids(viewer_uid)), - "muted_uids": sorted(get_muted_uids(viewer_uid)), - } - - -def online_users_bundle(cutoff_iso: str, limit: int = 30) -> list: - return get_online_users(cutoff_iso, limit) - - -def notification_prefs_bundle(user_uid: str) -> list: - return get_notification_prefs(user_uid) - - -def leaderboard_bundle( - limit: int = 50, offset: int = 0, viewer_uid: str | None = None -) -> dict: - return { - "leaderboard": get_leaderboard(limit, offset), - "viewer_rank": get_user_rank(viewer_uid) if viewer_uid else None, - } - - -def site_sidebar(authors_limit: int = 5) -> dict: - return { - "stats": get_site_stats(), - "top_authors": get_top_authors(authors_limit), - } - - -def awards_bundle( - receiver_uid: str, - page: int = 1, - per_page: int = 12, - profile_user: dict | None = None, -) -> dict: - items, pagination = get_user_awards(receiver_uid, page, per_page) - prominent = get_prominent_award(profile_user) if profile_user else None - return { - "items": items, - "pagination": pagination, - "prominent": prominent, - } - - -def attachments_batch(resource_type: str, resource_uids: list[str]) -> dict: - return get_attachments_by_type(resource_type, resource_uids) - - -def user_media_bundle(user_uid: str, page: int = 1, per_page: int = 24) -> dict: - items, pagination = get_user_media(user_uid, page, per_page) - return {"items": items, "pagination": pagination} - - -def seo_meta_batch(target_type: str, uids: list[str]) -> dict: - return get_seo_metadata_batch(target_type, uids) \ No newline at end of file diff --git a/devplacepy_services/database/db_patch.py b/devplacepy_services/database/db_patch.py deleted file mode 100644 index 337b97b2..00000000 --- a/devplacepy_services/database/db_patch.py +++ /dev/null @@ -1,119 +0,0 @@ -# retoor - -from __future__ import annotations - -import contextvars -import importlib -from typing import Any - -_DB_MODULES = ( - "schema", - "content", - "pagination", - "stats", - "forks", - "awards", - "customization", - "soft_delete", - "usage", - "email", - "follows", - "settings", - "seo_meta", - "deepsearch", - "activity", - "attachments_data", - "users", - "notifications", - "comments", - "ranking", - "engagement", - "relations", -) - -_EXTERNAL_DB_MODULES = ( - "devplacepy.services.audit.store", - "devplacepy.services.backup.store", - "devplacepy.services.openai_gateway.routing", - "devplacepy.services.openai_gateway.usage", - "devplacepy.services.openai_gateway.analytics", - "devplacepy.services.base", - "devplacepy.services.jobs.queue", - "devplacepy.services.containers.store", - "devplacepy.services.devii.store", - "devplacepy.services.devii.hub", - "devplacepy.attachments", - "devplacepy.project_files", -) - -# The read pool runs on asyncio.to_thread worker threads while the write -# lane runs on the main event loop task - a plain module-global "current -# connection" swapped in/out per call is a data race between them (a -# concurrent read could reassign the global to a read-only connection -# while a write is still mid-flight, raising "attempt to write a readonly -# database"). asyncio.to_thread propagates a COPY of the calling task's -# contextvars.Context into the new thread, so storing the active -# connection in a ContextVar gives each call its own isolated view with -# no cross-talk, while every db-layer module keeps a single shared proxy -# object as its permanent `db` binding. -_CURRENT_DB: contextvars.ContextVar[Any] = contextvars.ContextVar("current_db", default=None) - -# Fallback for any code path that touches the db outside an explicit -# use_db()-scoped call (e.g. startup's own init_db()) - write-once at -# broker startup, read-only afterward, so it carries no race of its own. -_fallback_db: Any = None - - -def _resolve() -> Any: - db = _CURRENT_DB.get() - if db is not None: - return db - if _fallback_db is not None: - return _fallback_db - raise RuntimeError("no database connection configured for this context") - - -class _ContextDb: - def __getattr__(self, name: str) -> Any: - return getattr(_resolve(), name) - - def __getitem__(self, name: str) -> Any: - return _resolve()[name] - - def __enter__(self): - return _resolve().__enter__() - - def __exit__(self, exc_type, exc, tb): - return _resolve().__exit__(exc_type, exc, tb) - - -_CONTEXT_DB = _ContextDb() - - -def patch_all_db() -> None: - import devplacepy.database as database_pkg - import devplacepy.database.core as core - - core.db = _CONTEXT_DB - database_pkg.db = _CONTEXT_DB - for name in _DB_MODULES: - mod = importlib.import_module(f"devplacepy.database.{name}") - if hasattr(mod, "db"): - mod.db = _CONTEXT_DB - for name in _EXTERNAL_DB_MODULES: - mod = importlib.import_module(name) - if hasattr(mod, "db"): - mod.db = _CONTEXT_DB - - -def set_fallback_db(db_conn) -> None: - global _fallback_db - _fallback_db = db_conn - - -def use_db(db_conn) -> contextvars.Token: - return _CURRENT_DB.set(db_conn) - - -def reset_db(token: contextvars.Token) -> None: - _CURRENT_DB.reset(token) diff --git a/devplacepy_services/database/invoke_registry.py b/devplacepy_services/database/invoke_registry.py deleted file mode 100644 index ea44c9af..00000000 --- a/devplacepy_services/database/invoke_registry.py +++ /dev/null @@ -1,40 +0,0 @@ -# retoor - -from __future__ import annotations - -import inspect -from typing import Any - -import devplacepy.database as db_module - -from devplacepy_services.base.db_codec import decode_arg, encode_result, is_write - - -def build_registry() -> dict[str, Any]: - registry: dict[str, Any] = {} - for name in db_module.__all__: - target = getattr(db_module, name, None) - if target is None or not callable(target): - continue - if inspect.isclass(target): - continue - registry[name] = target - return registry - - -REGISTRY = build_registry() - - -_encode_result = encode_result - - -def run_with_db(db_conn, fn, args, kwargs): - from devplacepy_services.database.db_patch import use_db, reset_db - - token = use_db(db_conn) - try: - decoded_args = decode_arg(args) - decoded_kwargs = decode_arg(kwargs) - return fn(*decoded_args, **decoded_kwargs) - finally: - reset_db(token) \ No newline at end of file diff --git a/devplacepy_services/database/main.py b/devplacepy_services/database/main.py deleted file mode 100644 index e85d6417..00000000 --- a/devplacepy_services/database/main.py +++ /dev/null @@ -1,52 +0,0 @@ -# retoor - -from __future__ import annotations - -from contextlib import asynccontextmanager - -from fastapi import FastAPI - -from devplacepy_services.base.health import health_router -from devplacepy_services.base.http import shutdown_pool, startup_pool -from devplacepy_services.base.middleware import ( - InternalAuthMiddleware, - InternalCallCounterMiddleware, - RequestStatsMiddleware, - SanitizeErrorsMiddleware, -) -from devplacepy_services.base.service import BaseMicroservice -from devplacepy_services.database import broker_setup -from devplacepy_services.database.routes import router as database_router - - -class DatabaseService(BaseMicroservice): - name = "database" - title = "Database" - default_port = 10601 - workers = 1 - stateful = True - depends_on = [] - - @asynccontextmanager - async def lifespan(self, app: FastAPI): - app.state.service_name = self.name - self.started_at = __import__("time").monotonic() - await startup_pool() - await broker_setup.startup() - yield - await broker_setup.shutdown() - await shutdown_pool() - - def build_app(self): - app = self.create_app() - app.add_middleware(SanitizeErrorsMiddleware) - app.add_middleware(InternalAuthMiddleware) - app.add_middleware(InternalCallCounterMiddleware) - app.add_middleware(RequestStatsMiddleware) - app.include_router(health_router(self)) - app.include_router(database_router) - return app - - -_service = DatabaseService() -app = _service.build_app() \ No newline at end of file diff --git a/devplacepy_services/database/routes.py b/devplacepy_services/database/routes.py deleted file mode 100644 index 3729811c..00000000 --- a/devplacepy_services/database/routes.py +++ /dev/null @@ -1,421 +0,0 @@ -# retoor - -from __future__ import annotations - -from typing import Any - -from fastapi import APIRouter, Depends -from pydantic import BaseModel, Field - -from devplacepy.database import bump_cache_version, get_int_setting, get_setting -from devplacepy_services.base.auth import internal_user -from devplacepy_services.base.compound import ( - AttachmentsIn, - AwardsBundleIn, - BookmarksIn, - CacheBumpIn, - CommentCountsIn, - CommentsIn, - FeedPageIn, - FollowBundleIn, - GameStateIn, - GistDetailIn, - InvokeIn, - LeaderboardBundleIn, - LeaderboardPageIn, - MessagesPageIn, - NotificationPrefsIn, - NotificationsPageIn, - OnlineUsersIn, - PollsIn, - PostDetailIn, - ProfileBundleIn, - ProjectDetailIn, - ReactionsIn, - RecentCommentsIn, - RelationsIn, - SeoMetaIn, - SiteSidebarIn, - UserMediaIn, - UsersByUidsIn, - VoteCountsIn, -) -from devplacepy_services.base.db_codec import is_write_sql -from devplacepy_services.base.schemas import InternalUser -from devplacepy_services.database import compounds_page, compounds_primitive -from devplacepy_services.database.broker_setup import get_broker -from devplacepy_services.database.invoke_registry import ( - REGISTRY, - _encode_result, - is_write, - run_with_db, -) - -router = APIRouter() - - -class DbOpIn(BaseModel): - op: str - table: str | None = None - method: str | None = None - args: list[Any] = Field(default_factory=list) - kwargs: dict[str, Any] = Field(default_factory=dict) - write: bool = False - - -def _db_tables(db_conn): - return list(db_conn.tables) - - -def _db_query(db_conn, sql: str, **params): - return list(db_conn.query(sql, **params)) - - -def _db_table_op(db_conn, table: str, method: str, args: list, kwargs: dict): - target = db_conn[table] - fn = getattr(target, method) - result = fn(*args, **kwargs) - if hasattr(result, "__iter__") and not isinstance(result, (str, bytes, dict)): - try: - return [_encode_result(row) for row in result] - except TypeError: - pass - return _encode_result(result) - - -@router.post("/internal/invoke") -async def internal_invoke(body: InvokeIn, _: InternalUser = Depends(internal_user)): - fn = REGISTRY.get(body.fn) - if fn is None: - return {"error": f"Unknown function: {body.fn}", "code": "unknown_function"} - write = body.write or is_write(body.fn) - broker = get_broker() - if write: - result = await broker.write(run_with_db, fn, body.args, body.kwargs) - else: - result = await broker.read(run_with_db, fn, body.args, body.kwargs) - return {"result": _encode_result(result)} - - -@router.post("/internal/db-op") -async def internal_db_op(body: DbOpIn, _: InternalUser = Depends(internal_user)): - broker = get_broker() - if body.op == "tables": - tables = await broker.read(_db_tables) - return tables - if body.op == "query": - sql = body.args[0] if body.args else "" - write = body.write or is_write_sql(sql) - if write: - rows = await broker.write(_db_query, sql, **body.kwargs) - else: - rows = await broker.read(_db_query, sql, **body.kwargs) - return [_encode_result(row) for row in rows] - if body.op == "table_op": - if not body.table or not body.method: - return {"error": "table and method required", "code": "invalid_request"} - write_methods = {"insert", "update", "delete", "create_column_by_example"} - write = body.write or body.method in write_methods - if write: - result = await broker.write( - _db_table_op, body.table, body.method, body.args, body.kwargs - ) - else: - result = await broker.read( - _db_table_op, body.table, body.method, body.args, body.kwargs - ) - return result - return {"error": f"Unknown op: {body.op}", "code": "unknown_op"} - - -@router.get("/settings/{key}") -async def settings_get(key: str, default: str = "", _: InternalUser = Depends(internal_user)): - broker = get_broker() - value = await broker.read(run_with_db, get_setting, [key, default], {}) - return {"key": key, "value": value} - - -@router.post("/cache/bump") -async def cache_bump(body: CacheBumpIn, _: InternalUser = Depends(internal_user)): - broker = get_broker() - await broker.write(run_with_db, bump_cache_version, [body.name], {}) - return {"ok": True, "name": body.name} - - -@router.post("/compound/users-by-uids") -async def compound_users_by_uids(body: UsersByUidsIn, _: InternalUser = Depends(internal_user)): - broker = get_broker() - return await broker.read(run_with_db, compounds_primitive.users_by_uids, [body.uids], {}) - - -@router.post("/compound/comment-counts") -async def compound_comment_counts(body: CommentCountsIn, _: InternalUser = Depends(internal_user)): - broker = get_broker() - return await broker.read( - run_with_db, compounds_primitive.comment_counts, [body.post_uids], {} - ) - - -@router.post("/compound/vote-counts") -async def compound_vote_counts(body: VoteCountsIn, _: InternalUser = Depends(internal_user)): - broker = get_broker() - return await broker.read( - run_with_db, compounds_primitive.vote_counts, [body.target_uids], {} - ) - - -@router.post("/compound/reactions") -async def compound_reactions(body: ReactionsIn, _: InternalUser = Depends(internal_user)): - broker = get_broker() - return await broker.read( - run_with_db, - compounds_primitive.reactions, - [body.target_type, body.target_uids, body.user], - {}, - ) - - -@router.post("/compound/bookmarks") -async def compound_bookmarks(body: BookmarksIn, _: InternalUser = Depends(internal_user)): - broker = get_broker() - return await broker.read( - run_with_db, - compounds_primitive.bookmarks, - [body.user_uid, body.target_type, body.target_uids], - {}, - ) - - -@router.post("/compound/polls") -async def compound_polls(body: PollsIn, _: InternalUser = Depends(internal_user)): - broker = get_broker() - return await broker.read( - run_with_db, compounds_primitive.polls, [body.post_uids, body.user], {} - ) - - -@router.post("/compound/recent-comments") -async def compound_recent_comments( - body: RecentCommentsIn, _: InternalUser = Depends(internal_user) -): - broker = get_broker() - return await broker.read( - run_with_db, - compounds_primitive.recent_comments, - [body.post_uids, body.limit, body.user], - {}, - ) - - -@router.post("/compound/comments") -async def compound_comments(body: CommentsIn, _: InternalUser = Depends(internal_user)): - broker = get_broker() - return await broker.read( - run_with_db, - compounds_primitive.comments, - [body.target_type, body.target_uid, body.user], - {}, - ) - - -@router.post("/compound/follow-bundle") -async def compound_follow_bundle(body: FollowBundleIn, _: InternalUser = Depends(internal_user)): - broker = get_broker() - return await broker.read( - run_with_db, - compounds_primitive.follow_bundle, - [body.user_uid, body.target_uids], - {}, - ) - - -@router.post("/compound/relations") -async def compound_relations(body: RelationsIn, _: InternalUser = Depends(internal_user)): - broker = get_broker() - return await broker.read( - run_with_db, compounds_primitive.relations_bundle, [body.viewer_uid], {} - ) - - -@router.post("/compound/online-users") -async def compound_online_users(body: OnlineUsersIn, _: InternalUser = Depends(internal_user)): - broker = get_broker() - return await broker.read( - run_with_db, - compounds_primitive.online_users_bundle, - [body.cutoff_iso, body.limit], - {}, - ) - - -@router.post("/compound/notification-prefs") -async def compound_notification_prefs( - body: NotificationPrefsIn, _: InternalUser = Depends(internal_user) -): - broker = get_broker() - return await broker.read( - run_with_db, - compounds_primitive.notification_prefs_bundle, - [body.user_uid], - {}, - ) - - -@router.post("/compound/leaderboard") -async def compound_leaderboard( - body: LeaderboardBundleIn, _: InternalUser = Depends(internal_user) -): - broker = get_broker() - return await broker.read( - run_with_db, - compounds_primitive.leaderboard_bundle, - [body.limit, body.offset, body.viewer_uid], - {}, - ) - - -@router.post("/compound/site-sidebar") -async def compound_site_sidebar(body: SiteSidebarIn, _: InternalUser = Depends(internal_user)): - broker = get_broker() - return await broker.read( - run_with_db, compounds_primitive.site_sidebar, [body.authors_limit], {} - ) - - -@router.post("/compound/awards") -async def compound_awards(body: AwardsBundleIn, _: InternalUser = Depends(internal_user)): - broker = get_broker() - return await broker.read( - run_with_db, - compounds_primitive.awards_bundle, - [body.receiver_uid, body.page, body.per_page, body.profile_user], - {}, - ) - - -@router.post("/compound/attachments") -async def compound_attachments(body: AttachmentsIn, _: InternalUser = Depends(internal_user)): - broker = get_broker() - return await broker.read( - run_with_db, - compounds_primitive.attachments_batch, - [body.resource_type, body.resource_uids], - {}, - ) - - -@router.post("/compound/user-media") -async def compound_user_media(body: UserMediaIn, _: InternalUser = Depends(internal_user)): - broker = get_broker() - return await broker.read( - run_with_db, - compounds_primitive.user_media_bundle, - [body.user_uid, body.page, body.per_page], - {}, - ) - - -@router.post("/compound/seo-meta") -async def compound_seo_meta(body: SeoMetaIn, _: InternalUser = Depends(internal_user)): - broker = get_broker() - return await broker.read( - run_with_db, - compounds_primitive.seo_meta_batch, - [body.target_type, body.uids], - {}, - ) - - -@router.post("/compound/feed-page") -async def compound_feed_page(body: FeedPageIn, _: InternalUser = Depends(internal_user)): - broker = get_broker() - return await broker.read( - run_with_db, compounds_page.build_feed_page, [], body.model_dump() - ) - - -@router.post("/compound/post-detail") -async def compound_post_detail(body: PostDetailIn, _: InternalUser = Depends(internal_user)): - broker = get_broker() - return await broker.read( - run_with_db, - compounds_page.build_post_detail, - [body.post_uid, body.user], - {}, - ) - - -@router.post("/compound/profile-bundle") -async def compound_profile_bundle( - body: ProfileBundleIn, _: InternalUser = Depends(internal_user) -): - broker = get_broker() - return await broker.read( - run_with_db, - compounds_page.build_profile_bundle, - [body.profile_uid, body.viewer], - {}, - ) - - -@router.post("/compound/messages-page") -async def compound_messages_page(body: MessagesPageIn, _: InternalUser = Depends(internal_user)): - broker = get_broker() - return await broker.read( - run_with_db, compounds_page.build_messages_page, [body.user_uid], {} - ) - - -@router.post("/compound/notifications-page") -async def compound_notifications_page( - body: NotificationsPageIn, _: InternalUser = Depends(internal_user) -): - broker = get_broker() - return await broker.read( - run_with_db, compounds_page.build_notifications_page, [body.user_uid], {} - ) - - -@router.post("/compound/leaderboard-page") -async def compound_leaderboard_page( - body: LeaderboardPageIn, _: InternalUser = Depends(internal_user) -): - broker = get_broker() - return await broker.read( - run_with_db, compounds_page.build_leaderboard_page, [body.viewer_uid], {} - ) - - -@router.post("/compound/project-detail") -async def compound_project_detail( - body: ProjectDetailIn, _: InternalUser = Depends(internal_user) -): - broker = get_broker() - return await broker.read( - run_with_db, compounds_page.build_project_detail, [body.project_uid], {} - ) - - -@router.post("/compound/gist-detail") -async def compound_gist_detail(body: GistDetailIn, _: InternalUser = Depends(internal_user)): - broker = get_broker() - return await broker.read( - run_with_db, - compounds_page.build_gist_detail, - [body.gist_uid, body.user], - {}, - ) - - -@router.post("/compound/game-state") -async def compound_game_state(body: GameStateIn, _: InternalUser = Depends(internal_user)): - broker = get_broker() - return await broker.read( - run_with_db, compounds_page.build_game_state, [body.user_uid], {} - ) - - -@router.post("/compound/admin-dashboard") -async def compound_admin_dashboard(_: InternalUser = Depends(internal_user)): - broker = get_broker() - return await broker.read(run_with_db, compounds_page.build_admin_dashboard, [], {}) \ No newline at end of file diff --git a/devplacepy_services/devii/SERVICE.md b/devplacepy_services/devii/SERVICE.md deleted file mode 100644 index b3780e96..00000000 --- a/devplacepy_services/devii/SERVICE.md +++ /dev/null @@ -1 +0,0 @@ -# Devii Service (stub) \ No newline at end of file diff --git a/devplacepy_services/devii/__init__.py b/devplacepy_services/devii/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/devplacepy_services/devii/main.py b/devplacepy_services/devii/main.py deleted file mode 100644 index a381ce41..00000000 --- a/devplacepy_services/devii/main.py +++ /dev/null @@ -1,40 +0,0 @@ -import devplacepy_services.base.bootstrap - -from contextlib import asynccontextmanager - -from fastapi import FastAPI - -from devplacepy.routers import devii -from devplacepy.services.devii import DeviiService -from devplacepy_services.base.service import BaseMicroservice, build_standard_app -from devplacepy_services.devii import store_brokers -from devplacepy_services.devii.store_routes import router as store_router - - -class DeviiMicroservice(BaseMicroservice): - name = "devii" - title = "Devii" - default_port = 10631 - workers = 2 - stateful = True - depends_on = ["database", "pubsub", "gateway"] - managed_services = [DeviiService()] - use_background = True - run_supervisor = True - - @asynccontextmanager - async def lifespan(self, app: FastAPI): - await store_brokers.startup() - async with super(DeviiMicroservice, self).lifespan(app): - yield - await store_brokers.shutdown() - - def build_app(self): - return build_standard_app( - self, - routers=[(store_router,), (devii.router, "/devii")], - ) - - -_service = DeviiMicroservice() -app = _service.build_app() \ No newline at end of file diff --git a/devplacepy_services/devii/store_brokers.py b/devplacepy_services/devii/store_brokers.py deleted file mode 100644 index 4dc36f6b..00000000 --- a/devplacepy_services/devii/store_brokers.py +++ /dev/null @@ -1,27 +0,0 @@ -# retoor - -from __future__ import annotations - -from devplacepy.config import DEVII_LESSONS_DB, DEVII_TASKS_DB -from devplacepy_services.base.sqlite_broker import SQLiteFileBroker - -_tasks = SQLiteFileBroker("devii_tasks", DEVII_TASKS_DB) -_lessons = SQLiteFileBroker("devii_lessons", DEVII_LESSONS_DB) - - -def broker_for(name: str) -> SQLiteFileBroker: - if name == "devii_tasks": - return _tasks - if name == "devii_lessons": - return _lessons - raise KeyError(name) - - -async def startup() -> None: - await _tasks.startup() - await _lessons.startup() - - -async def shutdown() -> None: - await _tasks.shutdown() - await _lessons.shutdown() \ No newline at end of file diff --git a/devplacepy_services/devii/store_routes.py b/devplacepy_services/devii/store_routes.py deleted file mode 100644 index 8efa0a11..00000000 --- a/devplacepy_services/devii/store_routes.py +++ /dev/null @@ -1,63 +0,0 @@ -# retoor - -from __future__ import annotations - -from typing import Any - -from fastapi import APIRouter, Depends -from pydantic import BaseModel, Field - -from devplacepy_services.base.auth import internal_user -from devplacepy_services.base.schemas import InternalUser -from devplacepy_services.database.invoke_registry import _encode_result -from devplacepy_services.database.routes import DbOpIn, _db_query, _db_table_op, _db_tables -from devplacepy_services.devii.store_brokers import broker_for - -router = APIRouter() - - -class StoreDbOpIn(BaseModel): - op: str - table: str | None = None - method: str | None = None - args: list[Any] = Field(default_factory=list) - kwargs: dict[str, Any] = Field(default_factory=dict) - write: bool = False - - -def _make_handler(store_name: str): - async def handler(body: StoreDbOpIn, _: InternalUser = Depends(internal_user)): - broker = broker_for(store_name) - if body.op == "tables": - return await broker.read(_db_tables) - if body.op == "query": - sql = body.args[0] if body.args else "" - rows = await broker.read(_db_query, sql, **body.kwargs) - return [_encode_result(row) for row in rows] - if body.op == "table_op": - if not body.table or not body.method: - return {"error": "table and method required", "code": "invalid_request"} - write_methods = {"insert", "update", "delete", "create_column_by_example"} - write = body.write or body.method in write_methods - if write: - return await broker.write( - _db_table_op, body.table, body.method, body.args, body.kwargs - ) - return await broker.read( - _db_table_op, body.table, body.method, body.args, body.kwargs - ) - return {"error": f"Unknown op: {body.op}", "code": "unknown_op"} - - return handler - - -router.add_api_route( - "/internal/store/devii_tasks/db-op", - _make_handler("devii_tasks"), - methods=["POST"], -) -router.add_api_route( - "/internal/store/devii_lessons/db-op", - _make_handler("devii_lessons"), - methods=["POST"], -) \ No newline at end of file diff --git a/devplacepy_services/email/SERVICE.md b/devplacepy_services/email/SERVICE.md deleted file mode 100644 index 78c586a4..00000000 --- a/devplacepy_services/email/SERVICE.md +++ /dev/null @@ -1 +0,0 @@ -# Email Service (stub) \ No newline at end of file diff --git a/devplacepy_services/email/__init__.py b/devplacepy_services/email/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/devplacepy_services/email/main.py b/devplacepy_services/email/main.py deleted file mode 100644 index a311af82..00000000 --- a/devplacepy_services/email/main.py +++ /dev/null @@ -1,20 +0,0 @@ -import devplacepy_services.base.bootstrap - -from devplacepy_services.base.service import BaseMicroservice, build_standard_app -from devplacepy_services.email.routes import router - - -class EmailService(BaseMicroservice): - name = "email" - title = "Email" - default_port = 10636 - workers = 1 - stateful = True - depends_on = ["database"] - - def build_app(self): - return build_standard_app(self, routers=[(router,)]) - - -_service = EmailService() -app = _service.build_app() \ No newline at end of file diff --git a/devplacepy_services/email/routes.py b/devplacepy_services/email/routes.py deleted file mode 100644 index ca92024a..00000000 --- a/devplacepy_services/email/routes.py +++ /dev/null @@ -1,9 +0,0 @@ -from fastapi import APIRouter -from fastapi.responses import JSONResponse - -router = APIRouter() - - -@router.get("/status") -async def email_status(): - return JSONResponse({"service": "email", "status": "stub"}) \ No newline at end of file diff --git a/devplacepy_services/gateway/SERVICE.md b/devplacepy_services/gateway/SERVICE.md deleted file mode 100644 index 828bf831..00000000 --- a/devplacepy_services/gateway/SERVICE.md +++ /dev/null @@ -1 +0,0 @@ -# Gateway Service (stub) \ No newline at end of file diff --git a/devplacepy_services/gateway/__init__.py b/devplacepy_services/gateway/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/devplacepy_services/gateway/main.py b/devplacepy_services/gateway/main.py deleted file mode 100644 index 40bfaac3..00000000 --- a/devplacepy_services/gateway/main.py +++ /dev/null @@ -1,24 +0,0 @@ -import devplacepy_services.base.bootstrap - -from devplacepy.routers import openai_gateway -from devplacepy.services.openai_gateway import GatewayService as OpenAIGatewayService -from devplacepy_services.base.service import BaseMicroservice, build_standard_app - - -class GatewayService(BaseMicroservice): - name = "gateway" - title = "Gateway" - default_port = 10620 - workers = "auto" - stateful = False - depends_on = ["database"] - managed_services = [OpenAIGatewayService()] - use_background = True - run_supervisor = True - - def build_app(self): - return build_standard_app(self, routers=[(openai_gateway.router, "/openai")]) - - -_service = GatewayService() -app = _service.build_app() \ No newline at end of file diff --git a/devplacepy_services/gitea/SERVICE.md b/devplacepy_services/gitea/SERVICE.md deleted file mode 100644 index e44045a4..00000000 --- a/devplacepy_services/gitea/SERVICE.md +++ /dev/null @@ -1 +0,0 @@ -# Gitea Service (stub) \ No newline at end of file diff --git a/devplacepy_services/gitea/__init__.py b/devplacepy_services/gitea/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/devplacepy_services/gitea/main.py b/devplacepy_services/gitea/main.py deleted file mode 100644 index 7d5647b9..00000000 --- a/devplacepy_services/gitea/main.py +++ /dev/null @@ -1,23 +0,0 @@ -import devplacepy_services.base.bootstrap - -from devplacepy.services.gitea.service import IssueTrackerService -from devplacepy_services.base.service import BaseMicroservice, build_standard_app - - -class GiteaService(BaseMicroservice): - name = "gitea" - title = "Gitea" - default_port = 10638 - workers = 1 - stateful = True - depends_on = ["database", "gateway"] - managed_services = [IssueTrackerService()] - use_background = True - run_supervisor = True - - def build_app(self): - return build_standard_app(self) - - -_service = GiteaService() -app = _service.build_app() \ No newline at end of file diff --git a/devplacepy_services/jobs/SERVICE.md b/devplacepy_services/jobs/SERVICE.md deleted file mode 100644 index ff34829e..00000000 --- a/devplacepy_services/jobs/SERVICE.md +++ /dev/null @@ -1 +0,0 @@ -# Jobs Service (stub) \ No newline at end of file diff --git a/devplacepy_services/jobs/__init__.py b/devplacepy_services/jobs/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/devplacepy_services/jobs/main.py b/devplacepy_services/jobs/main.py deleted file mode 100644 index 44738559..00000000 --- a/devplacepy_services/jobs/main.py +++ /dev/null @@ -1,49 +0,0 @@ -import devplacepy_services.base.bootstrap - -from devplacepy.routers import forks, tools, zips -from devplacepy.services.jobs.award_service import AwardService -from devplacepy.services.jobs.deepsearch.service import DeepsearchService -from devplacepy.services.jobs.fork_service import ForkService -from devplacepy.services.jobs.issue_create_service import IssueCreateService -from devplacepy.services.jobs.isslop.service import IsslopService -from devplacepy.services.jobs.planning_service import PlanningReportService -from devplacepy.services.jobs.seo.service import SeoService -from devplacepy.services.jobs.seo_meta_service import SeoMetaService -from devplacepy.services.jobs.zip_service import ZipService -from devplacepy_services.base.service import BaseMicroservice, build_standard_app - - -class JobsService(BaseMicroservice): - name = "jobs" - title = "Jobs" - default_port = 10630 - workers = 1 - stateful = True - depends_on = ["database", "pubsub", "gateway"] - managed_services = [ - ZipService(), - ForkService(), - SeoService(), - SeoMetaService(), - AwardService(), - DeepsearchService(), - IsslopService(), - IssueCreateService(), - PlanningReportService(), - ] - use_background = True - run_supervisor = True - - def build_app(self): - return build_standard_app( - self, - routers=[ - (zips.router, "/zips"), - (forks.router, "/forks"), - (tools.router, "/tools"), - ], - ) - - -_service = JobsService() -app = _service.build_app() \ No newline at end of file diff --git a/devplacepy_services/news/SERVICE.md b/devplacepy_services/news/SERVICE.md deleted file mode 100644 index ef5f758b..00000000 --- a/devplacepy_services/news/SERVICE.md +++ /dev/null @@ -1 +0,0 @@ -# News Service (stub) \ No newline at end of file diff --git a/devplacepy_services/news/__init__.py b/devplacepy_services/news/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/devplacepy_services/news/main.py b/devplacepy_services/news/main.py deleted file mode 100644 index 343eb883..00000000 --- a/devplacepy_services/news/main.py +++ /dev/null @@ -1,23 +0,0 @@ -import devplacepy_services.base.bootstrap - -from devplacepy.services.news import NewsService -from devplacepy_services.base.service import BaseMicroservice, build_standard_app - - -class NewsMicroservice(BaseMicroservice): - name = "news" - title = "News" - default_port = 10637 - workers = 1 - stateful = True - depends_on = ["database", "gateway"] - managed_services = [NewsService()] - use_background = True - run_supervisor = True - - def build_app(self): - return build_standard_app(self) - - -_service = NewsMicroservice() -app = _service.build_app() \ No newline at end of file diff --git a/devplacepy_services/pubsub/SERVICE.md b/devplacepy_services/pubsub/SERVICE.md deleted file mode 100644 index 9aac94d2..00000000 --- a/devplacepy_services/pubsub/SERVICE.md +++ /dev/null @@ -1 +0,0 @@ -# PubSub Service (stub) \ No newline at end of file diff --git a/devplacepy_services/pubsub/__init__.py b/devplacepy_services/pubsub/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/devplacepy_services/pubsub/main.py b/devplacepy_services/pubsub/main.py deleted file mode 100644 index e54407a2..00000000 --- a/devplacepy_services/pubsub/main.py +++ /dev/null @@ -1,31 +0,0 @@ -import devplacepy_services.base.bootstrap - -from devplacepy.routers import pubsub -from devplacepy.services.live_view_relay import LiveViewRelayService -from devplacepy.services.notification_relay import NotificationRelayService -from devplacepy.services.presence_relay import PresenceRelayService -from devplacepy.services.pubsub import PubSubService -from devplacepy_services.base.service import BaseMicroservice, build_standard_app - - -class PubsubService(BaseMicroservice): - name = "pubsub" - title = "PubSub" - default_port = 10602 - workers = 1 - stateful = True - depends_on = ["database"] - managed_services = [ - PubSubService(), - PresenceRelayService(), - NotificationRelayService(), - LiveViewRelayService(), - ] - run_supervisor = True - - def build_app(self): - return build_standard_app(self, routers=[(pubsub.router, "/pubsub")]) - - -_service = PubsubService() -app = _service.build_app() \ No newline at end of file diff --git a/devplacepy_services/telegram/SERVICE.md b/devplacepy_services/telegram/SERVICE.md deleted file mode 100644 index 511cd06d..00000000 --- a/devplacepy_services/telegram/SERVICE.md +++ /dev/null @@ -1 +0,0 @@ -# Telegram Service (stub) \ No newline at end of file diff --git a/devplacepy_services/telegram/__init__.py b/devplacepy_services/telegram/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/devplacepy_services/telegram/main.py b/devplacepy_services/telegram/main.py deleted file mode 100644 index d46c6681..00000000 --- a/devplacepy_services/telegram/main.py +++ /dev/null @@ -1,24 +0,0 @@ -import devplacepy_services.base.bootstrap - -from devplacepy.services.telegram.outbox_service import TelegramOutboxService -from devplacepy.services.telegram.service import TelegramService -from devplacepy_services.base.service import BaseMicroservice, build_standard_app - - -class TelegramMicroservice(BaseMicroservice): - name = "telegram" - title = "Telegram" - default_port = 10635 - workers = 1 - stateful = True - depends_on = ["database", "pubsub"] - managed_services = [TelegramService(), TelegramOutboxService()] - use_background = True - run_supervisor = True - - def build_app(self): - return build_standard_app(self) - - -_service = TelegramMicroservice() -app = _service.build_app() \ No newline at end of file diff --git a/devplacepy_services/web/SERVICE.md b/devplacepy_services/web/SERVICE.md deleted file mode 100644 index 6381d08a..00000000 --- a/devplacepy_services/web/SERVICE.md +++ /dev/null @@ -1 +0,0 @@ -# Web Service (stub) \ No newline at end of file diff --git a/devplacepy_services/web/__init__.py b/devplacepy_services/web/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/devplacepy_services/web/factory.py b/devplacepy_services/web/factory.py deleted file mode 100644 index f1d416c1..00000000 --- a/devplacepy_services/web/factory.py +++ /dev/null @@ -1,616 +0,0 @@ -import os - -import devplacepy.db_client - -import asyncio -import logging -import time -from collections import defaultdict -from contextlib import asynccontextmanager -from pathlib import Path - -from fastapi import FastAPI, Request -from fastapi.exceptions import RequestValidationError -from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse -from fastapi.staticfiles import StaticFiles -from starlette.middleware.gzip import GZipMiddleware - -from devplacepy.cache import TTLCache -from devplacepy.config import ( - PORT, - STATIC_DIR, - STATIC_VERSION, - UPLOADS_DIR, - ensure_data_dirs, -) -from devplacepy.db_client import ( - db, - get_blocked_uids, - get_comment_counts_by_post_uids, - get_int_setting, - get_news_images_by_uids, - get_setting, - get_table, - get_user_post_count, - get_user_stars, - get_users_by_uids, - get_vote_counts, - interleave_by_author, -) -from devplacepy.responses import json_error, respond, wants_json -from devplacepy.routers import ( - admin, - auth, - avatar, - awards, - bookmarks, - comments, - dbapi, - devrant, - docs, - feed, - follow, - game, - gists, - issues, - leaderboard, - media, - messages, - news, - notifications, - polls, - posts, - profile, - projects, - proxy, - push, - reactions, - relations, - seo, - uploads, - votes, -) -from devplacepy.schemas import LandingOut, ValidationErrorOut -from devplacepy.seo import base_seo_context, site_url, website_schema -from devplacepy.services import presence -from devplacepy.services.audit import record as audit -from devplacepy.services.correction import PENDING_SCOPE_KEY -from devplacepy.templating import templates -from devplacepy.utils import client_ip, get_current_user, safe_next, time_ago -from devplacepy_services.web.ingress import mount_ingress - -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", -) -logger = logging.getLogger(__name__) - -_rate_limit_store = defaultdict(list) -RATE_LIMIT = int(os.environ.get("DEVPLACE_RATE_LIMIT", "60")) -RATE_WINDOW = 60 -WEB_WORKERS = max(1, int(os.environ.get("DEVPLACE_WEB_WORKERS", "1"))) -RATE_LIMIT_DISABLED = os.environ.get("DEVPLACE_DISABLE_RATE_LIMIT") == "1" - -HOT_SETTINGS_TTL = 2.0 -_hot_settings_value: dict = {} -_hot_settings_at = 0.0 -_last_rate_sweep = 0.0 -RATE_SWEEP_INTERVAL = 60.0 - - -def _hot_settings() -> dict: - global _hot_settings_value, _hot_settings_at - now = time.monotonic() - if not _hot_settings_value or now - _hot_settings_at >= HOT_SETTINGS_TTL: - _hot_settings_value = { - "maintenance_mode": get_setting("maintenance_mode", "0"), - "rate_limit_per_minute": max( - 1, get_int_setting("rate_limit_per_minute", RATE_LIMIT) - ), - "rate_limit_window_seconds": max( - 1, get_int_setting("rate_limit_window_seconds", RATE_WINDOW) - ), - } - _hot_settings_at = now - return _hot_settings_value - - -def _sweep_rate_limit_store(window_start: float) -> None: - stale = [ - ip - for ip, timestamps in _rate_limit_store.items() - if not timestamps or timestamps[-1] <= window_start - ] - for ip in stale: - del _rate_limit_store[ip] - - -def _worker_rate_limit(limit: int) -> int: - return max(1, -(-limit // WEB_WORKERS)) - - -INLINE_MEDIA_EXTENSIONS = { - ".jpg", - ".jpeg", - ".png", - ".gif", - ".webp", - ".bmp", - ".tiff", - ".mp4", - ".webm", - ".ogv", - ".mov", - ".m4v", - ".mp3", -} - - -class UploadStaticFiles(StaticFiles): - async def get_response(self, path, scope): - response = await super().get_response(path, scope) - disposition = ( - "inline" - if Path(path).suffix.lower() in INLINE_MEDIA_EXTENSIONS - else "attachment" - ) - response.headers["Content-Disposition"] = disposition - response.headers["Cache-Control"] = "public, max-age=604800" - return response - - -class CachedStaticFiles(StaticFiles): - async def get_response(self, path, scope): - response = await super().get_response(path, scope) - if Path(path).name == "service-worker.js": - response.headers["Cache-Control"] = "no-cache" - else: - response.headers["Cache-Control"] = "public, max-age=31536000, immutable" - return response - - -class FallbackStaticFiles(StaticFiles): - async def get_response(self, path, scope): - response = await super().get_response(path, scope) - if Path(path).name == "service-worker.js": - response.headers["Cache-Control"] = "no-cache" - else: - response.headers["Cache-Control"] = "public, max-age=3600" - return response - - -@asynccontextmanager -async def lifespan(app: FastAPI): - ensure_data_dirs() - from devplacepy.services.statistics.tracking import start_visit_flusher - - start_visit_flusher() - - from devplacepy.config import SERVICE_LOCK_FILE - from devplacepy.services.dbapi.service import DbApiJobService - from devplacepy.services.manager import service_manager - from devplacepy.services.weblock import acquire_web_lock - - web_lock_owner = acquire_web_lock(SERVICE_LOCK_FILE) - if web_lock_owner: - service_manager.register(DbApiJobService()) - service_manager.set_lock_owner(True) - service_manager.supervise() - - logger.info(f"DevPlace web started on port {PORT}") - yield - logger.info("Shutting down web...") - if web_lock_owner: - await service_manager.shutdown_all() - from devplacepy.services.statistics.tracking import flush_visits - - flush_visits() - - -_AUTH_FORM_PAGES = { - "/auth/signup": ("signup.html", "Join DevPlace"), - "/auth/login": ("login.html", "Sign In"), - "/auth/forgot-password": ("forgot_password.html", "Reset Password"), -} - -_FRIENDLY_ERRORS = { - ("username", "too_short"): "Username must be between 3 and 32 characters", - ("username", "too_long"): "Username must be between 3 and 32 characters", - ("password", "too_short"): "Password must be at least 6 characters", -} - - -def _friendly_error(err): - field = err["loc"][-1] if err.get("loc") else "" - key = (field, err.get("type", "").replace("string_", "")) - if key in _FRIENDLY_ERRORS: - return _FRIENDLY_ERRORS[key] - msg = err.get("msg", "Invalid input") - prefix = "Value error, " - return msg[len(prefix) :] if msg.startswith(prefix) else msg - - -_home_cache = TTLCache(ttl=int(os.environ.get("DEVPLACE_HOME_CACHE_TTL", "60")), max_size=4) - - -def _landing_news(): - cached = _home_cache.get("news") - if cached is not None: - return cached - articles = [] - if "news" in db.tables: - news_table = get_table("news") - raw = list( - news_table.find(show_on_landing=1, order_by=["-synced_at"], _limit=6) - ) - images_by_news = get_news_images_by_uids([a["uid"] for a in raw]) - for a in raw: - articles.append( - { - "uid": a["uid"], - "slug": a.get("slug", ""), - "title": a.get("title", ""), - "description": (a.get("description", "") or "")[:250], - "url": a.get("url", ""), - "source_name": a.get("source_name", ""), - "grade": a.get("grade", 0), - "featured": a.get("featured", 0), - "synced_at": a.get("synced_at", "") or "", - "time_ago": time_ago(a["synced_at"]) if a.get("synced_at") else "", - "image_url": a.get("image_url", "") or images_by_news.get(a["uid"], ""), - } - ) - _home_cache.set("news", articles) - return articles - - -def _landing_recent_posts(blocked): - if not blocked: - cached = _home_cache.get("posts") - if cached is not None: - return cached - posts = [] - if "posts" in db.tables: - posts_table = get_table("posts") - fetch_limit = 24 if blocked else 6 - raw_posts = list( - posts_table.find(deleted_at=None, order_by=["-created_at"], _limit=fetch_limit) - ) - if blocked: - raw_posts = [p for p in raw_posts if p["user_uid"] not in blocked] - raw_posts = raw_posts[:6] - raw_posts = interleave_by_author(raw_posts) - if raw_posts: - post_uids = [p["uid"] for p in raw_posts] - author_uids = [p["user_uid"] for p in raw_posts] - authors = get_users_by_uids(author_uids) - comment_counts = get_comment_counts_by_post_uids(post_uids) - upvotes, downvotes = get_vote_counts(post_uids) - for p in raw_posts: - posts.append( - { - "post": p, - "author": authors.get(p["user_uid"]), - "time_ago": time_ago(p["created_at"]), - "comment_count": comment_counts.get(p["uid"], 0), - "stars": upvotes.get(p["uid"], 0) - downvotes.get(p["uid"], 0), - "slug": p.get("slug", "") or p["uid"], - } - ) - if not blocked: - _home_cache.set("posts", posts) - return posts - - -def create_web_app() -> FastAPI: - from devplacepy_services.base.middleware import ( - InternalCallCounterMiddleware, - RequestStatsMiddleware, - SanitizeErrorsMiddleware, - ) - - app = FastAPI( - title="DevPlace", - docs_url="/swagger", - redoc_url=None, - openapi_url="/openapi.json", - lifespan=lifespan, - ) - app.add_middleware(SanitizeErrorsMiddleware) - app.add_middleware(InternalCallCounterMiddleware) - app.add_middleware(RequestStatsMiddleware) - app.mount( - "/static/uploads", - UploadStaticFiles(directory=str(UPLOADS_DIR), check_dir=False), - name="uploads", - ) - app.mount( - f"/static/v{STATIC_VERSION}", - CachedStaticFiles(directory=str(STATIC_DIR)), - name="static_versioned", - ) - app.mount("/static", FallbackStaticFiles(directory=str(STATIC_DIR)), name="static") - - @app.exception_handler(404) - async def not_found(request: Request, exc): - if wants_json(request): - return json_error(404, "Not found") - seo_ctx = base_seo_context( - request, - title="Not Found - DevPlace", - description="The page you requested does not exist.", - robots="noindex", - ) - return templates.TemplateResponse( - request, - "error.html", - { - **seo_ctx, - "request": request, - "error_code": 404, - "error_message": "Page not found", - }, - status_code=404, - ) - - @app.exception_handler(500) - async def server_error(request: Request, exc): - logger.exception("500 error on %s %s", request.method, request.url.path) - if wants_json(request): - return json_error(500, "Internal server error") - seo_ctx = base_seo_context( - request, - title="Server Error - DevPlace", - description="Something went wrong.", - robots="noindex", - ) - return templates.TemplateResponse( - request, - "error.html", - { - **seo_ctx, - "request": request, - "error_code": 500, - "error_message": "Internal server error", - }, - status_code=500, - ) - - @app.exception_handler(RequestValidationError) - async def on_validation_error(request: Request, exc: RequestValidationError): - errors = [_friendly_error(e) for e in exc.errors()] - if wants_json(request): - fields: dict = {} - for raw, friendly in zip(exc.errors(), errors): - name = raw["loc"][-1] if raw.get("loc") else "_" - fields.setdefault(str(name), []).append(friendly) - return JSONResponse( - ValidationErrorOut(fields=fields, messages=errors).model_dump(mode="json"), - status_code=422, - ) - path = request.url.path - page = _AUTH_FORM_PAGES.get(path) - if page is None and path.startswith("/auth/reset-password/"): - page = ("reset_password.html", "Set New Password") - if page: - template_name, title = page - context = { - **base_seo_context(request, title=title, robots="noindex,nofollow"), - "request": request, - "errors": errors, - } - try: - form = await request.form() - context.update({k: v for k, v in form.items() if isinstance(v, str)}) - except Exception: - pass - if "token" in request.path_params: - context["token"] = request.path_params["token"] - return templates.TemplateResponse( - request, template_name, context, status_code=400 - ) - referer = safe_next(request.headers.get("referer"), "/feed") - return RedirectResponse(url=referer, status_code=303) - - mount_ingress(app) - - app.include_router(auth.router, prefix="/auth") - app.include_router(feed.router, prefix="/feed") - app.include_router(posts.router, prefix="/posts") - app.include_router(comments.router, prefix="/comments") - app.include_router(projects.router, prefix="/projects") - app.include_router(profile.router, prefix="/profile") - app.include_router(messages.router, prefix="/messages") - app.include_router(notifications.router, prefix="/notifications") - app.include_router(votes.router, prefix="/votes") - app.include_router(reactions.router, prefix="/reactions") - app.include_router(bookmarks.router, prefix="/bookmarks") - app.include_router(polls.router, prefix="/polls") - app.include_router(avatar.router, prefix="/avatar") - app.include_router(awards.router, prefix="/awards") - app.include_router(follow.router, prefix="/follow") - app.include_router(relations.router) - app.include_router(leaderboard.router, prefix="/leaderboard") - app.include_router(admin.router, prefix="/admin") - app.include_router(seo.router) - app.include_router(push.router) - app.include_router(docs.router) - app.include_router(issues.router, prefix="/issues") - app.include_router(gists.router, prefix="/gists") - app.include_router(news.router, prefix="/news") - app.include_router(uploads.router, prefix="/uploads") - app.include_router(media.router, prefix="/media") - app.include_router(proxy.router, prefix="/p") - app.include_router(devrant.router, prefix="/api") - app.include_router(dbapi.router, prefix="/dbapi") - app.include_router(game.router, prefix="/game") - - @app.middleware("http") - async def await_pending_corrections(request: Request, call_next): - response = await call_next(request) - pending = request.scope.get(PENDING_SCOPE_KEY) - if pending: - await asyncio.gather(*pending, return_exceptions=True) - return response - - @app.middleware("http") - async def add_security_headers(request: Request, call_next): - response = await call_next(request) - if not response.headers.get("X-Robots-Tag"): - response.headers["X-Robots-Tag"] = "index, follow" - response.headers["X-Content-Type-Options"] = "nosniff" - response.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains" - response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin" - if not request.url.path.startswith("/p/"): - response.headers["X-Frame-Options"] = "DENY" - response.headers["Content-Security-Policy"] = ( - "object-src 'none'; base-uri 'self'; " - "frame-ancestors 'none'; form-action 'self'" - ) - if request.url.path.startswith("/admin"): - response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate, max-age=0" - response.headers["Pragma"] = "no-cache" - response.headers["Expires"] = "0" - return response - - @app.middleware("http") - async def rate_limit_middleware(request: Request, call_next): - if RATE_LIMIT_DISABLED: - return await call_next(request) - if request.method in ( - "POST", - "PUT", - "DELETE", - "PATCH", - ) and not request.url.path.startswith(("/openai", "/xmlrpc")): - settings = _hot_settings() - limit = _worker_rate_limit(settings["rate_limit_per_minute"]) - window = settings["rate_limit_window_seconds"] - ip = client_ip(request, default="unknown") - now = time.time() - window_start = now - window - global _last_rate_sweep - if now - _last_rate_sweep >= RATE_SWEEP_INTERVAL: - _sweep_rate_limit_store(window_start) - _last_rate_sweep = now - timestamps = [t for t in _rate_limit_store.get(ip, ()) if t > window_start] - if len(timestamps) >= limit: - _rate_limit_store[ip] = timestamps - audit.record( - request, - "security.rate_limit.block", - result="denied", - summary=f"request from {ip} blocked by rate limit", - metadata={"ip": ip, "limit": limit, "window_seconds": window}, - ) - retry_after = {"Retry-After": str(window)} - if wants_json(request): - response = json_error(429, "Rate limit exceeded. Try again later.") - response.headers["Retry-After"] = str(window) - return response - return HTMLResponse( - "Rate limit exceeded. Try again later.", - status_code=429, - headers=retry_after, - ) - timestamps.append(now) - _rate_limit_store[ip] = timestamps - return await call_next(request) - - _MAINTENANCE_ALLOWED_PREFIXES = ("/static", "/avatar", "/auth", "/admin", "/openai") - - @app.middleware("http") - async def maintenance_middleware(request: Request, call_next): - if _hot_settings()["maintenance_mode"] != "1": - return await call_next(request) - if request.url.path.startswith(_MAINTENANCE_ALLOWED_PREFIXES): - return await call_next(request) - user = get_current_user(request) - if user and user.get("role") == "Admin": - return await call_next(request) - message = get_setting( - "maintenance_message", - "DevPlace is undergoing scheduled maintenance. Please check back shortly.", - ) - audit.record( - request, - "security.maintenance.block", - user=user, - result="denied", - summary="non-admin request blocked by maintenance mode", - ) - if wants_json(request): - return json_error(503, message) - seo_ctx = base_seo_context( - request, title="Maintenance - DevPlace", description=message, robots="noindex" - ) - return templates.TemplateResponse( - request, - "error.html", - {**seo_ctx, "request": request, "error_code": 503, "error_message": message}, - status_code=503, - ) - - @app.middleware("http") - async def track_presence(request: Request, call_next): - path = request.url.path - if not path.startswith(("/static", "/avatar")): - user = get_current_user(request) - if user: - presence.touch(user["uid"]) - return await call_next(request) - - @app.middleware("http") - async def visit_statistics(request: Request, call_next): - from devplacepy.services.statistics.tracking import track_visit - - response = await call_next(request) - track_visit(request, response.status_code) - return response - - @app.middleware("http") - async def response_timing(request: Request, call_next): - start = time.perf_counter() - request.state.request_start = start - response = await call_next(request) - response.headers["X-Response-Time"] = f"{(time.perf_counter() - start) * 1000:.1f}ms" - return response - - app.add_middleware(GZipMiddleware, minimum_size=512, compresslevel=6) - - @app.get("/") - async def landing(request: Request): - user = get_current_user(request) - - landing_articles = _landing_news() - blocked = get_blocked_uids(user["uid"]) if user else frozenset() - landing_posts = _landing_recent_posts(blocked) - - base = site_url(request) - seo_ctx = base_seo_context( - request, - title="DevPlace - The Developer Social Network", - description="Track industry shifts. Discover bold releases. Share what you're building in an open, uncensored environment.", - breadcrumbs=[], - schemas=[website_schema(base)], - ) - return respond( - request, - "landing.html", - { - **seo_ctx, - "request": request, - "user": user, - "is_authenticated": bool(user), - "user_post_count": get_user_post_count(user["uid"]) if user else 0, - "user_stars": get_user_stars(user["uid"]) if user else 0, - "landing_articles": landing_articles, - "landing_posts": landing_posts, - }, - model=LandingOut, - ) - - return app - - -app = create_web_app() \ No newline at end of file diff --git a/devplacepy_services/web/ingress.py b/devplacepy_services/web/ingress.py deleted file mode 100644 index d93636c8..00000000 --- a/devplacepy_services/web/ingress.py +++ /dev/null @@ -1,216 +0,0 @@ -import asyncio -import logging -from urllib.parse import urlparse - -import httpx -import uuid_utils -import websockets -from starlette.datastructures import Headers -from starlette.requests import Request -from starlette.responses import Response -from starlette.routing import Route -from starlette.websockets import WebSocket - -from devplacepy_services.base.config import service_url -from devplacepy_services.base.errors import error_response -from devplacepy_services.base.manifest import INGRESS_ROUTES as _INGRESS_SPECS -from devplacepy_services.base.proxy import HOP_HEADERS - -logger = logging.getLogger(__name__) - -INGRESS_ROUTES = [(route.prefix, route.service) for route in _INGRESS_SPECS] - -TIMEOUTS = { - "xmlrpc": 120.0, -} - - -def _forward_headers_from_scope(scope) -> dict[str, str]: - original = Headers(scope=scope) - headers = { - k: v for k, v in original.items() if k.lower() not in HOP_HEADERS - } - headers["X-Request-Id"] = headers.get("X-Request-Id") or uuid_utils.uuid7().hex - headers["Accept-Encoding"] = "identity" - # Mirror nginx's `proxy_set_header Host $host` (Appendix F) so an - # upstream-generated absolute URL (redirect, url_for) reflects the - # public :10500 endpoint the browser is actually talking to, not the - # upstream service's own internal bind address/port. - original_host = original.get("host") - if original_host: - headers["Host"] = original_host - return headers - - -def _forward_headers(request: Request) -> dict[str, str]: - return _forward_headers_from_scope(request.scope) - - -_WS_HANDSHAKE_HEADERS = frozenset( - { - "sec-websocket-key", - "sec-websocket-version", - "sec-websocket-extensions", - "sec-websocket-protocol", - "sec-websocket-accept", - } -) - - -def _forward_ws_headers(scope) -> dict[str, str]: - headers = _forward_headers_from_scope(scope) - for key in list(headers): - if key.lower() in _WS_HANDSHAKE_HEADERS: - del headers[key] - return headers - - -def _target_path(scope, prefix: str) -> str: - # Starlette's Mount rewrites scope["root_path"] to the cumulative mount - # prefix but leaves scope["path"] as the FULL original request path (it - # does not strip the prefix) - so the full path alone is already the - # correct upstream path; concatenating root_path in front double-prefixes it. - return scope.get("path", "") or prefix - - -def _upstream_http_url(service: str, path: str, query: str) -> str: - base = service_url(service).rstrip("/") - url = f"{base}{path}" - if query: - url = f"{url}?{query}" - return url - - -def _upstream_ws_url(service: str, path: str, query: str) -> str: - parsed = urlparse(service_url(service)) - scheme = "wss" if parsed.scheme == "https" else "ws" - netloc = parsed.netloc - url = f"{scheme}://{netloc}{path}" - if query: - url = f"{url}?{query}" - return url - - -class IngressProxy: - def __init__(self, prefix: str, service: str) -> None: - self.prefix = prefix - self.service = service - self.timeout = TIMEOUTS.get(service, 30.0) - - async def __call__(self, scope, receive, send) -> None: - if scope["type"] == "http": - await self._proxy_http(scope, receive, send) - elif scope["type"] == "websocket": - await self._proxy_ws(scope, receive, send) - - async def _proxy_http(self, scope, receive, send) -> None: - request = Request(scope, receive) - path = _target_path(scope, self.prefix) - query = scope.get("query_string", b"").decode() - url = _upstream_http_url(self.service, path, query) - body = await request.body() - headers = _forward_headers(request) - try: - async with httpx.AsyncClient( - timeout=self.timeout, follow_redirects=False - ) as client: - upstream = await client.request( - request.method, - url, - headers=headers, - content=body, - ) - except httpx.HTTPError as exc: - logger.warning("ingress %s upstream error: %s", self.prefix, exc) - response = error_response( - 502, "Upstream service unavailable", "upstream_error" - ) - await response(scope, receive, send) - return - out_headers = { - k: v - for k, v in upstream.headers.items() - if k.lower() not in HOP_HEADERS and k.lower() != "set-cookie" - } - response = Response( - content=upstream.content, - status_code=upstream.status_code, - headers=out_headers, - media_type=upstream.headers.get("content-type"), - ) - for cookie in upstream.headers.get_list("set-cookie"): - response.headers.append("set-cookie", cookie) - await response(scope, receive, send) - - async def _proxy_ws(self, scope, receive, send) -> None: - client_ws = WebSocket(scope, receive, send) - path = _target_path(scope, self.prefix) - query = scope.get("query_string", b"").decode() - upstream_url = _upstream_ws_url(self.service, path, query) - headers = _forward_ws_headers(scope) - await client_ws.accept() - try: - async with websockets.connect( - upstream_url, - open_timeout=10, - max_size=None, - additional_headers=headers, - ) as upstream: - await _pump(client_ws, upstream) - except Exception as exc: - logger.debug("ingress ws %s failed: %s", self.prefix, exc) - try: - await client_ws.close(code=1011) - except Exception: - pass - - -async def _pump(client_ws: WebSocket, upstream) -> None: - async def client_to_upstream(): - try: - while True: - message = await client_ws.receive() - if message["type"] == "websocket.disconnect": - break - if message.get("text") is not None: - await upstream.send(message["text"]) - elif message.get("bytes") is not None: - await upstream.send(message["bytes"]) - except Exception: - pass - finally: - await upstream.close() - - async def upstream_to_client(): - try: - async for message in upstream: - if isinstance(message, (bytes, bytearray)): - await client_ws.send_bytes(bytes(message)) - else: - await client_ws.send_text(message) - except Exception: - pass - finally: - try: - await client_ws.close() - except Exception: - pass - - await asyncio.gather(client_to_upstream(), upstream_to_client()) - - -def mount_ingress(app) -> None: - for prefix, service in INGRESS_ROUTES: - proxy = IngressProxy(prefix, service) - # A bare hit on the prefix itself (no trailing slash, nothing after - - # e.g. a POST to /xmlrpc or GET /tools) never matches Mount's own - # path regex (it requires a "/" plus content after the prefix), so - # Starlette's router-level redirect_slashes fallback 307s it to - # "/" before the Mount ever sees it. If the upstream service's - # own router registers an exact route at its mount root (as /tools - # and /xmlrpc both do), THAT redirects back to the bare prefix - - # an infinite loop between the two opposite trailing-slash - # conventions. Registering an explicit Route at the exact prefix - # bypasses Mount's regex/redirect fallback entirely for that one path. - app.router.routes.append(Route(prefix, endpoint=proxy, methods=None)) - app.mount(prefix, proxy) \ No newline at end of file diff --git a/devplacepy_services/web/main.py b/devplacepy_services/web/main.py deleted file mode 100644 index 4e89c9b2..00000000 --- a/devplacepy_services/web/main.py +++ /dev/null @@ -1,21 +0,0 @@ -from devplacepy_services.base.health import health_router -from devplacepy_services.base.service import BaseMicroservice -from devplacepy_services.web.factory import create_web_app - - -class WebService(BaseMicroservice): - name = "web" - title = "Web" - default_port = 10500 - workers = "auto" - stateful = False - depends_on = ["database", "pubsub"] - - def build_app(self): - app = create_web_app() - app.include_router(health_router(self)) - return app - - -_service = WebService() -app = _service.build_app() diff --git a/devplacepy_services/xmlrpc/SERVICE.md b/devplacepy_services/xmlrpc/SERVICE.md deleted file mode 100644 index 14520ff5..00000000 --- a/devplacepy_services/xmlrpc/SERVICE.md +++ /dev/null @@ -1 +0,0 @@ -# XML-RPC Service (stub) \ No newline at end of file diff --git a/devplacepy_services/xmlrpc/__init__.py b/devplacepy_services/xmlrpc/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/devplacepy_services/xmlrpc/main.py b/devplacepy_services/xmlrpc/main.py deleted file mode 100644 index 2f102cc5..00000000 --- a/devplacepy_services/xmlrpc/main.py +++ /dev/null @@ -1,24 +0,0 @@ -import devplacepy_services.base.bootstrap - -from devplacepy.routers import xmlrpc -from devplacepy.services.xmlrpc import XmlrpcService -from devplacepy_services.base.service import BaseMicroservice, build_standard_app - - -class XmlrpcMicroservice(BaseMicroservice): - name = "xmlrpc" - title = "XML-RPC" - default_port = 10649 - workers = 1 - stateful = True - depends_on = ["database"] - managed_services = [XmlrpcService()] - use_background = True - run_supervisor = True - - def build_app(self): - return build_standard_app(self, routers=[(xmlrpc.router, "/xmlrpc")]) - - -_service = XmlrpcMicroservice() -app = _service.build_app() \ No newline at end of file diff --git a/fplan.md b/fplan.md deleted file mode 100644 index 37cffc81..00000000 --- a/fplan.md +++ /dev/null @@ -1,589 +0,0 @@ -# Code Farm Economy Rebalance — Implementation Plan - -Status: planning document only, no code changed yet. Target: execute phase by phase against the live `devplacepy` repository. The game is in production; every phase below is additive-only (new columns/tables/functions with safe defaults, no renames, no destructive migrations) so a partially-completed rollout never breaks an existing farm, and `init_db` stays idempotent as today. - -## Ground rules for whoever executes this plan - -1. **Never rename or remove an existing column, table, function signature, route, or schema field.** Extend with new optional fields/params carrying safe defaults. -2. **Every new `game_farms`/`game_plots` column must default to a value that reproduces today's behavior for a legacy row that has never touched the new feature** — `0` for counters/booleans, `""` for timestamps, matching the existing `store/common.py::_lvl()` convention (`int(farm.get(key) or 0)`). -3. **Reuse the established patterns before inventing new ones.** This codebase already has four proven shapes for "a shop of upgrades": `CI_TIERS` (tiered, coin-priced, single column), `PERKS` (leveled, coin-priced, resets on prestige), `LEGACY_UPGRADES` (leveled, star-priced, survives prestige), and the atomic-upsert-counter pattern in `database._add_usage`. Every new system below is explicitly modeled on one of these four — do not invent a fifth shape without a documented reason. -4. **No new background service, no new tick.** The game's hard architectural invariant is "pure + timestamp-driven, no background tick" (`services/game/CLAUDE.md`). Every new mechanic below is either (a) computed lazily inside `serialize_farm`/`store` calls the same way `_auto_harvest` already is, or (b) an explicit admin/user action. None of them run on a scheduler. -5. **Follow the "four faces of one route" workflow from the root `CLAUDE.md`** for every new route: HTML + JSON via `respond(..., model=XOut)`, a Devii `Action` in `services/devii/actions/catalog/game.py`, a `docs_api` entry in the Code Farm group, and a badge/achievement hook where it fits the existing pattern. -6. **Validate, never test-run automatically.** After each phase: `python -c "from devplacepy.main import app"`, grep the touched files for em-dash characters/entities, and manually check JS/CSS/HTML balance. Do **not** run `make test` unless the user explicitly asks — write new tests in the matching tier (`tests/unit/services/game/`, `tests/api/game/`, `tests/e2e/game/`) so the user can run them before deploying. -7. **Update `devplacepy/services/game/CLAUDE.md`, `README.md`, and this repo's docs (`docs_api`, `/docs` prose if relevant) at the end of every phase**, per the root `CLAUDE.md` feature workflow — do not defer documentation to the end of the whole plan. - ---- - -## Why a plan this size, and what is deliberately scoped down - -The critique is being taken in full. Six places in it ask for something the current codebase has no state for at all (raid "attempts," real-time raid scouting, weekly long-term goals, cross-user alliances, automatic timed resets). Building those literally, as new bespoke systems, would multiply the surface area and the risk. Instead each of those is folded into or approximated with a mechanism the codebase already has proven safe, and every such decision is called out explicitly below in **"Scope decisions and deviations"** so it can be reviewed before Phase 5+6 ship. Nothing is dropped — everything is mapped to a concrete, buildable primitive. - ---- - -## Phase order - -| Phase | Content | Depends on | Risk | -|---|---|---|---| -| 0 | Foundations: split `economy.py` into a package, introduce a single coin/xp credit choke point | none | low (mechanical, behavior-preserving) | -| 1 | Market Saturation | 0 | low | -| 2 | Coin sinks: Infrastructure tiers, upkeep Defense building, Cosmetics/titles | 0 | medium | -| 3 | Mastery track + new crop families | 0, 2 (cosmetics reused for era rewards later) | medium | -| 4 | Secondary leaderboards | 0, 3 (harvests_week, mastery) | low | -| 5 | Era/season system | 0, 1, 2, 3, 4 | high — recommend a second pass | -| 6 | Engagement loops: underdog bonus, weekly contracts, notification polish, (Alliance/Coop sketched, deferred) | 0, 3 | medium | - -This mirrors the critique's own "Implementation order" section, with Phase 0 inserted first because Phases 1, 3, 4, and 5 all add a shadow counter next to every coin credit — doing that refactor once, up front, removes the single biggest source of "forgot one call site" bugs in the whole plan. - ---- - -## Phase 0 — Foundations (no gameplay change) - -### 0.1 Split `services/game/economy.py` into a package - -The codebase already has the exact precedent for this: `store.py` was previously one file and is now the `store/` package (`actions.py`, `common.py`, `farm.py`, `quests.py`, `serialize.py`) re-exported from `store/__init__.py`. `economy.py` is about to roughly triple in size (crops, CI, leveling, perks, legacy, market, infrastructure, defense, mastery, cosmetics, leaderboard scoring, era — 12 concern areas), so apply the same split mechanically, before any new content is added: - -- `devplacepy/services/game/economy/__init__.py` — re-exports every name from the submodules below, so `from devplacepy.services.game import economy; economy.CROPS` keeps working unchanged everywhere (`store/*.py`, `routers/game/*.py`, tests). -- `economy/crops.py` — `Crop`, `CROPS`, `CROP_BY_KEY`, `crop_for`, `unlocked_crops`, `crop_payload`. -- `economy/ci.py` — `CiTier`, `CI_TIERS`, `CI_BY_TIER`, `MAX_CI_TIER`, `ci_speed`, `next_ci_tier`, `farm_speed`, `grow_seconds_for`, `water_bonus_seconds`. -- `economy/leveling.py` — `MAX_LEVEL`, `xp_threshold`, `level_for_xp`, `level_progress`, `STARTING_COINS`, `STARTING_PLOTS`, `MAX_PLOTS`, `plot_cost`. -- `economy/perks.py` — `Perk`, `PERKS`, `PERK_BY_KEY`, `perk_for`, `perk_cost`, `perk_value_text`. -- `economy/legacy.py` — `LegacyUpgrade`, `LEGACY_UPGRADES`, `LEGACY_BY_KEY`, `legacy_for`, `legacy_cost`, `legacy_multiplier`, `legacy_value_text`, `prestige_base_plots`, `effective_steal_grace`, `effective_steal_fraction`, `LEGACY_SPEED_STEP`, `LEGACY_MULT_STEP`, `LEGACY_DEFENSE_GRACE`, `LEGACY_DEFENSE_FRACTION`. -- `economy/prestige.py` — `PRESTIGE_MIN_LEVEL`, `PRESTIGE_BONUS`, `prestige_multiplier`, `STAR_BASE`, `stars_for_refactor`. -- `economy/rewards.py` — `effective_plant_cost`, `effective_reward_coins`, `effective_reward_xp`, `steal_reward_coins`, `is_golden`, `GOLDEN_CHANCE`, `GOLDEN_MULTIPLIER`, `WATER_BONUS_PCT`, `MAX_WATERS_PER_PLOT`, `WATER_REWARD_COINS`, `WATER_REWARD_XP`, `STEAL_GRACE_SECONDS`, `STEAL_FRACTION`, `STEAL_COOLDOWN_SECONDS`. -- `economy/daily.py` — `DAILY_BASE`, `DAILY_STREAK_STEP`, `DAILY_STREAK_CAP`, `daily_reward`, `FERTILIZE_FRACTION`, `FERTILIZE_TAX`, `fertilize_click_cost`. -- `economy/quests.py` — `QuestDef`, `QUEST_DEFS`, `QUEST_KINDS`, `DAILY_QUEST_COUNT`, `daily_quests`. -- `economy/scoring.py` — `SCORE_*` constants, `farm_score`. -- `economy/market.py`, `economy/infrastructure.py`, `economy/defense.py`, `economy/mastery.py`, `economy/cosmetics.py`, `economy/era.py` — empty stub modules created now, populated in Phases 1-6. - -This is a pure mechanical move: cut/paste function bodies verbatim, no formula changes. Verify with `python -c "from devplacepy.main import app"` and a full-text `grep -rn "from devplacepy.services.game import economy" devplacepy/ tests/` to confirm every caller still resolves every name it uses (the `__init__.py` re-export must be exhaustive — build it by `grep -oP '^(def|class) \w+' economy/*.py` and export every one). - -### 0.2 Single coin/xp credit choke point - -Today, `store/actions.py` updates `game_farms.coins`/`xp`/`total_harvests` inline, separately, at five call sites: `harvest`, `_auto_harvest`, `steal` (thief side), `claim_daily`, `claim_quest`, and `water` (visitor reward). Phases 1, 3, 4, and 5 each need to add one more shadow counter next to every coin credit (market-tick recording, `lifetime_coins_earned`, `harvests_week`, `era_coins`). Doing that at five scattered sites five times is exactly the kind of duplication the project's DRY convention forbids. Introduce one function now, before any of those counters exist, so every later phase touches one place: - -In `store/common.py`, add: - -``` -def credit_farm(farm: dict, *, coins: int = 0, xp: int = 0, harvests: int = 0, is_kernel_harvest: bool = False, extra: dict | None = None) -> dict: -``` - -- Computes the new `coins`, `xp` (re-deriving `level` via `economy.level_for_xp`), `total_harvests` exactly as the current inline code does at each site today. -- Merges any `extra` fields (a plain dict of additional column updates) into the same single `_update_farm` call — this is the extension point every later phase hooks into instead of adding a sixth ad-hoc update site. -- Returns the updated farm dict (mirrors what each call site already does by re-reading/merging locally). -- `is_kernel_harvest` is accepted now (unused) so Phase 4's time-to-Kernel tracking has a stable signature to fill in later without touching every call site again. - -Refactor `harvest`, `_auto_harvest`, `steal`, `claim_daily`, `claim_quest`, `water` in `store/actions.py` to call `credit_farm(...)` instead of their current inline `_update_farm` math. This step must be **behaviorally invisible** — same coins, same xp, same level, same total_harvests as before, verified by re-reading the diff against the current formulas rather than by running the suite (per the "never run tests unless asked" rule), and by asking the user to run `tests/unit/services/game/store.py` + `tests/api/game/mutations.py` before this phase is considered done, since they already assert exact reward amounts. - ---- - -## Phase 1 — Market Saturation - -### Design - -A global per-crop production tracker computed from real harvest events (not a fabricated log — a genuinely new table, since today's harvest path never persists an event, only the aggregate `total_harvests` counter). Recent-harvest volume per crop reduces that crop's payout; low-tier crops get a small relief buff while the market is saturated. No existing column or table is touched. - -**Deliberate scope decision:** only owner-initiated harvests and auto-harvests count toward saturation, not steals. A steal moves an already-grown build's value from owner to thief; it does not add new supply to the economy. Counting it would double-count the same unit of production. This is a real simplification of "calculated from existing harvest logs" (there is no existing log; a new lightweight one is added) but keeps the causal story of the feature honest: it throttles printing, not raiding. - -### New table - -`game_market_ticks` (added to the ensure-block in `devplacepy/database/schema.py` right after the existing `game_quests` block, same non-soft-deletable treatment as the other `game_*` tables): - -| column | type/default | notes | -|---|---|---| -| `uid` | str | generate_uid(), gets the standard `idx_game_market_ticks_uid` via the `_uid_index` loop | -| `crop_key` | str, `""` | | -| `hour_bucket` | str, `""` | UTC `"YYYY-MM-DDTHH"` | -| `harvests` | int, `0` | | -| `updated_at` | str, `""` | | - -Unique index `idx_game_market_ticks_bucket` on `(crop_key, hour_bucket)` — this is both the query index and the `ON CONFLICT` target for the upsert below. - -### New module `services/game/store/market.py` (added to the `store/` package, exported from `store/__init__.py`) - -- `_ticks()` → `get_table("game_market_ticks")`. -- `_hour_bucket(now: datetime) -> str`. -- `record_harvest_tick(crop_key: str, now: datetime) -> None` — atomic upsert, modeled exactly on `database._add_usage`'s pattern: - ``` - with db: - db.query( - "INSERT INTO game_market_ticks (uid, crop_key, hour_bucket, harvests, updated_at) " - "VALUES (:uid, :crop_key, :hour_bucket, 1, :now) " - "ON CONFLICT(crop_key, hour_bucket) DO UPDATE SET harvests = harvests + 1, updated_at = :now", - uid=generate_uid(), crop_key=crop_key, hour_bucket=_hour_bucket(now), now=_iso(now), - ) - ``` - The `with db:` wrapper is load-bearing (memory: `dataset-dbquery-no-autocommit` — a raw `db.query` write that does not commit holds the SQLite write lock). -- `_saturation_cache = TTLCache(ttl=30, max_size=32)` keyed by `crop_key` — a display/economic cache, not a correctness cache, same class as the existing `_leaderboard_cache`; no `cache_state` version wiring needed (matches the documented rule: cross-worker version-sync is reserved for correctness-critical caches, this one only affects a rolling 48h window's precision by up to 30s). -- `recent_harvests(crop_key: str, window_hours: int) -> int` — `SELECT COALESCE(SUM(harvests), 0) FROM game_market_ticks WHERE crop_key = :crop_key AND hour_bucket >= :cutoff`, cutoff = `_hour_bucket(now - window_hours)`. -- `prune_ticks(older_than_hours: int = 96) -> int` — deletes buckets older than the cutoff, returns rows removed. Wired into a new CLI command (see below) so the table never grows unbounded, matching the existing prune convention (`zips prune`, `forks prune`, `seo prune`, etc.). - -### `economy/market.py` - -``` -MARKET_WINDOW_HOURS = 48 -MARKET_SATURATION_TIERS: tuple[tuple[int, float], ...] = ( - (0, 1.00), (40, 0.85), (120, 0.70), (300, 0.55), (600, 0.40), -) -MARKET_BUFFED_CROPS = ("shell", "python", "webapp", "api") -MARKET_BUFF_CAP = 1.15 - -def market_saturation_factor(recent_harvests: int) -> float: ... -def market_buff_factor(crop_key: str, saturation_factor: float) -> float: ... -``` - -### Wiring into `economy/rewards.py` - -Add an optional trailing parameter, default preserves old behavior for any caller (tests, future code) that omits it: - -``` -def effective_reward_coins(crop, yield_level=0, prestige=0, legacy_mult_level=0, market_factor=1.0) -> int: - ...factor *= market_factor... - -def steal_reward_coins(crop, yield_level=0, prestige=0, legacy_mult_level=0, defense_level=0, market_factor=1.0) -> int: - ...passes market_factor through to effective_reward_coins... -``` - -`economy/crops.py::crop_payload(...)` also gains `market_factor: float = 1.0` so the shop preview shows the live, saturation-adjusted number before the player plants (avoids "why did I get less than shown"). - -### Wiring into `store/actions.py` - -At `harvest`, `_auto_harvest`, and `steal`, before computing the coin reward: `recent = market.recent_harvests(crop.key, economy.MARKET_WINDOW_HOURS)`, `sat = economy.market_saturation_factor(recent)`, `factor = sat * economy.market_buff_factor(crop.key, sat)`, pass `market_factor=factor` into the reward call. After a successful `harvest`/`_auto_harvest` (not `steal`, per the scope decision above), call `market.record_harvest_tick(crop.key, now)`. - -`store/serialize.py::serialize_farm` computes the same `factor` per crop when building `economy.crop_payload(...)` for the shop list. - -### Schema - -`GameCropOut` gains `market_state: str = "normal"` (`"normal"`/`"saturated"`/`"boosted"`, derived from whether `market_factor` is `<1`, `>1`, or `==1`). This is the only new field; `reward_coins` already reflects the real payout since it now includes the factor. - -### Frontend - -`_game_grid.html` / `_game_shop.html` and `GameFarm.js::_shopHtml`/`_gridHtml` render a small inline label next to a crop's reward when `market_state != "normal"` (`"Saturated -30%"` / `"Boosted +15%"`), no new CSS classes beyond one small modifier reusing existing badge styling patterns. - -### CLI - -`devplace game market prune` → calls `store.market.prune_ticks()`, added to `devplacepy/cli/` next to the other prune subcommands, documented in the root `CLAUDE.md` Commands table. - -### Devii / docs - -No new route, so no new Devii action — `game_state`/`game_view_farm`/`game_plant` responses automatically carry the new `market_state` field once it exists on `GameCropOut`. Add one sentence to the Code Farm `docs_api` group intro describing the field. - -### Tests - -`tests/unit/services/game/economy.py` — `market_saturation_factor`/`market_buff_factor` pure-function cases. `tests/unit/services/game/store.py` — `record_harvest_tick`/`recent_harvests` roundtrip. `tests/api/game/mutations.py` — harvesting the same crop repeatedly in a short window measurably reduces payout past the first saturation threshold. - ---- - -## Phase 2 — Coin sinks - -Three sub-systems, each modeled on an existing shape. - -### 2a. Infrastructure tiers (one-time purchases, modeled on `LEGACY_UPGRADES`'s "owned" framing but boolean, per the critique's literal "new upgrade table, default owned = false") - -`economy/infrastructure.py`: - -``` -@dataclass(frozen=True) -class Infrastructure: - key: str - name: str - icon: str - description: str - cost: int - min_prestige: int - -INFRASTRUCTURE: tuple[Infrastructure, ...] = ( - Infrastructure("registry", "Private Registry", "📦", - "Rust, Compiler, and Kernel crops grow 15% faster", 25_000_000, 3), - Infrastructure("canary", "Canary Deployments", "🐤", - "Every harvest has a 12% chance to double and a 6% chance to only refund its planting cost", 75_000_000, 8), - Infrastructure("observability", "Observability Suite", "🔭", - "Raises the minimum coins you keep when raided from 10% to 30%", 150_000_000, 15), -) -INFRA_BY_KEY = {i.key: i for i in INFRASTRUCTURE} -``` - -**Scope decision:** the critique's "see raid attempts in real time" for Observability is dropped — there is no scouting/detection mechanic anywhere in the game, and inventing one (who counts as "attempting," how it's surfaced, whether it needs a new pub/sub topic) is a distinct feature, not a coin sink. Observability instead gets a second, coin-preserving effect (raising the steal-fraction floor) that serves the same "stop feeling helpless against whales' raids" goal without new state. - -New `game_farms` columns: `infra_registry=0`, `infra_canary=0`, `infra_observability=0` (int 0/1). - -New `store/infrastructure.py`: -- `buy_infrastructure(user, key) -> dict` — `GameError` if unknown key, already owned, `prestige < INFRA_BY_KEY[key].min_prestige`, or insufficient coins; deducts cost (`credit_farm(farm, coins=-cost, extra={f"infra_{key}": 1})`); returns `{"key", "spent"}`. - -Wiring: -- `registry` → `economy/ci.py::farm_speed`/`grow_seconds_for` gain a `registry_boost: bool = False` param, applied only for crop keys `{"rust", "haskell", "kernel"}`, threaded from `store` the same way `legacy_speed_level` already is. -- `canary` → in `harvest()`/`_auto_harvest()`, if `farm["infra_canary"]`, roll `random.random()` once per harvest (plain `random`, not the deterministic `is_golden` hash — canary is a fresh roll every time, not a per-planting property, so determinism has no purpose here) against `CANARY_DOUBLE_CHANCE=0.12` then `CANARY_FAIL_CHANCE=0.06`, adjusting the coin gain accordingly (double, or floor at the crop's `effective_plant_cost` refund, or normal). -- `observability` → `economy/legacy.py::effective_steal_fraction`'s floor (`max(0.1, ...)`) becomes `max(0.3 if owner_has_observability else 0.1, ...)`, threaded as a new `observability: bool = False` parameter alongside `defense_level`. - -### 2b. Upkeep Defense building (the wealth-proportional sink — the critique's most important ask) - -`economy/defense.py`: - -``` -@dataclass(frozen=True) -class DefenseTier: - level: int - name: str - upgrade_cost: int - upkeep_daily: int - steal_fraction_floor: float - grace_bonus: int - -DEFENSE_TIERS: tuple[DefenseTier, ...] = ( - DefenseTier(0, "Undefended", 0, 0, 0.10, 0), - DefenseTier(1, "Firewall", 5_000, 500, 0.10, 15), - DefenseTier(2, "WAF", 40_000, 2_500, 0.08, 30), - DefenseTier(3, "SOC Monitoring", 300_000, 15_000, 0.06, 60), - DefenseTier(4, "Zero Trust Mesh", 2_000_000, 100_000, 0.04, 120), -) -UPKEEP_WEALTH_PCT = 0.002 # 0.2% of current coin balance per day, whichever is larger than the flat fee -UPKEEP_GRACE_DAYS = 2 # unpaid days tolerated before the tier decays by one level - -def daily_upkeep(tier: DefenseTier, coins: int) -> int: - return max(tier.upkeep_daily, round(coins * UPKEEP_WEALTH_PCT)) -``` - -The `max(flat, wealth_pct)` formula is what makes this a genuine whale sink: a 282M-coin balance owes `max(100_000, 564_000) = 564_000`/day at tier 4, not a trivial flat fee, while a small farm pays the cheap flat floor. - -New `game_farms` columns: `defense_level=0` (int), `defense_last_upkeep_at=""` (str timestamp). - -New `store/defense.py`: -- `upgrade_defense(user) -> dict` — one-time purchase of the next tier, mirrors `upgrade_perk`'s shape exactly (raises on max tier / insufficient coins). -- `charge_upkeep(farm: dict, now: datetime) -> dict` — called lazily at the very top of `serialize_farm`, in the same place and spirit as `_auto_harvest` (lazy, timestamp-driven, no tick): if `defense_level > 0` and at least one full day elapsed since `defense_last_upkeep_at`, charge `daily_upkeep(tier, coins)` per elapsed day (capped at `UPKEEP_GRACE_DAYS` days of arrears); if coins are insufficient to cover even one day, decrement `defense_level` by one instead of going negative, and reset the timer — this implements "if you don't pay, protection weakens" literally. Returns the updated farm (or the original if nothing was due), same idiom as `_auto_harvest`'s return contract. - -Wiring: `defense_level`'s `steal_fraction_floor`/`grace_bonus` combine additively with the existing `legacy_defense` level inside `effective_steal_fraction`/`effective_steal_grace` (both already take a `defense_level` int; extend the call site in `store` to pass `legacy_defense_level + defense_level`-equivalent inputs, or add a second explicit parameter — pick whichever keeps the function signatures readable; document the final choice in the code, not just here). - -Schema: `GameFarmOut` gains `defense_level: int = 0`, `defense_tier_name: str = ""`, `defense_upkeep_daily: int = 0`, `defense_next_cost: int = 0`. - -Route: `POST /game/defense/upgrade` (no body — mirrors `POST /game/upgrade` for CI), added to `routers/game/index.py` through the same `_respond_action` choke. Devii action `game_upgrade_defense`. `docs_api` entry. Template: new `_game_defense.html` partial + `data-defense-host` in `game.html`, `GameFarm.js::_defenseHtml`. - -### 2c. Cosmetics and titles (pure status, coins or coins+stars) - -New table `game_cosmetics` (own table, not flat columns, because this catalog is meant to grow over time and later feed Era rewards in Phase 5 — unlike the fixed five-slot Infrastructure/Defense tiers, this is an open-ended, appendable list): - -| column | type/default | -|---|---| -| `uid` | str | -| `user_uid` | str | -| `cosmetic_key` | str | -| `purchased_at` | str | -| `created_at` | str | - -Unique index `idx_game_cosmetics_owner` on `(user_uid, cosmetic_key)`. - -`economy/cosmetics.py`: -``` -@dataclass(frozen=True) -class Cosmetic: - key: str - name: str - icon: str - description: str - cost_coins: int - era_key: str | None = None # None = always purchasable; set in Phase 5 for era-exclusive items - -COSMETICS: tuple[Cosmetic, ...] = ( - Cosmetic("title_architect", "The Architect", "🏛️", "A permanent title shown on the leaderboard", 500_000), - Cosmetic("title_refactorer", "Serial Refactorer", "♻️", "Requires prestige >= 3 to purchase", 250_000), - Cosmetic("title_kernel_hacker", "Kernel Hacker", "⚙️", "Requires having harvested a Kernel", 1_000_000), - Cosmetic("skin_neon", "Neon Terminal", "🌈", "A cosmetic plot skin, no gameplay effect", 750_000), -) -COSMETIC_BY_KEY = {c.key: c for c in COSMETICS} -``` - -`store/cosmetics.py`: `buy_cosmetic(user, key)`, `owned_cosmetic_keys(user_uid) -> set[str]`, `equip_title(user, key)` (raises if not owned). - -New `game_farms` column `active_title=""` (str, one of `COSMETIC_BY_KEY` keys of kind title, or empty). - -Routes: `POST /game/cosmetics/buy` (`GameCosmeticForm{key}`), `POST /game/cosmetics/equip` (`GameCosmeticForm{key}`). Schema: `GameFarmOut` gains `owned_cosmetics: list[str] = []`, `active_title: str = ""`; `GameLeaderboardEntryOut` gains `title: str = ""`. - -**Scope boundary, stated explicitly:** the active title renders on the Code Farm leaderboard and farm-view page only for v1. It does not touch `_avatar_link.html`, the profile page, or any other sitewide identity surface — that would be a much larger, riskier change (global avatar/identity partial touched by every render site in the app) for a feature that is currently scoped to one subsystem. - -### Phase 2 badges - -New `ACHIEVEMENTS` entries (group `"Code Farm"`): `infra_bought` -> `[(1, "Enterprise Ready")]`, `defense_upgraded` -> `[(1, "Fort Knox")]`, `cosmetic_bought` -> `[(1, "Style Points")]`. `track_action` calls at the three new success points. - -### Phase 2 tests - -Unit tests for `daily_upkeep`, `infra` cost/gating, `cosmetic` gating. API tests for each new route's success/failure paths (insufficient coins, already owned, prestige gate). E2E test for the new shop panel appearing and a purchase flowing through. - ---- - -## Phase 3 — Mastery track and new crop families - -### Design - -An orthogonal permanent track, unlocked once a farm's prestige has crossed a threshold, spent on a small tree of upgrades that open new gameplay rather than bigger numbers, per the critique's stated design philosophy. Modeled directly on `LEGACY_UPGRADES` (leveled, survives prestige, its own currency). - -**Two-counter design, load-bearing:** Mastery needs both a *spendable balance* (`mastery_points`, decreases when spent) and a *lifetime-earned total* (`mastery_points_earned_total`, never decreases) because the new crop families must stay unlocked even after a player spends their mastery points down to zero. Gating unlocks on the spendable balance would re-lock content the moment it's spent — a real bug if not caught here. - -### `economy/prestige.py` additions - -``` -MASTERY_UNLOCK_PRESTIGE = 50 -MASTERY_PRESTIGE_STEP = 10 - -def mastery_points_awarded(old_prestige: int, new_prestige: int) -> int: - if new_prestige < MASTERY_UNLOCK_PRESTIGE: - return 0 - baseline = max(old_prestige, MASTERY_UNLOCK_PRESTIGE - 1) - return (new_prestige - MASTERY_UNLOCK_PRESTIGE) // MASTERY_PRESTIGE_STEP - max(0, baseline - MASTERY_UNLOCK_PRESTIGE) // MASTERY_PRESTIGE_STEP -``` - -(Prestige increments by exactly 1 per refactor, so in practice this awards 1 point whenever the new prestige is a multiple of 10 at or above 50 — written as a range difference so it is also correct if a future change ever lets prestige jump by more than 1.) - -### `economy/mastery.py` - -``` -@dataclass(frozen=True) -class MasteryUpgrade: - key: str - name: str - icon: str - description: str - max_level: int - base_cost: int - cost_growth: float - -MASTERY_UPGRADES: tuple[MasteryUpgrade, ...] = ( - MasteryUpgrade("autoreplant", "Continuous Delivery", "🔁", - "Automatically replant the same crop right after harvest, if affordable", 1, 3, 1.0), - MasteryUpgrade("analytics", "Farm Analytics", "📊", - "Unlocks lifetime stats on your farm HUD", 1, 2, 1.0), - MasteryUpgrade("contracts", "Legacy Contracts", "📜", - "Unlocks a weekly long-term contract slot for Stars and a temporary boost", 1, 4, 1.0), -) -MASTERY_BY_KEY = {m.key: m for m in MASTERY_UPGRADES} - -def mastery_cost(m: MasteryUpgrade, level: int) -> int: - return round(m.base_cost * (m.cost_growth ** level)) -``` - -**DRY consolidation, called out explicitly:** the critique lists "Legacy Contracts" under Mastery (Section 2) and "Daily/Weekly contracts" under engagement loops (Section 6) as if they were two systems. They are implemented as **one** mechanism here: the `mastery_contracts` upgrade unlocks a fourth, weekly-cadence slot in the existing `game_quests` engine (see Phase 6). Building two parallel long-goal systems would violate the project's DRY convention for no gameplay benefit. - -### `game_farms` new columns - -`mastery_points=0`, `mastery_points_earned_total=0`, `mastery_autoreplant=0`, `mastery_analytics=0`, `mastery_contracts=0`, `lifetime_coins_earned=0`, `lifetime_harvests=0`. - -`lifetime_coins_earned`/`lifetime_harvests` are accumulate-only counters incremented via the Phase 0 `credit_farm` choke point (one line added there, not five). They start at 0 for every existing farm on the day this ships — that under-counts a veteran's true lifetime total, which is an accepted, explicitly-noted trade-off (the alternative, backfilling from `total_harvests`/no historical coin ledger, is not possible since no such ledger exists; `total_harvests` already exists and is NOT reset, so the analytics panel should show that pre-existing counter alongside the new since-Mastery ones, clearly labeled). - -### `store/actions.py::prestige()` change - -After computing `new_prestige`, add `mastery_points_awarded(old_prestige, new_prestige)` to both `mastery_points` and `mastery_points_earned_total` in the same reset/update dict — both columns are, like `stars` and `legacy_*`, deliberately **excluded** from the fields that get zeroed on refactor. - -### New `store/mastery.py` - -- `upgrade_mastery(user, key) -> dict` — spends `mastery_points` (not coins), mirrors `upgrade_legacy`'s exact shape. - -Wiring: -- `autoreplant` — extract a small internal helper `_plant_plot(farm, plot, crop, now) -> dict` out of the existing `plant()` body in `store/actions.py` (now genuinely has two callers: `plant()` itself and the auto-replant hook, so this is justified DRY, not premature abstraction). In `harvest()`/`_auto_harvest()`, after crediting and clearing, if `farm["mastery_autoreplant"]` and the same crop is still affordable and unlocked, call `_plant_plot` immediately. -- `analytics` — `GameFarmOut` gains `mastery_analytics_unlocked: bool = False`, `lifetime_coins_earned: int = 0`, `lifetime_harvests: int = 0` (always present on the schema per the existing "empty/zero unless unlocked" convention already used for `perks`/`quests`/`legacy`; template/JS render the panel only when `mastery_analytics_unlocked` is true). -- `contracts` — see Phase 6. - -### New crop families - -Extend the `Crop` dataclass (in `economy/crops.py`) with two new **trailing, defaulted** fields — safe because every existing `Crop(...)` construction in the `CROPS` tuple is positional with exactly 8 args and none of the 8 existing fields has a default, so appending defaulted fields after them is valid dataclass semantics and changes nothing for the 7 existing crops: - -``` -@dataclass(frozen=True) -class Crop: - key: str - name: str - icon: str - cost: int - grow_seconds: int - reward_coins: int - reward_xp: int - min_level: int - min_mastery: int = 0 - steal_immune: bool = False - era_key: str | None = None # populated in Phase 5 -``` - -New entries appended to `CROPS`: -``` -Crop("distsys", "Distributed System", "🕸️", 5_000, 14_400, 9_500, 900, MAX_LEVEL, min_mastery=1), -Crop("mlpipe", "ML Pipeline", "🧠", 12_000, 21_600, 21_000, 1_800, MAX_LEVEL, min_mastery=1), -Crop("secfort", "Security Fortress", "🔐", 30_000, 28_800, 48_000, 3_200, MAX_LEVEL, min_mastery=1, steal_immune=True), -``` - -`min_level=MAX_LEVEL` means the level gate is always satisfied once a player is capped (a prerequisite for prestige anyway), so the real gate is `min_mastery`, satisfied by `mastery_points_earned_total >= 1` (i.e. having reached prestige 50 at least once and earned a Mastery point — the tree does not need to be spent, only unlocked, to grow these crops; this matches the critique's framing that Mastery "opens new gameplay" independent of what's purchased). - -`unlocked_crops(level, mastery_earned=0)` signature gains the new parameter with a default of 0 (backward compatible for any other caller), filters `crop.min_level <= level and crop.min_mastery <= mastery_earned`. - -`steal_immune` wiring: in `store/actions.py::steal()`, if `crop.steal_immune`, raise `GameError("This build cannot be raided.")` before any grace/cooldown check; `serialize_plot` sets `can_steal=False`, `steal_reason="immune"` unconditionally for such a plot. - -### Phase 3 tests - -Unit: `mastery_points_awarded` edge cases (crossing 50, 60, skipping — though prestige only moves by 1, still test the range-difference formula), `unlocked_crops` gating by mastery, `steal_immune` short-circuit. API: `POST /game/legacy`-equivalent `POST /game/mastery` route full cycle. E2E: new crops appear in the shop only after reaching prestige 50 in a seeded test farm (or a lower test-only threshold override, see the existing pattern for other prestige-gated e2e tests). - ---- - -## Phase 4 — Secondary leaderboards - -### Design - -`store/farm.py::leaderboard()` today does one thing: rank by `economy.farm_score` over the full in-memory `_farms().find()` set. Add sibling ranking functions over the **same already-loaded set** (compute all boards from one query when a caller needs more than one, to avoid N separate full-table scans) rather than N new queries. - -### New `game_farms` columns - -- `harvests_week=0`, `harvests_week_start=""` (ISO date of the current tracking week's Monday) — incremented via `credit_farm`'s `harvests` param already threaded in Phase 0; lazily reset to 0 when `now`'s ISO week differs from `harvests_week_start`, checked at the same lazy point as `_auto_harvest`/`charge_upkeep` inside `serialize_farm`. -- `prestiged_at=""` (str timestamp) — set every time `prestige()` runs. -- `last_kernel_harvest_prestige=-1` (int, `-1` sentinel meaning "never"), `time_to_kernel_seconds=0` — updated in `harvest()`/`_auto_harvest()` when the harvested `crop.key == "kernel"` and `farm["last_kernel_harvest_prestige"] != farm["prestige"]`: `seconds = (now - parse(prestiged_at)).total_seconds()`, store both fields. - -### `economy/scoring.py` additions - -``` -FAIR_PLAY_ACTIVITY_WEIGHT = 50 -FAIR_PLAY_HOARD_DIVISOR = 200_000 -MIN_RAIDS_FOR_EFFICIENCY_BOARD = 3 - -def fair_play_score(harvests_week: int, coins: int) -> int: - return harvests_week * FAIR_PLAY_ACTIVITY_WEIGHT - min(coins, 10**9) // FAIR_PLAY_HOARD_DIVISOR -``` - -### `store/farm.py` additions - -- `leaderboard_prestige(limit=25)` — sort by `(prestige, stars)` desc. -- `leaderboard_harvests_week(limit=25)` — sort by `harvests_week` desc. -- `leaderboard_raid_efficiency(limit=25)` — `SELECT thief_uid, COUNT(*) as raids, SUM(coins) as total FROM game_steals GROUP BY thief_uid HAVING raids >= MIN_RAIDS_FOR_EFFICIENCY_BOARD ORDER BY (total * 1.0 / raids) DESC LIMIT :limit`. **Scope decision, stated explicitly:** this ranks average coins per *successful* raid, not "coins stolen / raids attempted" as literally written in the critique — failed attempts (blocked by cooldown or protection) are never persisted today, and adding a write on every failed attempt would be an easy target for script-spam with no real gameplay value. The metric is renamed "Raid Efficiency" and documented with this caveat in `docs_api` and the game's own leaderboard UI tooltip, so it is never silently misleading. -- `leaderboard_time_to_kernel(limit=25)` — filters `last_kernel_harvest_prestige == prestige and prestige > 0`, sorts `time_to_kernel_seconds` ascending. -- `leaderboard_fair_play(limit=25)` — sorts by `economy.fair_play_score(harvests_week, coins)` descending. - -### Route - -Extend the existing `GET /game/leaderboard` (do not add a new path — the critique's "parallel boards updated live" is one endpoint with a selector, matching how `admin.ai-usage.{hours}` already parameterizes a topic by value rather than by new routes): add `board: str = Query("score")` to `game_leaderboard` in `routers/game/index.py`, dispatch through a `{name: function}` map, default `"score"` reproduces exactly today's response for any caller that doesn't pass the parameter — **zero behavior change for existing Devii/docs/frontend callers that omit it.** - -### Schema - -`GameLeaderboardEntryOut` gains, all defaulted to a neutral value so the existing `score` board's JSON is unaffected: `raid_avg: float = 0.0`, `time_to_kernel_seconds: int = 0`, `fair_score: int = 0`, `title: str = ""` (from Phase 2c). - -### Frontend - -`game.html`'s `data-game-leaderboard` host gets a `